Merge branch 'develop' of https://github.com/beego/beego into fix-router-method-expression

CHANGELOG.md

@@ -11,4 +11,7 @@
 - Support session Filter chain. [4404](https://github.com/beego/beego/pull/4404)
 - Feature issue #4402 finish router get example. [4416](https://github.com/beego/beego/pull/4416)
 - Implement context.Context support and deprecate `QueryM2MWithCtx` and `QueryTableWithCtx` [4424](https://github.com/beego/beego/pull/4424)
-- Finish timeout option for tasks #4441 [4441](https://github.com/beego/beego/pull/4441)
+- Finish timeout option for tasks #4441 [4441](https://github.com/beego/beego/pull/4441)
+- Error Module brief design & using httplib module to validate this design. [4453](https://github.com/beego/beego/pull/4453)
+- Fix 4444: panic when 404 not found. [4446](https://github.com/beego/beego/pull/4446)
+- Fix 4435: fix panic when controller dir not found. [4452](https://github.com/beego/beego/pull/4452)

ERROR_SPECIFICATION.md

@@ -0,0 +1 @@
+# Error Module

client/httplib/error_code.go

@@ -0,0 +1,131 @@
+// Copyright 2020 beego
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httplib
+
+import (
+	"github.com/beego/beego/v2/core/berror"
+)
+
+var InvalidUrl = berror.DefineCode(4001001, moduleName, "InvalidUrl", `
+You pass a invalid url to httplib module. Please check your url, be careful about special character. 
+`)
+
+var InvalidUrlProtocolVersion = berror.DefineCode(4001002, moduleName, "InvalidUrlProtocolVersion", `
+You pass a invalid protocol version. In practice, we use HTTP/1.0, HTTP/1.1, HTTP/1.2
+But something like HTTP/3.2 is valid for client, and the major version is 3, minor version is 2.
+but you must confirm that server support those abnormal protocol version.
+`)
+
+var UnsupportedBodyType = berror.DefineCode(4001003, moduleName, "UnsupportedBodyType", `
+You use a invalid data as request body.
+For now, we only support type string and byte[].
+`)
+
+var InvalidXMLBody = berror.DefineCode(4001004, moduleName, "InvalidXMLBody", `
+You pass invalid data which could not be converted to XML documents. In general, if you pass structure, it works well.
+Sometimes you got XML document and you want to make it as request body. So you call XMLBody.
+If you do this, you got this code. Instead, you should call Header to set Content-type and call Body to set body data.
+`)
+
+var InvalidYAMLBody = berror.DefineCode(4001005, moduleName, "InvalidYAMLBody", `
+You pass invalid data which could not be converted to YAML documents. In general, if you pass structure, it works well.
+Sometimes you got YAML document and you want to make it as request body. So you call YAMLBody.
+If you do this, you got this code. Instead, you should call Header to set Content-type and call Body to set body data.
+`)
+
+var InvalidJSONBody = berror.DefineCode(4001006, moduleName, "InvalidJSONBody", `
+You pass invalid data which could not be converted to JSON documents. In general, if you pass structure, it works well.
+Sometimes you got JSON document and you want to make it as request body. So you call JSONBody.
+If you do this, you got this code. Instead, you should call Header to set Content-type and call Body to set body data.
+`)
+
+
+// start with 5 --------------------------------------------------------------------------
+
+var CreateFormFileFailed = berror.DefineCode(5001001, moduleName, "CreateFormFileFailed", `
+In normal case than handling files with BeegoRequest, you should not see this error code.
+Unexpected EOF, invalid characters, bad file descriptor may cause this error.
+`)
+
+var ReadFileFailed = berror.DefineCode(5001002, moduleName, "ReadFileFailed", `
+There are several cases that cause this error:
+1. file not found. Please check the file name;
+2. file not found, but file name is correct. If you use relative file path, it's very possible for you to see this code.
+make sure that this file is in correct directory which Beego looks for;
+3. Beego don't have the privilege to read this file, please change file mode; 
+`)
+
+var CopyFileFailed = berror.DefineCode(5001003, moduleName, "CopyFileFailed", `
+When we try to read file content and then copy it to another writer, and failed.
+1. Unexpected EOF;
+2. Bad file descriptor;
+3. Write conflict;
+
+Please check your file content, and confirm that file is not processed by other process (or by user manually).
+`)
+
+var CloseFileFailed = berror.DefineCode(5001004, moduleName, "CloseFileFailed", `
+After handling files, Beego try to close file but failed. Usually it was caused by bad file descriptor.
+`)
+
+var SendRequestFailed = berror.DefineCode(5001005, moduleName, "SendRequestRetryExhausted", `
+Beego send HTTP request, but it failed.
+If you config retry times, it means that Beego had retried and failed.
+When you got this error, there are vary kind of reason:
+1. Network unstable and timeout. In this case, sometimes server has received the request.
+2. Server error. Make sure that server works well.
+3. The request is invalid, which means that you pass some invalid parameter.
+`)
+
+var ReadGzipBodyFailed = berror.DefineCode(5001006, moduleName, "BuildGzipReaderFailed", `
+Beego parse gzip-encode body failed. Usually Beego got invalid response.
+Please confirm that server returns gzip data.
+`)
+
+var CreateFileIfNotExistFailed = berror.DefineCode(5001007, moduleName, "CreateFileIfNotExist", `
+Beego want to create file if not exist and failed. 
+In most cases, it means that Beego doesn't have the privilege to create this file.
+Please change file mode to ensure that Beego is able to create files on specific directory.
+Or you can run Beego with higher authority.
+In some cases, you pass invalid filename. Make sure that the file name is valid on your system.
+`)
+
+var UnmarshalJSONResponseToObjectFailed = berror.DefineCode(5001008, moduleName,
+	"UnmarshalResponseToObjectFailed", `
+Beego trying to unmarshal response's body to structure but failed.
+Make sure that:
+1. You pass valid structure pointer to the function;
+2. The body is valid json document
+`)
+
+var UnmarshalXMLResponseToObjectFailed = berror.DefineCode(5001009, moduleName,
+	"UnmarshalResponseToObjectFailed", `
+Beego trying to unmarshal response's body to structure but failed.
+Make sure that:
+1. You pass valid structure pointer to the function;
+2. The body is valid XML document
+`)
+
+var UnmarshalYAMLResponseToObjectFailed = berror.DefineCode(5001010, moduleName,
+	"UnmarshalResponseToObjectFailed", `
+Beego trying to unmarshal response's body to structure but failed.
+Make sure that:
+1. You pass valid structure pointer to the function;
+2. The body is valid YAML document
+`)
+
+
+
+

client/httplib/filter/prometheus/filter.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -26,24 +27,30 @@ import (
 )
 
 type FilterChainBuilder struct {
-	summaryVec prometheus.ObserverVec
 	AppName    string
 	ServerName string
 	RunMode    string
 }
 
+var summaryVec prometheus.ObserverVec
+var initSummaryVec sync.Once
+
 func (builder *FilterChainBuilder) FilterChain(next httplib.Filter) httplib.Filter {
 
-	builder.summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+	initSummaryVec.Do(func() {
-		Name:      "beego",
+		summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Subsystem: "remote_http_request",
+			Name:      "beego",
-		ConstLabels: map[string]string{
+			Subsystem: "remote_http_request",
-			"server":  builder.ServerName,
+			ConstLabels: map[string]string{
-			"env":     builder.RunMode,
+				"server":  builder.ServerName,
-			"appname": builder.AppName,
+				"env":     builder.RunMode,
-		},
+				"appname": builder.AppName,
-		Help: "The statics info for remote http requests",
+			},
-	}, []string{"proto", "scheme", "method", "host", "path", "status", "isError"})
+			Help: "The statics info for remote http requests",
+		}, []string{"proto", "scheme", "method", "host", "path", "status", "isError"})
+
+		prometheus.MustRegister(summaryVec)
+	})
 
 	return func(ctx context.Context, req *httplib.BeegoHTTPRequest) (*http.Response, error) {
 		startTime := time.Now()
@@ -72,6 +79,6 @@ func (builder *FilterChainBuilder) report(startTime time.Time, endTime time.Time
 
 	dur := int(endTime.Sub(startTime) / time.Millisecond)
 
-	builder.summaryVec.WithLabelValues(proto, scheme, method, host, path,
+	summaryVec.WithLabelValues(proto, scheme, method, host, path,
 		strconv.Itoa(status), strconv.FormatBool(err != nil)).Observe(float64(dur))
 }

client/httplib/httplib.go

@@ -40,7 +40,6 @@ import (
 	"encoding/xml"
 	"io"
 	"io/ioutil"
-	"log"
 	"mime/multipart"
 	"net"
 	"net/http"
@@ -52,8 +51,10 @@ import (
 	"time"
 
 	"gopkg.in/yaml.v2"
-)
 
+	"github.com/beego/beego/v2/core/berror"
+	"github.com/beego/beego/v2/core/logs"
+)
 
 // it will be the last filter and execute request.Do
 var doRequestFilter = func(ctx context.Context, req *BeegoHTTPRequest) (*http.Response, error) {
@@ -61,11 +62,14 @@ var doRequestFilter = func(ctx context.Context, req *BeegoHTTPRequest) (*http.Re
 }
 
 // NewBeegoRequest returns *BeegoHttpRequest with specific method
+// TODO add error as return value
+// I think if we don't return error
+// users are hard to check whether we create Beego request successfully
 func NewBeegoRequest(rawurl, method string) *BeegoHTTPRequest {
 	var resp http.Response
 	u, err := url.Parse(rawurl)
 	if err != nil {
-		log.Println("Httplib:", err)
+		logs.Error("%+v", berror.Wrapf(err, InvalidUrl, "invalid raw url: %s", rawurl))
 	}
 	req := http.Request{
 		URL:        u,
@@ -110,8 +114,6 @@ func Head(url string) *BeegoHTTPRequest {
 	return NewBeegoRequest(url, "HEAD")
 }
 
-
-
 // BeegoHTTPRequest provides more useful methods than http.Request for requesting a url.
 type BeegoHTTPRequest struct {
 	url     string
@@ -211,7 +213,7 @@ func (b *BeegoHTTPRequest) SetHost(host string) *BeegoHTTPRequest {
 }
 
 // SetProtocolVersion sets the protocol version for incoming requests.
-// Client requests always use HTTP/1.1.
+// Client requests always use HTTP/1.1
 func (b *BeegoHTTPRequest) SetProtocolVersion(vers string) *BeegoHTTPRequest {
 	if len(vers) == 0 {
 		vers = "HTTP/1.1"
@@ -222,8 +224,9 @@ func (b *BeegoHTTPRequest) SetProtocolVersion(vers string) *BeegoHTTPRequest {
 		b.req.Proto = vers
 		b.req.ProtoMajor = major
 		b.req.ProtoMinor = minor
+		return b
 	}
-
+	logs.Error("%+v", berror.Errorf(InvalidUrlProtocolVersion, "invalid protocol: %s", vers))
 	return b
 }
 
@@ -291,6 +294,7 @@ func (b *BeegoHTTPRequest) PostFile(formname, filename string) *BeegoHTTPRequest
 
 // Body adds request raw body.
 // Supports string and []byte.
+// TODO return error if data is invalid
 func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest {
 	switch t := data.(type) {
 	case string:
@@ -307,6 +311,8 @@ func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest {
 			return ioutil.NopCloser(bf), nil
 		}
 		b.req.ContentLength = int64(len(t))
+	default:
+		logs.Error("%+v", berror.Errorf(UnsupportedBodyType, "unsupported body data type: %s", t))
 	}
 	return b
 }
@@ -316,7 +322,7 @@ func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
 	if b.req.Body == nil && obj != nil {
 		byts, err := xml.Marshal(obj)
 		if err != nil {
-			return b, err
+			return b, berror.Wrap(err, InvalidXMLBody, "obj could not be converted to XML data")
 		}
 		b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
 		b.req.GetBody = func() (io.ReadCloser, error) {
@@ -333,7 +339,7 @@ func (b *BeegoHTTPRequest) YAMLBody(obj interface{}) (*BeegoHTTPRequest, error)
 	if b.req.Body == nil && obj != nil {
 		byts, err := yaml.Marshal(obj)
 		if err != nil {
-			return b, err
+			return b, berror.Wrap(err, InvalidYAMLBody, "obj could not be converted to YAML data")
 		}
 		b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
 		b.req.ContentLength = int64(len(byts))
@@ -347,7 +353,7 @@ func (b *BeegoHTTPRequest) JSONBody(obj interface{}) (*BeegoHTTPRequest, error)
 	if b.req.Body == nil && obj != nil {
 		byts, err := json.Marshal(obj)
 		if err != nil {
-			return b, err
+			return b, berror.Wrap(err, InvalidJSONBody, "obj could not be converted to JSON body")
 		}
 		b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
 		b.req.ContentLength = int64(len(byts))
@@ -375,28 +381,15 @@ func (b *BeegoHTTPRequest) buildURL(paramBody string) {
 			bodyWriter := multipart.NewWriter(pw)
 			go func() {
 				for formname, filename := range b.files {
-					fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
+					b.handleFileToBody(bodyWriter, formname, filename)
-					if err != nil {
-						log.Println("Httplib:", err)
-					}
-					fh, err := os.Open(filename)
-					if err != nil {
-						log.Println("Httplib:", err)
-					}
-					// iocopy
-					_, err = io.Copy(fileWriter, fh)
-					fh.Close()
-					if err != nil {
-						log.Println("Httplib:", err)
-					}
 				}
 				for k, v := range b.params {
 					for _, vv := range v {
-						bodyWriter.WriteField(k, vv)
+						_ = bodyWriter.WriteField(k, vv)
 					}
 				}
-				bodyWriter.Close()
+				_ = bodyWriter.Close()
-				pw.Close()
+				_ = pw.Close()
 			}()
 			b.Header("Content-Type", bodyWriter.FormDataContentType())
 			b.req.Body = ioutil.NopCloser(pr)
@@ -412,6 +405,29 @@ func (b *BeegoHTTPRequest) buildURL(paramBody string) {
 	}
 }
 
+func (b *BeegoHTTPRequest) handleFileToBody(bodyWriter *multipart.Writer, formname string, filename string) {
+	fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
+	const errFmt = "Httplib: %+v"
+	if err != nil {
+		logs.Error(errFmt, berror.Wrapf(err, CreateFormFileFailed,
+			"could not create form file, formname: %s, filename: %s", formname, filename))
+	}
+	fh, err := os.Open(filename)
+
+	if err != nil {
+		logs.Error(errFmt, berror.Wrapf(err, ReadFileFailed, "could not open this file %s", filename))
+	}
+	// iocopy
+	_, err = io.Copy(fileWriter, fh)
+	if err != nil {
+		logs.Error(errFmt, berror.Wrapf(err, CopyFileFailed, "could not copy this file %s", filename))
+	}
+	err = fh.Close()
+	if err != nil {
+		logs.Error(errFmt, berror.Wrapf(err, CloseFileFailed, "could not close this file %s", filename))
+	}
+}
+
 func (b *BeegoHTTPRequest) getResponse() (*http.Response, error) {
 	if b.resp.StatusCode != 0 {
 		return b.resp, nil
@@ -440,62 +456,20 @@ func (b *BeegoHTTPRequest) DoRequestWithCtx(ctx context.Context) (resp *http.Res
 	return root(ctx, b)
 }
 
-func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (resp *http.Response, err error) {
+func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (*http.Response, error) {
-	var paramBody string
+	paramBody := b.buildParamBody()
-	if len(b.params) > 0 {
-		var buf bytes.Buffer
-		for k, v := range b.params {
-			for _, vv := range v {
-				buf.WriteString(url.QueryEscape(k))
-				buf.WriteByte('=')
-				buf.WriteString(url.QueryEscape(vv))
-				buf.WriteByte('&')
-			}
-		}
-		paramBody = buf.String()
-		paramBody = paramBody[0 : len(paramBody)-1]
-	}
 
 	b.buildURL(paramBody)
 	urlParsed, err := url.Parse(b.url)
 	if err != nil {
-		return nil, err
+		return nil, berror.Wrapf(err, InvalidUrl, "parse url failed, the url is %s", b.url)
 	}
 
 	b.req.URL = urlParsed
 
-	trans := b.setting.Transport
+	trans := b.buildTrans()
 
-	if trans == nil {
+	jar := b.buildCookieJar()
-		// create default transport
-		trans = &http.Transport{
-			TLSClientConfig:     b.setting.TLSClientConfig,
-			Proxy:               b.setting.Proxy,
-			Dial:                TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
-			MaxIdleConnsPerHost: 100,
-		}
-	} else {
-		// if b.transport is *http.Transport then set the settings.
-		if t, ok := trans.(*http.Transport); ok {
-			if t.TLSClientConfig == nil {
-				t.TLSClientConfig = b.setting.TLSClientConfig
-			}
-			if t.Proxy == nil {
-				t.Proxy = b.setting.Proxy
-			}
-			if t.Dial == nil {
-				t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
-			}
-		}
-	}
-
-	var jar http.CookieJar
-	if b.setting.EnableCookie {
-		if defaultCookieJar == nil {
-			createDefaultCookie()
-		}
-		jar = defaultCookieJar
-	}
 
 	client := &http.Client{
 		Transport: trans,
@@ -511,12 +485,16 @@ func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (resp *http.Response,
 	}
 
 	if b.setting.ShowDebug {
-		dump, err := httputil.DumpRequest(b.req, b.setting.DumpBody)
+		dump, e := httputil.DumpRequest(b.req, b.setting.DumpBody)
-		if err != nil {
+		if e != nil {
-			log.Println(err.Error())
+			logs.Error("%+v", e)
 		}
 		b.dump = dump
 	}
+	return b.sendRequest(client)
+}
+
+func (b *BeegoHTTPRequest) sendRequest(client *http.Client) (resp *http.Response, err error) {
 	// retries default value is 0, it will run once.
 	// retries equal to -1, it will run forever until success
 	// retries is setted, it will retries fixed times.
@@ -524,11 +502,68 @@ func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (resp *http.Response,
 	for i := 0; b.setting.Retries == -1 || i <= b.setting.Retries; i++ {
 		resp, err = client.Do(b.req)
 		if err == nil {
-			break
+			return
 		}
 		time.Sleep(b.setting.RetryDelay)
 	}
-	return resp, err
+	return nil, berror.Wrap(err, SendRequestFailed, "sending request fail")
+}
+
+func (b *BeegoHTTPRequest) buildCookieJar() http.CookieJar {
+	var jar http.CookieJar
+	if b.setting.EnableCookie {
+		if defaultCookieJar == nil {
+			createDefaultCookie()
+		}
+		jar = defaultCookieJar
+	}
+	return jar
+}
+
+func (b *BeegoHTTPRequest) buildTrans() http.RoundTripper {
+	trans := b.setting.Transport
+
+	if trans == nil {
+		// create default transport
+		trans = &http.Transport{
+			TLSClientConfig:     b.setting.TLSClientConfig,
+			Proxy:               b.setting.Proxy,
+			DialContext:         TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
+			MaxIdleConnsPerHost: 100,
+		}
+	} else {
+		// if b.transport is *http.Transport then set the settings.
+		if t, ok := trans.(*http.Transport); ok {
+			if t.TLSClientConfig == nil {
+				t.TLSClientConfig = b.setting.TLSClientConfig
+			}
+			if t.Proxy == nil {
+				t.Proxy = b.setting.Proxy
+			}
+			if t.DialContext == nil {
+				t.DialContext = TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
+			}
+		}
+	}
+	return trans
+}
+
+func (b *BeegoHTTPRequest) buildParamBody() string {
+	var paramBody string
+	if len(b.params) > 0 {
+		var buf bytes.Buffer
+		for k, v := range b.params {
+			for _, vv := range v {
+				buf.WriteString(url.QueryEscape(k))
+				buf.WriteByte('=')
+				buf.WriteString(url.QueryEscape(vv))
+				buf.WriteByte('&')
+			}
+		}
+		paramBody = buf.String()
+		paramBody = paramBody[0 : len(paramBody)-1]
+	}
+	return paramBody
 }
 
 // String returns the body string in response.
@@ -559,10 +594,10 @@ func (b *BeegoHTTPRequest) Bytes() ([]byte, error) {
 	if b.setting.Gzip && resp.Header.Get("Content-Encoding") == "gzip" {
 		reader, err := gzip.NewReader(resp.Body)
 		if err != nil {
-			return nil, err
+			return nil, berror.Wrap(err, ReadGzipBodyFailed, "building gzip reader failed")
 		}
 		b.body, err = ioutil.ReadAll(reader)
-		return b.body, err
+		return b.body, berror.Wrap(err, ReadGzipBodyFailed, "reading gzip data failed")
 	}
 	b.body, err = ioutil.ReadAll(resp.Body)
 	return b.body, err
@@ -605,7 +640,7 @@ func pathExistAndMkdir(filename string) (err error) {
 			return nil
 		}
 	}
-	return err
+	return berror.Wrapf(err, CreateFileIfNotExistFailed, "try to create(if not exist) failed: %s", filename)
 }
 
 // ToJSON returns the map that marshals from the body bytes as json in response.
@@ -615,7 +650,8 @@ func (b *BeegoHTTPRequest) ToJSON(v interface{}) error {
 	if err != nil {
 		return err
 	}
-	return json.Unmarshal(data, v)
+	return berror.Wrap(json.Unmarshal(data, v),
+		UnmarshalJSONResponseToObjectFailed, "unmarshal json body to object failed.")
 }
 
 // ToXML returns the map that marshals from the body bytes as xml in response .
@@ -625,7 +661,8 @@ func (b *BeegoHTTPRequest) ToXML(v interface{}) error {
 	if err != nil {
 		return err
 	}
-	return xml.Unmarshal(data, v)
+	return berror.Wrap(xml.Unmarshal(data, v),
+		UnmarshalXMLResponseToObjectFailed, "unmarshal xml body to object failed.")
 }
 
 // ToYAML returns the map that marshals from the body bytes as yaml in response .
@@ -635,7 +672,8 @@ func (b *BeegoHTTPRequest) ToYAML(v interface{}) error {
 	if err != nil {
 		return err
 	}
-	return yaml.Unmarshal(data, v)
+	return berror.Wrap(yaml.Unmarshal(data, v),
+		UnmarshalYAMLResponseToObjectFailed, "unmarshal yaml body to object failed.")
 }
 
 // Response executes request client gets response manually.
@@ -644,8 +682,18 @@ func (b *BeegoHTTPRequest) Response() (*http.Response, error) {
 }
 
 // TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field.
+// Deprecated
+// we will move this at the end of 2021
+// please use TimeoutDialerCtx
 func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
 	return func(netw, addr string) (net.Conn, error) {
+		return TimeoutDialerCtx(cTimeout, rwTimeout)(context.Background(), netw, addr)
+	}
+}
+
+func TimeoutDialerCtx(cTimeout time.Duration,
+	rwTimeout time.Duration) func(ctx context.Context, net, addr string) (c net.Conn, err error) {
+	return func(ctx context.Context, netw, addr string) (net.Conn, error) {
 		conn, err := net.DialTimeout(netw, addr, cTimeout)
 		if err != nil {
 			return nil, err

client/httplib/httplib_test.go

@@ -345,6 +345,29 @@ func TestNewBeegoRequest(t *testing.T) {
 	req := NewBeegoRequest("http://beego.me", "GET")
 	assert.NotNil(t, req)
 	assert.Equal(t, "GET", req.req.Method)
+
+	// invalid case but still go request
+	req = NewBeegoRequest("httpa\ta://beego.me", "GET")
+	assert.NotNil(t, req)
+}
+
+func TestBeegoHTTPRequest_SetProtocolVersion(t *testing.T) {
+	req := NewBeegoRequest("http://beego.me", "GET")
+	req.SetProtocolVersion("HTTP/3.10")
+	assert.Equal(t, "HTTP/3.10", req.req.Proto)
+	assert.Equal(t, 3, req.req.ProtoMajor)
+	assert.Equal(t, 10, req.req.ProtoMinor)
+
+	req.SetProtocolVersion("")
+	assert.Equal(t, "HTTP/1.1", req.req.Proto)
+	assert.Equal(t, 1, req.req.ProtoMajor)
+	assert.Equal(t, 1, req.req.ProtoMinor)
+
+	// invalid case
+	req.SetProtocolVersion("HTTP/aaa1.1")
+	assert.Equal(t, "HTTP/1.1", req.req.Proto)
+	assert.Equal(t, 1, req.req.ProtoMajor)
+	assert.Equal(t, 1, req.req.ProtoMinor)
 }
 
 func TestPut(t *testing.T) {
@@ -384,6 +407,16 @@ func TestBeegoHTTPRequest_Body(t *testing.T) {
 	req.Body([]byte(body))
 	assert.Equal(t, int64(len(body)), req.req.ContentLength)
 	assert.NotNil(t, req.req.GetBody)
+	assert.NotNil(t, req.req.Body)
+
+	body = "hhhh, i am test"
+	req.Body(body)
+	assert.Equal(t, int64(len(body)), req.req.ContentLength)
+	assert.NotNil(t, req.req.GetBody)
+	assert.NotNil(t, req.req.Body)
+
+	// invalid case
+	req.Body(13)
 }
 
 

client/httplib/module.go

@@ -0,0 +1,17 @@
+// Copyright 2021 beego
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httplib
+
+const moduleName = "httplib"

client/orm/filter/prometheus/filter.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -33,24 +34,29 @@ import (
 // if we want to records the metrics of QuerySetter
 // actually we only records metrics of invoking "QueryTable"
 type FilterChainBuilder struct {
-	summaryVec prometheus.ObserverVec
 	AppName    string
 	ServerName string
 	RunMode    string
 }
 
+var summaryVec prometheus.ObserverVec
+var initSummaryVec sync.Once
+
 func (builder *FilterChainBuilder) FilterChain(next orm.Filter) orm.Filter {
 
-	builder.summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+	initSummaryVec.Do(func() {
-		Name:      "beego",
+		summaryVec = prometheus.NewSummaryVec(prometheus.SummaryOpts{
-		Subsystem: "orm_operation",
+			Name:      "beego",
-		ConstLabels: map[string]string{
+			Subsystem: "orm_operation",
-			"server":  builder.ServerName,
+			ConstLabels: map[string]string{
-			"env":     builder.RunMode,
+				"server":  builder.ServerName,
-			"appname": builder.AppName,
+				"env":     builder.RunMode,
-		},
+				"appname": builder.AppName,
-		Help: "The statics info for orm operation",
+			},
-	}, []string{"method", "name", "insideTx", "txName"})
+			Help: "The statics info for orm operation",
+		}, []string{"method", "name", "insideTx", "txName"})
+		prometheus.MustRegister(summaryVec)
+	})
 
 	return func(ctx context.Context, inv *orm.Invocation) []interface{} {
 		startTime := time.Now()
@@ -74,12 +80,12 @@ func (builder *FilterChainBuilder) report(ctx context.Context, inv *orm.Invocati
 		builder.reportTxn(ctx, inv)
 		return
 	}
-	builder.summaryVec.WithLabelValues(inv.Method, inv.GetTableName(),
+	summaryVec.WithLabelValues(inv.Method, inv.GetTableName(),
 		strconv.FormatBool(inv.InsideTx), inv.TxName).Observe(float64(dur))
 }
 
 func (builder *FilterChainBuilder) reportTxn(ctx context.Context, inv *orm.Invocation) {
 	dur := time.Now().Sub(inv.TxStartTime) / time.Millisecond
-	builder.summaryVec.WithLabelValues(inv.Method, inv.TxName,
+	summaryVec.WithLabelValues(inv.Method, inv.TxName,
 		strconv.FormatBool(inv.InsideTx), inv.TxName).Observe(float64(dur))
 }

client/orm/filter/prometheus/filter_test.go

@@ -32,7 +32,7 @@ func TestFilterChainBuilder_FilterChain1(t *testing.T) {
 	builder := &FilterChainBuilder{}
 	filter := builder.FilterChain(next)
 
-	assert.NotNil(t, builder.summaryVec)
+	assert.NotNil(t, summaryVec)
 	assert.NotNil(t, filter)
 
 	inv := &orm.Invocation{}

core/berror/codes.go

@@ -0,0 +1,91 @@
+// Copyright 2020 beego
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package berror
+
+import (
+	"fmt"
+	"sync"
+)
+
+
+
+// A Code is an unsigned 32-bit error code as defined in the beego spec.
+type Code interface {
+	Code() uint32
+	Module() string
+	Desc() string
+	Name() string
+}
+
+var defaultCodeRegistry = &codeRegistry{
+	codes: make(map[uint32]*codeDefinition, 127),
+}
+
+// DefineCode defining a new Code
+// Before defining a new code, please read Beego specification.
+// desc could be markdown doc
+func DefineCode(code uint32, module string, name string, desc string) Code {
+	res := &codeDefinition{
+		code: code,
+		module: module,
+		desc: desc,
+	}
+	defaultCodeRegistry.lock.Lock()
+	defer defaultCodeRegistry.lock.Unlock()
+
+	if _, ok := defaultCodeRegistry.codes[code]; ok {
+		panic(fmt.Sprintf("duplicate code, code %d has been registered", code))
+	}
+	defaultCodeRegistry.codes[code] = res
+	return res
+}
+
+type codeRegistry struct {
+	lock sync.RWMutex
+	codes map[uint32]*codeDefinition
+}
+
+
+func (cr *codeRegistry) Get(code uint32) (Code, bool) {
+	cr.lock.RLock()
+	defer cr.lock.RUnlock()
+	c, ok := cr.codes[code]
+	return c, ok
+}
+
+type codeDefinition struct {
+	code uint32
+	module string
+	desc string
+	name string
+}
+
+
+func (c *codeDefinition) Name() string {
+	return c.name
+}
+
+func (c *codeDefinition) Code() uint32 {
+	return c.code
+}
+
+func (c *codeDefinition) Module() string {
+	return c.module
+}
+
+func (c *codeDefinition) Desc() string {
+	return c.desc
+}
+

core/berror/error.go

@@ -0,0 +1,69 @@
+// Copyright 2020 beego
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package berror
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// code, msg
+const errFmt = "ERROR-%d, %s"
+
+// Err returns an error representing c and msg.  If c is OK, returns nil.
+func Error(c Code, msg string) error {
+	return fmt.Errorf(errFmt, c.Code(), msg)
+}
+
+// Errorf returns error
+func Errorf(c Code, format string, a ...interface{}) error {
+	return Error(c, fmt.Sprintf(format, a...))
+}
+
+func Wrap(err error, c Code, msg string) error {
+	if err == nil {
+		return nil
+	}
+	return errors.Wrap(err, fmt.Sprintf(errFmt, c.Code(), msg))
+}
+
+func Wrapf(err error, c Code, format string, a ...interface{}) error {
+	return Wrap(err, c, fmt.Sprintf(format, a...))
+}
+
+// FromError is very simple. It just parse error msg and check whether code has been register
+// if code not being register, return unknown
+// if err.Error() is not valid beego error code, return unknown
+func FromError(err error) (Code, bool) {
+	msg := err.Error()
+	codeSeg := strings.SplitN(msg, ",", 2)
+	if strings.HasPrefix(codeSeg[0], "ERROR-") {
+		codeStr := strings.SplitN(codeSeg[0], "-", 2)
+		if len(codeStr) < 2 {
+			return Unknown, false
+		}
+		codeInt, e := strconv.ParseUint(codeStr[1], 10, 32)
+		if e != nil {
+			return Unknown, false
+		}
+		if code, ok := defaultCodeRegistry.Get(uint32(codeInt)); ok {
+			return code, true
+		}
+	}
+	return Unknown, false
+}

core/berror/error_test.go

@@ -0,0 +1,77 @@
+// Copyright 2020 beego
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package berror
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var testCode1 = DefineCode(1, "unit_test", "TestError", "Hello, test code1")
+
+var testErr = errors.New("hello, this is error")
+
+func TestErrorf(t *testing.T) {
+	msg := Errorf(testCode1, "errorf %s", "aaaa")
+	assert.NotNil(t, msg)
+	assert.Equal(t, "ERROR-1, errorf aaaa", msg.Error())
+}
+
+func TestWrapf(t *testing.T) {
+	err := Wrapf(testErr, testCode1, "Wrapf %s", "aaaa")
+	assert.NotNil(t, err)
+	assert.True(t, errors.Is(err, testErr))
+}
+
+func TestFromError(t *testing.T) {
+	err := errors.New("ERROR-1, errorf aaaa")
+	code, ok := FromError(err)
+	assert.True(t, ok)
+	assert.Equal(t, testCode1, code)
+	assert.Equal(t, "unit_test", code.Module())
+	assert.Equal(t, "Hello, test code1", code.Desc())
+
+	err = errors.New("not beego error")
+	code, ok = FromError(err)
+	assert.False(t, ok)
+	assert.Equal(t, Unknown, code)
+
+	err = errors.New("ERROR-2, not register")
+	code, ok = FromError(err)
+	assert.False(t, ok)
+	assert.Equal(t, Unknown, code)
+
+	err = errors.New("ERROR-aaa, invalid code")
+	code, ok = FromError(err)
+	assert.False(t, ok)
+	assert.Equal(t, Unknown, code)
+
+	err = errors.New("aaaaaaaaaaaaaa")
+	code, ok = FromError(err)
+	assert.False(t, ok)
+	assert.Equal(t, Unknown, code)
+
+	err = errors.New("ERROR-2-3, invalid error")
+	code, ok = FromError(err)
+	assert.False(t, ok)
+	assert.Equal(t, Unknown, code)
+
+	err = errors.New("ERROR, invalid error")
+	code, ok = FromError(err)
+	assert.False(t, ok)
+	assert.Equal(t, Unknown, code)
+}

core/berror/pre_define_code.go

@@ -0,0 +1,52 @@
+// Copyright 2021 beego
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package berror
+
+import (
+	"fmt"
+)
+
+// pre define code
+
+// Unknown indicates got some error which is not defined
+var Unknown = DefineCode(5000001, "error", "Unknown",fmt.Sprintf(`
+Unknown error code. Usually you will see this code in three cases:
+1. You forget to define Code or function DefineCode not being executed;
+2. This is not Beego's error but you call FromError();
+3. Beego got unexpected error and don't know how to handle it, and then return Unknown error
+
+A common practice to DefineCode looks like:
+%s
+
+In this way, you may forget to import this package, and got Unknown error. 
+
+Sometimes, you believe you got Beego error, but actually you don't, and then you call FromError(err)
+
+`, goCodeBlock(`
+import your_package
+
+func init() {
+    DefineCode(5100100, "your_module", "detail")
+    // ...
+}
+`)))
+
+func goCodeBlock(code string) string {
+	return codeBlock("go", code)
+}
+func codeBlock(lan string, code string) string {
+	return fmt.Sprintf("```%s\n%s\n```", lan, code)
+}
+

server/web/admin.go

@@ -108,8 +108,11 @@ func registerAdmin() error {
 		c := &adminController{
 			servers: make([]*HttpServer, 0, 2),
 		}
+
+		// copy config to avoid conflict
+		adminCfg := *BConfig
 		beeAdminApp = &adminApp{
-			HttpServer: NewHttpServerWithCfg(BConfig),
+			HttpServer: NewHttpServerWithCfg(&adminCfg),
 		}
 		// keep in mind that all data should be html escaped to avoid XSS attack
 		beeAdminApp.Router("/", c, "get:AdminIndex")

server/web/filter/prometheus/filter.go

@@ -17,23 +17,49 @@ package prometheus
 import (
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/beego/beego/v2"
+	"github.com/beego/beego/v2/core/logs"
 	"github.com/beego/beego/v2/server/web"
 	"github.com/beego/beego/v2/server/web/context"
 )
 
+const unknownRouterPattern = "UnknownRouterPattern"
+
 // FilterChainBuilder is an extension point,
 // when we want to support some configuration,
 // please use this structure
 type FilterChainBuilder struct {
 }
 
+var summaryVec prometheus.ObserverVec
+var initSummaryVec sync.Once
+
 // FilterChain returns a FilterFunc. The filter will records some metrics
 func (builder *FilterChainBuilder) FilterChain(next web.FilterFunc) web.FilterFunc {
+
+	initSummaryVec.Do(func() {
+		summaryVec = builder.buildVec()
+		err := prometheus.Register(summaryVec)
+		if _, ok := err.(*prometheus.AlreadyRegisteredError); err != nil && !ok {
+			logs.Error("web module register prometheus vector failed, %+v", err)
+		}
+		registerBuildInfo()
+	})
+
+	return func(ctx *context.Context) {
+		startTime := time.Now()
+		next(ctx)
+		endTime := time.Now()
+		go report(endTime.Sub(startTime), ctx, summaryVec)
+	}
+}
+
+func (builder *FilterChainBuilder) buildVec() *prometheus.SummaryVec {
 	summaryVec := prometheus.NewSummaryVec(prometheus.SummaryOpts{
 		Name:      "beego",
 		Subsystem: "http_request",
@@ -44,17 +70,7 @@ func (builder *FilterChainBuilder) FilterChain(next web.FilterFunc) web.FilterFu
 		},
 		Help: "The statics info for http request",
 	}, []string{"pattern", "method", "status"})
-
+	return summaryVec
-	prometheus.MustRegister(summaryVec)
-
-	registerBuildInfo()
-
-	return func(ctx *context.Context) {
-		startTime := time.Now()
-		next(ctx)
-		endTime := time.Now()
-		go report(endTime.Sub(startTime), ctx, summaryVec)
-	}
 }
 
 func registerBuildInfo() {
@@ -75,13 +91,17 @@ func registerBuildInfo() {
 		},
 	}, []string{})
 
-	prometheus.MustRegister(buildInfo)
+	_ = prometheus.Register(buildInfo)
 	buildInfo.WithLabelValues().Set(1)
 }
 
-func report(dur time.Duration, ctx *context.Context, vec *prometheus.SummaryVec) {
+func report(dur time.Duration, ctx *context.Context, vec prometheus.ObserverVec) {
 	status := ctx.Output.Status
-	ptn := ctx.Input.GetData("RouterPattern").(string)
+	ptnItf := ctx.Input.GetData("RouterPattern")
+	ptn := unknownRouterPattern
+	if ptnItf != nil {
+		ptn = ptnItf.(string)
+	}
 	ms := dur / time.Millisecond
 	vec.WithLabelValues(ptn, ctx.Input.Method(), strconv.Itoa(status)).Observe(float64(ms))
 }

server/web/filter/prometheus/filter_test.go

@@ -40,3 +40,17 @@ func TestFilterChain(t *testing.T) {
 	assert.True(t, ctx.Input.GetData("invocation").(bool))
 	time.Sleep(1 * time.Second)
 }
+
+func TestFilterChainBuilder_report(t *testing.T) {
+
+	ctx := context.NewContext()
+	r, _ := http.NewRequest("GET", "/prometheus/user", nil)
+	w := httptest.NewRecorder()
+	ctx.Reset(w, r)
+	fb := &FilterChainBuilder{}
+	// without router info
+	report(time.Second, ctx, fb.buildVec())
+
+	ctx.Input.SetData("RouterPattern", "my-route")
+	report(time.Second, ctx, fb.buildVec())
+}

server/web/hooks.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"path/filepath"
 
+	"github.com/coreos/etcd/pkg/fileutil"
+
 	"github.com/beego/beego/v2/core/logs"
 	"github.com/beego/beego/v2/server/web/context"
 	"github.com/beego/beego/v2/server/web/session"
@@ -99,7 +101,12 @@ func registerGzip() error {
 
 func registerCommentRouter() error {
 	if BConfig.RunMode == DEV {
-		if err := parserPkg(filepath.Join(WorkPath, BConfig.WebConfig.CommentRouterPath)); err != nil {
+		ctrlDir := filepath.Join(WorkPath, BConfig.WebConfig.CommentRouterPath)
+		if !fileutil.Exist(ctrlDir) {
+			logs.Warn("controller package not found, won't generate router: ", ctrlDir)
+			return nil
+		}
+		if err := parserPkg(ctrlDir); err != nil {
 			return err
 		}
 	}