server/beego
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Define error code for httplib

Browse Source
This commit is contained in:
Ming Deng
2021-01-23 21:11:06 +08:00
parent a91a5d01d1
commit 3c0dbe2914
12 changed files with 590 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

218
client/httplib/httplib.go
View File

@@ -40,7 +40,6 @@ import (
"encoding/xml"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net"
"net/http"
@@ -52,8 +51,10 @@ import (
"time"
"gopkg.in/yaml.v2"
)
"github.com/beego/beego/v2/core/berror"
"github.com/beego/beego/v2/core/logs"
)
// it will be the last filter and execute request.Do
var doRequestFilter = func(ctx context.Context, req *BeegoHTTPRequest) (*http.Response, error) {
@@ -61,11 +62,14 @@ var doRequestFilter = func(ctx context.Context, req *BeegoHTTPRequest) (*http.Re
}
// NewBeegoRequest returns *BeegoHttpRequest with specific method
// TODO add error as return value
// I think if we don't return error
// users are hard to check whether we create Beego request successfully
func NewBeegoRequest(rawurl, method string) *BeegoHTTPRequest {
var resp http.Response
u, err := url.Parse(rawurl)
if err != nil {
log.Println("Httplib:", err)
logs.Error("%+v", berror.Wrapf(err, InvalidUrl, "invalid raw url: %s", rawurl))
}
req := http.Request{
URL: u,
@@ -110,8 +114,6 @@ func Head(url string) *BeegoHTTPRequest {
return NewBeegoRequest(url, "HEAD")
}
// BeegoHTTPRequest provides more useful methods than http.Request for requesting a url.
type BeegoHTTPRequest struct {
url string
@@ -211,7 +213,7 @@ func (b *BeegoHTTPRequest) SetHost(host string) *BeegoHTTPRequest {
}
// SetProtocolVersion sets the protocol version for incoming requests.
// Client requests always use HTTP/1.1.
// Client requests always use HTTP/1.1
func (b *BeegoHTTPRequest) SetProtocolVersion(vers string) *BeegoHTTPRequest {
if len(vers) == 0 {
vers = "HTTP/1.1"
@@ -222,8 +224,9 @@ func (b *BeegoHTTPRequest) SetProtocolVersion(vers string) *BeegoHTTPRequest {
b.req.Proto = vers
b.req.ProtoMajor = major
b.req.ProtoMinor = minor
return b
}
logs.Error("%+v", berror.Errorf(InvalidUrlProtocolVersion, "invalid protocol: %s", vers))
return b
}
@@ -291,6 +294,7 @@ func (b *BeegoHTTPRequest) PostFile(formname, filename string) *BeegoHTTPRequest
// Body adds request raw body.
// Supports string and []byte.
// TODO return error if data is invalid
func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest {
switch t := data.(type) {
case string:
@@ -307,6 +311,8 @@ func (b *BeegoHTTPRequest) Body(data interface{}) *BeegoHTTPRequest {
return ioutil.NopCloser(bf), nil
}
b.req.ContentLength = int64(len(t))
default:
logs.Error("%+v", berror.Errorf(UnsupportedBodyType, "unsupported body data type: %s", t))
}
return b
}
@@ -316,7 +322,7 @@ func (b *BeegoHTTPRequest) XMLBody(obj interface{}) (*BeegoHTTPRequest, error) {
if b.req.Body == nil && obj != nil {
byts, err := xml.Marshal(obj)
if err != nil {
return b, err
return b, berror.Wrap(err, InvalidXMLBody, "obj could not be converted to XML data")
}
b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
b.req.GetBody = func() (io.ReadCloser, error) {
@@ -333,7 +339,7 @@ func (b *BeegoHTTPRequest) YAMLBody(obj interface{}) (*BeegoHTTPRequest, error)
if b.req.Body == nil && obj != nil {
byts, err := yaml.Marshal(obj)
if err != nil {
return b, err
return b, berror.Wrap(err, InvalidYAMLBody, "obj could not be converted to YAML data")
}
b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
b.req.ContentLength = int64(len(byts))
@@ -347,7 +353,7 @@ func (b *BeegoHTTPRequest) JSONBody(obj interface{}) (*BeegoHTTPRequest, error)
if b.req.Body == nil && obj != nil {
byts, err := json.Marshal(obj)
if err != nil {
return b, err
return b, berror.Wrap(err, InvalidJSONBody, "obj could not be converted to JSON body")
}
b.req.Body = ioutil.NopCloser(bytes.NewReader(byts))
b.req.ContentLength = int64(len(byts))
@@ -375,28 +381,15 @@ func (b *BeegoHTTPRequest) buildURL(paramBody string) {
bodyWriter := multipart.NewWriter(pw)
go func() {
for formname, filename := range b.files {
fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
if err != nil {
log.Println("Httplib:", err)
}
fh, err := os.Open(filename)
if err != nil {
log.Println("Httplib:", err)
}
// iocopy
_, err = io.Copy(fileWriter, fh)
fh.Close()
if err != nil {
log.Println("Httplib:", err)
}
b.handleFileToBody(bodyWriter, formname, filename)
}
for k, v := range b.params {
for _, vv := range v {
bodyWriter.WriteField(k, vv)
_ = bodyWriter.WriteField(k, vv)
}
}
bodyWriter.Close()
pw.Close()
_ = bodyWriter.Close()
_ = pw.Close()
}()
b.Header("Content-Type", bodyWriter.FormDataContentType())
b.req.Body = ioutil.NopCloser(pr)
@@ -412,6 +405,29 @@ func (b *BeegoHTTPRequest) buildURL(paramBody string) {
}
}
func (b *BeegoHTTPRequest) handleFileToBody(bodyWriter *multipart.Writer, formname string, filename string) {
fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
const errFmt = "Httplib: %+v"
if err != nil {
logs.Error(errFmt, berror.Wrapf(err, CreateFormFileFailed,
"could not create form file, formname: %s, filename: %s", formname, filename))
}
fh, err := os.Open(filename)
if err != nil {
logs.Error(errFmt, berror.Wrapf(err, ReadFileFailed, "could not open this file %s", filename))
}
// iocopy
_, err = io.Copy(fileWriter, fh)
if err != nil {
logs.Error(errFmt, berror.Wrapf(err, CopyFileFailed, "could not copy this file %s", filename))
}
err = fh.Close()
if err != nil {
logs.Error(errFmt, berror.Wrapf(err, CloseFileFailed, "could not close this file %s", filename))
}
}
func (b *BeegoHTTPRequest) getResponse() (*http.Response, error) {
if b.resp.StatusCode != 0 {
return b.resp, nil
@@ -440,62 +456,20 @@ func (b *BeegoHTTPRequest) DoRequestWithCtx(ctx context.Context) (resp *http.Res
return root(ctx, b)
}
func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (resp *http.Response, err error) {
var paramBody string
if len(b.params) > 0 {
var buf bytes.Buffer
for k, v := range b.params {
for _, vv := range v {
buf.WriteString(url.QueryEscape(k))
buf.WriteByte('=')
buf.WriteString(url.QueryEscape(vv))
buf.WriteByte('&')
}
}
paramBody = buf.String()
paramBody = paramBody[0 : len(paramBody)-1]
}
func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (*http.Response, error) {
paramBody := b.buildParamBody()
b.buildURL(paramBody)
urlParsed, err := url.Parse(b.url)
if err != nil {
return nil, err
return nil, berror.Wrapf(err, InvalidUrl, "parse url failed, the url is %s", b.url)
}
b.req.URL = urlParsed
trans := b.setting.Transport
trans := b.buildTrans()
if trans == nil {
// create default transport
trans = &http.Transport{
TLSClientConfig: b.setting.TLSClientConfig,
Proxy: b.setting.Proxy,
Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
MaxIdleConnsPerHost: 100,
}
} else {
// if b.transport is *http.Transport then set the settings.
if t, ok := trans.(*http.Transport); ok {
if t.TLSClientConfig == nil {
t.TLSClientConfig = b.setting.TLSClientConfig
}
if t.Proxy == nil {
t.Proxy = b.setting.Proxy
}
if t.Dial == nil {
t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
}
}
}
var jar http.CookieJar
if b.setting.EnableCookie {
if defaultCookieJar == nil {
createDefaultCookie()
}
jar = defaultCookieJar
}
jar := b.buildCookieJar()
client := &http.Client{
Transport: trans,
@@ -511,12 +485,16 @@ func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (resp *http.Response,
}
if b.setting.ShowDebug {
dump, err := httputil.DumpRequest(b.req, b.setting.DumpBody)
if err != nil {
log.Println(err.Error())
dump, e := httputil.DumpRequest(b.req, b.setting.DumpBody)
if e != nil {
logs.Error("%+v", e)
}
b.dump = dump
}
return b.sendRequest(client)
}
func (b *BeegoHTTPRequest) sendRequest(client *http.Client) (resp *http.Response, err error) {
// retries default value is 0, it will run once.
// retries equal to -1, it will run forever until success
// retries is setted, it will retries fixed times.
@@ -524,11 +502,68 @@ func (b *BeegoHTTPRequest) doRequest(ctx context.Context) (resp *http.Response,
for i := 0; b.setting.Retries == -1 || i <= b.setting.Retries; i++ {
resp, err = client.Do(b.req)
if err == nil {
break
return
}
time.Sleep(b.setting.RetryDelay)
}
return resp, err
return nil, berror.Wrap(err, SendRequestFailed, "sending request fail")
}
func (b *BeegoHTTPRequest) buildCookieJar() http.CookieJar {
var jar http.CookieJar
if b.setting.EnableCookie {
if defaultCookieJar == nil {
createDefaultCookie()
}
jar = defaultCookieJar
}
return jar
}
func (b *BeegoHTTPRequest) buildTrans() http.RoundTripper {
trans := b.setting.Transport
if trans == nil {
// create default transport
trans = &http.Transport{
TLSClientConfig: b.setting.TLSClientConfig,
Proxy: b.setting.Proxy,
DialContext: TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
MaxIdleConnsPerHost: 100,
}
} else {
// if b.transport is *http.Transport then set the settings.
if t, ok := trans.(*http.Transport); ok {
if t.TLSClientConfig == nil {
t.TLSClientConfig = b.setting.TLSClientConfig
}
if t.Proxy == nil {
t.Proxy = b.setting.Proxy
}
if t.DialContext == nil {
t.DialContext = TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
}
}
}
return trans
}
func (b *BeegoHTTPRequest) buildParamBody() string {
var paramBody string
if len(b.params) > 0 {
var buf bytes.Buffer
for k, v := range b.params {
for _, vv := range v {
buf.WriteString(url.QueryEscape(k))
buf.WriteByte('=')
buf.WriteString(url.QueryEscape(vv))
buf.WriteByte('&')
}
}
paramBody = buf.String()
paramBody = paramBody[0 : len(paramBody)-1]
}
return paramBody
}
// String returns the body string in response.
@@ -559,10 +594,10 @@ func (b *BeegoHTTPRequest) Bytes() ([]byte, error) {
if b.setting.Gzip && resp.Header.Get("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(resp.Body)
if err != nil {
return nil, err
return nil, berror.Wrap(err, ReadGzipBodyFailed, "building gzip reader failed")
}
b.body, err = ioutil.ReadAll(reader)
return b.body, err
return b.body, berror.Wrap(err, ReadGzipBodyFailed, "reading gzip data failed")
}
b.body, err = ioutil.ReadAll(resp.Body)
return b.body, err
@@ -605,7 +640,7 @@ func pathExistAndMkdir(filename string) (err error) {
return nil
}
}
return err
return berror.Wrapf(err, CreateFileIfNotExistFailed, "try to create(if not exist) failed: %s", filename)
}
// ToJSON returns the map that marshals from the body bytes as json in response.
@@ -615,7 +650,8 @@ func (b *BeegoHTTPRequest) ToJSON(v interface{}) error {
if err != nil {
return err
}
return json.Unmarshal(data, v)
return berror.Wrap(json.Unmarshal(data, v),
UnmarshalJSONResponseToObjectFailed, "unmarshal json body to object failed.")
}
// ToXML returns the map that marshals from the body bytes as xml in response .
@@ -625,7 +661,8 @@ func (b *BeegoHTTPRequest) ToXML(v interface{}) error {
if err != nil {
return err
}
return xml.Unmarshal(data, v)
return berror.Wrap(xml.Unmarshal(data, v),
UnmarshalXMLResponseToObjectFailed, "unmarshal xml body to object failed.")
}
// ToYAML returns the map that marshals from the body bytes as yaml in response .
@@ -635,7 +672,8 @@ func (b *BeegoHTTPRequest) ToYAML(v interface{}) error {
if err != nil {
return err
}
return yaml.Unmarshal(data, v)
return berror.Wrap(yaml.Unmarshal(data, v),
UnmarshalYAMLResponseToObjectFailed, "unmarshal yaml body to object failed.")
}
// Response executes request client gets response manually.
@@ -644,8 +682,18 @@ func (b *BeegoHTTPRequest) Response() (*http.Response, error) {
}
// TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field.
// Deprecated
// we will move this at the end of 2021
// please use TimeoutDialerCtx
func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
return func(netw, addr string) (net.Conn, error) {
return TimeoutDialerCtx(cTimeout, rwTimeout)(context.Background(), netw, addr)
}
}
func TimeoutDialerCtx(cTimeout time.Duration,
rwTimeout time.Duration) func(ctx context.Context, net, addr string) (c net.Conn, err error) {
return func(ctx context.Context, netw, addr string) (net.Conn, error) {
conn, err := net.DialTimeout(netw, addr, cTimeout)
if err != nil {
return nil, err