From 564a7eb9ff419e0f2b027f8e65c3bdd3130dbb5c Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Mon, 19 Apr 2021 17:35:59 +0800 Subject: [PATCH 01/15] add http client --- client/httplib/client/client.go | 290 +++++++++++++++++++ client/httplib/client/client_option.go | 33 +++ client/httplib/client/http_request_option.go | 25 ++ 3 files changed, 348 insertions(+) create mode 100644 client/httplib/client/client.go create mode 100644 client/httplib/client/client_option.go create mode 100644 client/httplib/client/http_request_option.go diff --git a/client/httplib/client/client.go b/client/httplib/client/client.go new file mode 100644 index 00000000..2d0afa9c --- /dev/null +++ b/client/httplib/client/client.go @@ -0,0 +1,290 @@ +// Copyright 2020 beego +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "net/http" + "strings" + + "github.com/beego/beego/v2/client/httplib" + "github.com/beego/beego/v2/core/berror" +) + +type Client struct { + Name string + Endpoint string + CommonOpts []BeegoHttpRequestOption + + Setting *httplib.BeegoHTTPSettings + pointer *ResponsePointer +} + +type ResponsePointer struct { + response **http.Response + statusCode **int + header **http.Header + headerValues map[string]**string //用户传一个key，然后将key存在map的key里，header的value存在value里 + contentLength **int64 +} + +// NewClient +func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, error) { + res := &Client{ + Name: name, + Endpoint: endpoint, + } + for _, o := range opts { + err := o(res) + if err != nil { + return nil, err + } + } + return res, nil +} + +// Response will set response to the pointer +func (c *Client) Response(resp **http.Response) *Client { + if c.pointer == nil { + newC := *c + newC.pointer = &ResponsePointer{ + response: resp, + } + return &newC + } + c.pointer.response = resp + return c +} + +// StatusCode will set response StatusCode to the pointer +func (c *Client) StatusCode(code **int) *Client { + if c.pointer == nil { + newC := *c + newC.pointer = &ResponsePointer{ + statusCode: code, + } + return &newC + } + c.pointer.statusCode = code + return c +} + +// Headers will set response Headers to the pointer +func (c *Client) Headers(headers **http.Header) *Client { + if c.pointer == nil { + newC := *c + newC.pointer = &ResponsePointer{ + header: headers, + } + return &newC + } + c.pointer.header = headers + return c +} + +// HeaderValue will set response HeaderValue to the pointer +func (c *Client) HeaderValue(key string, value **string) *Client { + if c.pointer == nil { + newC := *c + newC.pointer = &ResponsePointer{ + headerValues: map[string]**string{ + key: value, + }, + } + return &newC + } + if c.pointer.headerValues == nil { + c.pointer.headerValues = map[string]**string{} + } + c.pointer.headerValues[key] = value + return c +} + +// ContentType will set response ContentType to the pointer +func (c *Client) ContentType(contentType **string) *Client { + return c.HeaderValue("Content-Type", contentType) +} + +// ContentLength will set response ContentLength to the pointer +func (c *Client) ContentLength(contentLength **int64) *Client { + if c.pointer == nil { + newC := *c + newC.pointer = &ResponsePointer{ + contentLength: contentLength, + } + return &newC + } + c.pointer.contentLength = contentLength + return c +} + +// setPointers set the http response value to pointer +func (c *Client) setPointers(resp *http.Response) { + if c.pointer == nil { + return + } + if c.pointer.response != nil { + *c.pointer.response = resp + } + if c.pointer.statusCode != nil { + *c.pointer.statusCode = &resp.StatusCode + } + if c.pointer.header != nil { + *c.pointer.header = &resp.Header + } + if c.pointer.headerValues != nil { + for k, v := range c.pointer.headerValues { + s := resp.Header.Get(k) + *v = &s + } + } + if c.pointer.contentLength != nil { + *c.pointer.contentLength = &resp.ContentLength + } +} + +// initRequest will apply all the client setting, common option and request option +func (c *Client) newRequest(method, path string, opts []BeegoHttpRequestOption) (*httplib.BeegoHTTPRequest, error) { + var req *httplib.BeegoHTTPRequest + switch method { + case http.MethodGet: + req = httplib.Get(c.Endpoint + path) + case http.MethodPost: + req = httplib.Post(c.Endpoint + path) + case http.MethodPut: + req = httplib.Put(c.Endpoint + path) + case http.MethodDelete: + req = httplib.Delete(c.Endpoint + path) + case http.MethodHead: + req = httplib.Head(c.Endpoint + path) + } + + req = req.Setting(*c.Setting) + for _, o := range c.CommonOpts { + err := o(req) + if err != nil { + return nil, err + } + } + for _, o := range opts { + err := o(req) + if err != nil { + return nil, err + } + } + return req, nil +} + +// handleResponse try to parse body to meaningful value +func (c *Client) handleResponse(value interface{}, req *httplib.BeegoHTTPRequest) error { + // send request + resp, err := req.Response() + if err != nil { + return err + } + c.setPointers(resp) + + // handle basic type + switch v := value.(type) { + case **string: + s, err := req.String() + if err != nil { + return nil + } + *v = &s + return nil + case **[]byte: + bs, err := req.Bytes() + if err != nil { + return nil + } + *v = &bs + return nil + } + + // try to parse it as content type + switch strings.Split(resp.Header.Get("Content-Type"), ";")[0] { + case "application/json": + return req.ToJSON(value) + case "text/xml": + return req.ToXML(value) + case "text/yaml", "application/x-yaml": + return req.ToYAML(value) + } + + // try to parse it anyway + if err := req.ToJSON(value); err == nil { + return nil + } + if err := req.ToYAML(value); err == nil { + return nil + } + if err := req.ToXML(value); err == nil { + return nil + } + + // TODO add new error type about can't parse body + return berror.Error(httplib.UnsupportedBodyType, "unsupported body data") +} + +// Get Send a GET request and try to give its result value +func (c *Client) Get(value interface{}, path string, opts ...BeegoHttpRequestOption) error { + req, err := c.newRequest(http.MethodGet, path, opts) + if err != nil { + return err + } + return c.handleResponse(value, req) +} + +// Post Send a POST request and try to give its result value +func (c *Client) Post(value interface{}, path string, body interface{}, opts ...BeegoHttpRequestOption) error { + req, err := c.newRequest(http.MethodPost, path, opts) + if err != nil { + return err + } + if body != nil { + req = req.Body(body) + } + return c.handleResponse(value, req) +} + +// Put Send a Put request and try to give its result value +func (c *Client) Put(value interface{}, path string, body interface{}, opts ...BeegoHttpRequestOption) error { + req, err := c.newRequest(http.MethodPut, path, opts) + if err != nil { + return err + } + if body != nil { + req = req.Body(body) + } + return c.handleResponse(value, req) +} + +// Delete Send a Delete request and try to give its result value +func (c *Client) Delete(value interface{}, path string, opts ...BeegoHttpRequestOption) error { + req, err := c.newRequest(http.MethodDelete, path, opts) + if err != nil { + return err + } + return c.handleResponse(value, req) +} + +// Head Send a Head request and try to give its result value +func (c *Client) Head(value interface{}, path string, opts ...BeegoHttpRequestOption) error { + req, err := c.newRequest(http.MethodHead, path, opts) + if err != nil { + return err + } + return c.handleResponse(value, req) +} diff --git a/client/httplib/client/client_option.go b/client/httplib/client/client_option.go new file mode 100644 index 00000000..0733b682 --- /dev/null +++ b/client/httplib/client/client_option.go @@ -0,0 +1,33 @@ +// Copyright 2020 beego +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "net/http" + "net/url" + "time" +) + +type ClientOption func(client *Client) error + +// client设置 +func WithTimeout(connectTimeout, readWriteTimeout time.Duration) ClientOption +func WithEnableCookie(enable bool) ClientOption +func WithUserAgent(userAgent string) ClientOption +func WithCookie(cookie *http.Cookie) ClientOption +func WithTransport(transport http.RoundTripper) ClientOption +func WithProxy(proxy func(*http.Request) (*url.URL, error)) ClientOption +func WithCheckRedirect(redirect func(req *http.Request, via []*http.Request) error) ClientOption +func WithAccept(accept string) ClientOption diff --git a/client/httplib/client/http_request_option.go b/client/httplib/client/http_request_option.go new file mode 100644 index 00000000..d4030901 --- /dev/null +++ b/client/httplib/client/http_request_option.go @@ -0,0 +1,25 @@ +// Copyright 2020 beego +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import "github.com/beego/beego/v2/client/httplib" + +type BeegoHttpRequestOption func(request *httplib.BeegoHTTPRequest) error + +// Req设置 +func WithTokenFactory(tokenFactory func() (string, error)) BeegoHttpRequestOption +func WithBasicAuth(basicAuth func() (string, error)) BeegoHttpRequestOption +func WithFilters(fcs ...httplib.FilterChain) BeegoHttpRequestOption +func WithContentType(contentType string) BeegoHttpRequestOption From 575bf62fd35c555ca2ac9610be0da580786e4879 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Tue, 20 Apr 2021 17:52:33 +0800 Subject: [PATCH 02/15] add httpclient add options --- client/httplib/client/client_option.go | 33 --- client/httplib/client/http_request_option.go | 25 -- client/httplib/client_option.go | 175 ++++++++++++ client/httplib/client_option_test.go | 250 +++++++++++++++++ .../{client/client.go => httpclient.go} | 48 ++-- client/httplib/httpclient_test.go | 258 ++++++++++++++++++ client/httplib/httplib_test.go | 2 +- client/httplib/setting.go | 5 + 8 files changed, 716 insertions(+), 80 deletions(-) delete mode 100644 client/httplib/client/client_option.go delete mode 100644 client/httplib/client/http_request_option.go create mode 100644 client/httplib/client_option.go create mode 100644 client/httplib/client_option_test.go rename client/httplib/{client/client.go => httpclient.go} (87%) create mode 100644 client/httplib/httpclient_test.go diff --git a/client/httplib/client/client_option.go b/client/httplib/client/client_option.go deleted file mode 100644 index 0733b682..00000000 --- a/client/httplib/client/client_option.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2020 beego -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package client - -import ( - "net/http" - "net/url" - "time" -) - -type ClientOption func(client *Client) error - -// client设置 -func WithTimeout(connectTimeout, readWriteTimeout time.Duration) ClientOption -func WithEnableCookie(enable bool) ClientOption -func WithUserAgent(userAgent string) ClientOption -func WithCookie(cookie *http.Cookie) ClientOption -func WithTransport(transport http.RoundTripper) ClientOption -func WithProxy(proxy func(*http.Request) (*url.URL, error)) ClientOption -func WithCheckRedirect(redirect func(req *http.Request, via []*http.Request) error) ClientOption -func WithAccept(accept string) ClientOption diff --git a/client/httplib/client/http_request_option.go b/client/httplib/client/http_request_option.go deleted file mode 100644 index d4030901..00000000 --- a/client/httplib/client/http_request_option.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2020 beego -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package client - -import "github.com/beego/beego/v2/client/httplib" - -type BeegoHttpRequestOption func(request *httplib.BeegoHTTPRequest) error - -// Req设置 -func WithTokenFactory(tokenFactory func() (string, error)) BeegoHttpRequestOption -func WithBasicAuth(basicAuth func() (string, error)) BeegoHttpRequestOption -func WithFilters(fcs ...httplib.FilterChain) BeegoHttpRequestOption -func WithContentType(contentType string) BeegoHttpRequestOption diff --git a/client/httplib/client_option.go b/client/httplib/client_option.go new file mode 100644 index 00000000..f38c50dc --- /dev/null +++ b/client/httplib/client_option.go @@ -0,0 +1,175 @@ +// Copyright 2020 beego +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httplib + +import ( + "crypto/tls" + "net/http" + "net/url" + "time" +) + +type ClientOption func(client *Client) error +type BeegoHttpRequestOption func(request *BeegoHTTPRequest) error + +// WithEnableCookie will enable cookie in all subsequent request +func WithEnableCookie(enable bool) ClientOption { + return func(client *Client) error { + client.Setting.EnableCookie = enable + return nil + } +} + +// WithEnableCookie will adds UA in all subsequent request +func WithUserAgent(userAgent string) ClientOption { + return func(client *Client) error { + client.Setting.UserAgent = userAgent + return nil + } +} + +// WithTLSClientConfig will adds tls config in all subsequent request +func WithTLSClientConfig(config *tls.Config) ClientOption { + return func(client *Client) error { + client.Setting.TLSClientConfig = config + return nil + } +} + +// WithTransport will set transport field in all subsequent request +func WithTransport(transport http.RoundTripper) ClientOption { + return func(client *Client) error { + client.Setting.Transport = transport + return nil + } +} + +// WithProxy will set http proxy field in all subsequent request +func WithProxy(proxy func(*http.Request) (*url.URL, error)) ClientOption { + return func(client *Client) error { + client.Setting.Proxy = proxy + return nil + } +} + +// WithCheckRedirect will specifies the policy for handling redirects in all subsequent request +func WithCheckRedirect(redirect func(req *http.Request, via []*http.Request) error) ClientOption { + return func(client *Client) error { + client.Setting.CheckRedirect = redirect + return nil + } +} + +// WithHTTPSetting can replace beegoHTTPSeting +func WithHTTPSetting(setting BeegoHTTPSettings) ClientOption { + return func(client *Client) error { + client.Setting = &setting + return nil + } +} + +// WithEnableGzip will enable gzip in all subsequent request +func WithEnableGzip(enable bool) ClientOption { + return func(client *Client) error { + client.Setting.Gzip = enable + return nil + } +} + +// BeegoHttpRequestOption + +// WithTimeout sets connect time out and read-write time out for BeegoRequest. +func WithTimeout(connectTimeout, readWriteTimeout time.Duration) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.SetTimeout(connectTimeout, readWriteTimeout) + return nil + } +} + +// WithHeader adds header item string in request. +func WithHeader(key, value string) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.Header(key, value) + return nil + } +} + +// WithCookie adds a cookie to the request. +func WithCookie(cookie *http.Cookie) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.Header("Cookie", cookie.String()) + return nil + } +} + +// Withtokenfactory adds a custom function to set Authorization +func WithTokenFactory(tokenFactory func() (string, error)) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + t, err := tokenFactory() + if err != nil { + return err + } + request.Header("Authorization", t) + return nil + } +} + +// WithBasicAuth adds a custom function to set basic auth +func WithBasicAuth(basicAuth func() (string, string, error)) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + username, password, err := basicAuth() + if err != nil { + return err + } + request.SetBasicAuth(username, password) + return nil + } +} + +// WithFilters will use the filter as the invocation filters +func WithFilters(fcs ...FilterChain) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.SetFilters(fcs...) + return nil + } +} + +// WithContentType adds ContentType in header +func WithContentType(contentType string) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.Header("Content-Type", contentType) + return nil + } +} + +// WithParam adds query param in to request. +func WithParam(key, value string) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.Param(key, value) + return nil + } +} + +// WithRetry set retry times and delay for the request +// default is 0 (never retry) +// -1 retry indefinitely (forever) +// Other numbers specify the exact retry amount +func WithRetry(times int, delay time.Duration) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) error { + request.Retries(times) + request.RetryDelay(delay) + return nil + } +} diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go new file mode 100644 index 00000000..d989bd94 --- /dev/null +++ b/client/httplib/client_option_test.go @@ -0,0 +1,250 @@ +// Copyright 2020 beego +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httplib + +import ( + "errors" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestOption_WithEnableCookie(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/", + WithEnableCookie(true)) + if err != nil { + t.Fatal(err) + } + + v := "smallfish" + var str *string + err = client.Get(&str, "/cookies/set?k1="+v) + if err != nil { + t.Fatal(err) + } + t.Log(*str) + + err = client.Get(&str, "/cookies") + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, v) + if n == -1 { + t.Fatal(v + " not found in cookie") + } +} + +func TestOption_WithUserAgent(t *testing.T) { + v := "beego" + client, err := NewClient("test", "http://httpbin.org/", + WithUserAgent(v)) + if err != nil { + t.Fatal(err) + } + + var str *string + err = client.Get(&str, "/headers") + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, v) + if n == -1 { + t.Fatal(v + " not found in user-agent") + } +} + +func TestOption_WithCheckRedirect(t *testing.T) { + client, err := NewClient("test", "https://goolnk.com/33BD2j", + WithCheckRedirect(func(redirectReq *http.Request, redirectVia []*http.Request) error { + return errors.New("Redirect triggered") + })) + if err != nil { + t.Fatal(err) + } + err = client.Get(nil, "") + assert.NotNil(t, err) +} + +func TestOption_WithHTTPSetting(t *testing.T) { + v := "beego" + var setting BeegoHTTPSettings + setting.EnableCookie = true + setting.UserAgent = v + setting.Transport = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 50, + IdleConnTimeout: 90 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + setting.ReadWriteTimeout = 5 * time.Second + + client, err := NewClient("test", "http://httpbin.org/", + WithHTTPSetting(setting)) + if err != nil { + t.Fatal(err) + } + + var str *string + err = client.Get(&str, "/get") + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, v) + if n == -1 { + t.Fatal(v + " not found in user-agent") + } +} + +func TestOption_WithHeader(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + client.CommonOpts = append(client.CommonOpts, WithHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")) + + var str *string + err = client.Get(&str, "/headers") + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, "Mozilla/5.0") + if n == -1 { + t.Fatal("Mozilla/5.0 not found in user-agent") + } +} + +func TestOption_WithTokenFactory(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + client.CommonOpts = append(client.CommonOpts, + WithTokenFactory(func() (string, error) { + return "testauth", nil + })) + + var str *string + err = client.Get(&str, "/headers") + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, "testauth") + if n == -1 { + t.Fatal("Auth is not set in request") + } +} + +func TestOption_WithBasicAuth(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var str *string + err = client.Get(&str, "/basic-auth/user/passwd", + WithBasicAuth(func() (string, string, error) { + return "user", "passwd", nil + })) + if err != nil { + t.Fatal(err) + } + t.Log(str) + n := strings.Index(*str, "authenticated") + if n == -1 { + t.Fatal("authenticated not found in response") + } +} + +func TestOption_WithContentType(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + v := "application/json" + var str *string + err = client.Get(&str, "/headers", WithContentType(v)) + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, v) + if n == -1 { + t.Fatal(v + " not found in header") + } +} + +func TestOption_WithParam(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + v := "smallfish" + var str *string + err = client.Get(&str, "/get", WithParam("username", v)) + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(*str, v) + if n == -1 { + t.Fatal(v + " not found in header") + } +} + +func TestOption_WithRetry(t *testing.T) { + client, err := NewClient("test", "https://goolnk.com/33BD2j", + WithCheckRedirect(func(redirectReq *http.Request, redirectVia []*http.Request) error { + return errors.New("Redirect triggered") + })) + if err != nil { + t.Fatal(err) + } + + retryAmount := 1 + retryDelay := 1400 * time.Millisecond + startTime := time.Now().UnixNano() / int64(time.Millisecond) + + err = client.Get(nil, "", WithRetry(retryAmount, retryDelay)) + + assert.NotNil(t, err) + endTime := time.Now().UnixNano() / int64(time.Millisecond) + elapsedTime := endTime - startTime + delayedTime := int64(retryAmount) * retryDelay.Milliseconds() + if elapsedTime < delayedTime { + t.Errorf("Not enough retries. Took %dms. Delay was meant to take %dms", elapsedTime, delayedTime) + } +} diff --git a/client/httplib/client/client.go b/client/httplib/httpclient.go similarity index 87% rename from client/httplib/client/client.go rename to client/httplib/httpclient.go index 2d0afa9c..ea6e4316 100644 --- a/client/httplib/client/client.go +++ b/client/httplib/httpclient.go @@ -12,26 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package client +package httplib import ( "net/http" "strings" - "github.com/beego/beego/v2/client/httplib" "github.com/beego/beego/v2/core/berror" ) +// Client provides an HTTP client supporting chain call type Client struct { Name string Endpoint string CommonOpts []BeegoHttpRequestOption - Setting *httplib.BeegoHTTPSettings - pointer *ResponsePointer + Setting *BeegoHTTPSettings + pointer *responsePointer } -type ResponsePointer struct { +type responsePointer struct { response **http.Response statusCode **int header **http.Header @@ -39,12 +39,14 @@ type ResponsePointer struct { contentLength **int64 } -// NewClient +// NewClient return a new http client func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, error) { res := &Client{ Name: name, Endpoint: endpoint, } + setting := GetDefaultSetting() + res.Setting = &setting for _, o := range opts { err := o(res) if err != nil { @@ -58,7 +60,7 @@ func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, err func (c *Client) Response(resp **http.Response) *Client { if c.pointer == nil { newC := *c - newC.pointer = &ResponsePointer{ + newC.pointer = &responsePointer{ response: resp, } return &newC @@ -71,7 +73,7 @@ func (c *Client) Response(resp **http.Response) *Client { func (c *Client) StatusCode(code **int) *Client { if c.pointer == nil { newC := *c - newC.pointer = &ResponsePointer{ + newC.pointer = &responsePointer{ statusCode: code, } return &newC @@ -84,7 +86,7 @@ func (c *Client) StatusCode(code **int) *Client { func (c *Client) Headers(headers **http.Header) *Client { if c.pointer == nil { newC := *c - newC.pointer = &ResponsePointer{ + newC.pointer = &responsePointer{ header: headers, } return &newC @@ -97,7 +99,7 @@ func (c *Client) Headers(headers **http.Header) *Client { func (c *Client) HeaderValue(key string, value **string) *Client { if c.pointer == nil { newC := *c - newC.pointer = &ResponsePointer{ + newC.pointer = &responsePointer{ headerValues: map[string]**string{ key: value, }, @@ -120,7 +122,7 @@ func (c *Client) ContentType(contentType **string) *Client { func (c *Client) ContentLength(contentLength **int64) *Client { if c.pointer == nil { newC := *c - newC.pointer = &ResponsePointer{ + newC.pointer = &responsePointer{ contentLength: contentLength, } return &newC @@ -155,19 +157,19 @@ func (c *Client) setPointers(resp *http.Response) { } // initRequest will apply all the client setting, common option and request option -func (c *Client) newRequest(method, path string, opts []BeegoHttpRequestOption) (*httplib.BeegoHTTPRequest, error) { - var req *httplib.BeegoHTTPRequest +func (c *Client) newRequest(method, path string, opts []BeegoHttpRequestOption) (*BeegoHTTPRequest, error) { + var req *BeegoHTTPRequest switch method { case http.MethodGet: - req = httplib.Get(c.Endpoint + path) + req = Get(c.Endpoint + path) case http.MethodPost: - req = httplib.Post(c.Endpoint + path) + req = Post(c.Endpoint + path) case http.MethodPut: - req = httplib.Put(c.Endpoint + path) + req = Put(c.Endpoint + path) case http.MethodDelete: - req = httplib.Delete(c.Endpoint + path) + req = Delete(c.Endpoint + path) case http.MethodHead: - req = httplib.Head(c.Endpoint + path) + req = Head(c.Endpoint + path) } req = req.Setting(*c.Setting) @@ -187,7 +189,7 @@ func (c *Client) newRequest(method, path string, opts []BeegoHttpRequestOption) } // handleResponse try to parse body to meaningful value -func (c *Client) handleResponse(value interface{}, req *httplib.BeegoHTTPRequest) error { +func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error { // send request resp, err := req.Response() if err != nil { @@ -195,6 +197,10 @@ func (c *Client) handleResponse(value interface{}, req *httplib.BeegoHTTPRequest } c.setPointers(resp) + if value == nil { + return nil + } + // handle basic type switch v := value.(type) { case **string: @@ -217,7 +223,7 @@ func (c *Client) handleResponse(value interface{}, req *httplib.BeegoHTTPRequest switch strings.Split(resp.Header.Get("Content-Type"), ";")[0] { case "application/json": return req.ToJSON(value) - case "text/xml": + case "text/xml", "application/xml": return req.ToXML(value) case "text/yaml", "application/x-yaml": return req.ToYAML(value) @@ -235,7 +241,7 @@ func (c *Client) handleResponse(value interface{}, req *httplib.BeegoHTTPRequest } // TODO add new error type about can't parse body - return berror.Error(httplib.UnsupportedBodyType, "unsupported body data") + return berror.Error(UnsupportedBodyType, "unsupported body data") } // Get Send a GET request and try to give its result value diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go new file mode 100644 index 00000000..0464c9e5 --- /dev/null +++ b/client/httplib/httpclient_test.go @@ -0,0 +1,258 @@ +// Copyright 2020 beego +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httplib + +import ( + "encoding/xml" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewClient(t *testing.T) { + client, err := NewClient("test1", "http://beego.me", WithEnableCookie(true)) + assert.NoError(t, err) + assert.NotNil(t, client) + assert.Equal(t, true, client.Setting.EnableCookie) +} + +func TestClient_Response(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var resp *http.Response + err = client.Response(&resp).Get(nil, "status/203") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 203, resp.StatusCode) +} + +func TestClient_StatusCode(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var statusCode *int + err = client.StatusCode(&statusCode).Get(nil, "status/203") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 203, *statusCode) +} + +func TestClient_Headers(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var header *http.Header + err = client.Headers(&header).Get(nil, "bytes/123") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "123", header.Get("Content-Length")) +} + +func TestClient_HeaderValue(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var val *string + err = client.Headers(nil).HeaderValue("Content-Length", &val).Get(nil, "bytes/123") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "123", *val) +} + +func TestClient_ContentType(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var contentType *string + err = client.ContentType(&contentType).Get(nil, "bytes/123") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "application/octet-stream", *contentType) +} + +func TestClient_ContentLength(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + var contentLength *int64 + err = client.ContentLength(&contentLength).Get(nil, "bytes/123") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, int64(123), *contentLength) +} + +type total struct { + Slideshow slideshow `json:"slideshow" yaml:"slideshow"` +} + +type slideshow struct { + XMLName xml.Name `xml:"slideshow"` + + Title string `json:"title" yaml:"title" xml:"title,attr"` + Author string `json:"author" yaml:"author" xml:"author,attr"` + Date string `json:"date" yaml:"date" xml:"date,attr"` + Slides []slide `json:"slides" yaml:"slides" xml:"slide"` +} + +type slide struct { + XMLName xml.Name `xml:"slide"` + + Title string `json:"title" yaml:"title" xml:"title"` +} + +func TestClient_Get(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org/") + if err != nil { + t.Fatal(err) + } + + // basic type + var s *string + err = client.Get(&s, "/base64/SFRUUEJJTiBpcyBhd2Vzb21l") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "HTTPBIN is awesome", *s) + + var bytes *[]byte + err = client.Get(&bytes, "/base64/SFRUUEJJTiBpcyBhd2Vzb21l") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, []byte("HTTPBIN is awesome"), *bytes) + + // json + var tp *total + err = client.Get(&tp, "/json") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "Sample Slide Show", tp.Slideshow.Title) + assert.Equal(t, 2, len(tp.Slideshow.Slides)) + assert.Equal(t, "Overview", tp.Slideshow.Slides[1].Title) + + // xml + var ssp *slideshow + err = client.Get(&ssp, "/base64/PD94bWwgPz48c2xpZGVzaG93CnRpdGxlPSJTYW1wbGUgU2xpZGUgU2hvdyIKZGF0ZT0iRGF0ZSBvZiBwdWJsaWNhdGlvbiIKYXV0aG9yPSJZb3VycyBUcnVseSI+PHNsaWRlIHR5cGU9ImFsbCI+PHRpdGxlPldha2UgdXAgdG8gV29uZGVyV2lkZ2V0cyE8L3RpdGxlPjwvc2xpZGU+PHNsaWRlIHR5cGU9ImFsbCI+PHRpdGxlPk92ZXJ2aWV3PC90aXRsZT48aXRlbT5XaHkgPGVtPldvbmRlcldpZGdldHM8L2VtPiBhcmUgZ3JlYXQ8L2l0ZW0+PGl0ZW0vPjxpdGVtPldobyA8ZW0+YnV5czwvZW0+IFdvbmRlcldpZGdldHM8L2l0ZW0+PC9zbGlkZT48L3NsaWRlc2hvdz4=") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "Sample Slide Show", ssp.Title) + assert.Equal(t, 2, len(ssp.Slides)) + assert.Equal(t, "Overview", ssp.Slides[1].Title) + + // yaml + tp = nil + err = client.Get(&tp, "/base64/c2xpZGVzaG93OgogIGF1dGhvcjogWW91cnMgVHJ1bHkKICBkYXRlOiBkYXRlIG9mIHB1YmxpY2F0aW9uCiAgc2xpZGVzOgogIC0gdGl0bGU6IFdha2UgdXAgdG8gV29uZGVyV2lkZ2V0cyEKICAgIHR5cGU6IGFsbAogIC0gaXRlbXM6CiAgICAtIFdoeSA8ZW0+V29uZGVyV2lkZ2V0czwvZW0+IGFyZSBncmVhdAogICAgLSBXaG8gPGVtPmJ1eXM8L2VtPiBXb25kZXJXaWRnZXRzCiAgICB0aXRsZTogT3ZlcnZpZXcKICAgIHR5cGU6IGFsbAogIHRpdGxlOiBTYW1wbGUgU2xpZGUgU2hvdw==") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, "Sample Slide Show", tp.Slideshow.Title) + assert.Equal(t, 2, len(tp.Slideshow.Slides)) + assert.Equal(t, "Overview", tp.Slideshow.Slides[1].Title) + +} + +func TestClient_Post(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org") + if err != nil { + t.Fatal(err) + } + + var s *string + err = client.Get(&s, "/json") + if err != nil { + t.Fatal(err) + } + + var resp *http.Response + err = client.Response(&resp).Post(&s, "/post", *s) + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, resp) + assert.Equal(t, http.MethodPost, resp.Request.Method) +} + +func TestClient_Put(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org") + if err != nil { + t.Fatal(err) + } + + var s *string + err = client.Get(&s, "/json") + if err != nil { + t.Fatal(err) + } + + var resp *http.Response + err = client.Response(&resp).Put(&s, "/put", *s) + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, resp) + assert.Equal(t, http.MethodPut, resp.Request.Method) +} + +func TestClient_Delete(t *testing.T) { + client, err := NewClient("test", "http://httpbin.org") + if err != nil { + t.Fatal(err) + } + + var resp *http.Response + err = client.Response(&resp).Delete(nil, "/delete") + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, resp) + assert.Equal(t, http.MethodDelete, resp.Request.Method) +} + +func TestClient_Head(t *testing.T) { + client, err := NewClient("test", "http://beego.me") + if err != nil { + t.Fatal(err) + } + + var resp *http.Response + err = client.Response(&resp).Head(nil, "") + if err != nil { + t.Fatal(err) + } + assert.NotNil(t, resp) + assert.Equal(t, http.MethodHead, resp.Request.Method) +} diff --git a/client/httplib/httplib_test.go b/client/httplib/httplib_test.go index 9133ad5f..fc9f25a4 100644 --- a/client/httplib/httplib_test.go +++ b/client/httplib/httplib_test.go @@ -39,7 +39,7 @@ func TestResponse(t *testing.T) { } func TestDoRequest(t *testing.T) { - req := Get("https://goolnk.com/33BD2j") + req := Get("https://goolnk.com/") retryAmount := 1 req.Retries(1) req.RetryDelay(1400 * time.Millisecond) diff --git a/client/httplib/setting.go b/client/httplib/setting.go index df8eff4b..542c39be 100644 --- a/client/httplib/setting.go +++ b/client/httplib/setting.go @@ -55,6 +55,11 @@ func SetDefaultSetting(setting BeegoHTTPSettings) { defaultSetting = setting } +// SetDefaultSetting return current default setting +func GetDefaultSetting() BeegoHTTPSettings { + return defaultSetting +} + var defaultSetting = BeegoHTTPSettings{ UserAgent: "beegoServer", ConnectTimeout: 60 * time.Second, From 672a59d780cfe31e6264a2ec4a554212e45474dd Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Tue, 20 Apr 2021 17:57:58 +0800 Subject: [PATCH 03/15] fix: Restore a deleted field --- client/httplib/httplib_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/httplib/httplib_test.go b/client/httplib/httplib_test.go index fc9f25a4..9133ad5f 100644 --- a/client/httplib/httplib_test.go +++ b/client/httplib/httplib_test.go @@ -39,7 +39,7 @@ func TestResponse(t *testing.T) { } func TestDoRequest(t *testing.T) { - req := Get("https://goolnk.com/") + req := Get("https://goolnk.com/33BD2j") retryAmount := 1 req.Retries(1) req.RetryDelay(1400 * time.Millisecond) From 6f36998df8c60d3ebe7318208e98c0465e266d87 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Tue, 20 Apr 2021 18:09:11 +0800 Subject: [PATCH 04/15] add change log --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dbe15d0..cc190f62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # developing +- Add http client and option func. [4455](https://github.com/beego/beego/issues/4455) - Web mock and test support. [4565](https://github.com/beego/beego/pull/4565) [4574](https://github.com/beego/beego/pull/4574) - Error codes definition of cache module. [4493](https://github.com/beego/beego/pull/4493) - Remove generateCommentRoute http hook. Using `bee generate routers` commands instead.[4486](https://github.com/beego/beego/pull/4486) [bee PR 762](https://github.com/beego/bee/pull/762) From 84946743d977a9d901f70c7c8290f308b348030d Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Thu, 22 Apr 2021 17:11:27 +0800 Subject: [PATCH 05/15] refactor: improve http client implement --- client/httplib/client_option.go | 74 ++++------- client/httplib/client_option_test.go | 12 +- client/httplib/httpclient.go | 192 +++++---------------------- client/httplib/httplib.go | 59 ++++++++ client/httplib/httplib_test.go | 4 + client/httplib/setting.go | 2 +- 6 files changed, 133 insertions(+), 210 deletions(-) diff --git a/client/httplib/client_option.go b/client/httplib/client_option.go index f38c50dc..e7402b8c 100644 --- a/client/httplib/client_option.go +++ b/client/httplib/client_option.go @@ -21,70 +21,62 @@ import ( "time" ) -type ClientOption func(client *Client) error -type BeegoHttpRequestOption func(request *BeegoHTTPRequest) error +type ClientOption func(client *Client) +type BeegoHttpRequestOption func(request *BeegoHTTPRequest) // WithEnableCookie will enable cookie in all subsequent request func WithEnableCookie(enable bool) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.EnableCookie = enable - return nil } } // WithEnableCookie will adds UA in all subsequent request func WithUserAgent(userAgent string) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.UserAgent = userAgent - return nil } } // WithTLSClientConfig will adds tls config in all subsequent request func WithTLSClientConfig(config *tls.Config) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.TLSClientConfig = config - return nil } } // WithTransport will set transport field in all subsequent request func WithTransport(transport http.RoundTripper) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.Transport = transport - return nil } } // WithProxy will set http proxy field in all subsequent request func WithProxy(proxy func(*http.Request) (*url.URL, error)) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.Proxy = proxy - return nil } } // WithCheckRedirect will specifies the policy for handling redirects in all subsequent request func WithCheckRedirect(redirect func(req *http.Request, via []*http.Request) error) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.CheckRedirect = redirect - return nil } } // WithHTTPSetting can replace beegoHTTPSeting func WithHTTPSetting(setting BeegoHTTPSettings) ClientOption { - return func(client *Client) error { - client.Setting = &setting - return nil + return func(client *Client) { + client.Setting = setting } } // WithEnableGzip will enable gzip in all subsequent request func WithEnableGzip(enable bool) ClientOption { - return func(client *Client) error { + return func(client *Client) { client.Setting.Gzip = enable - return nil } } @@ -92,73 +84,60 @@ func WithEnableGzip(enable bool) ClientOption { // WithTimeout sets connect time out and read-write time out for BeegoRequest. func WithTimeout(connectTimeout, readWriteTimeout time.Duration) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { + return func(request *BeegoHTTPRequest) { request.SetTimeout(connectTimeout, readWriteTimeout) - return nil } } // WithHeader adds header item string in request. func WithHeader(key, value string) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { + return func(request *BeegoHTTPRequest) { request.Header(key, value) - return nil } } // WithCookie adds a cookie to the request. func WithCookie(cookie *http.Cookie) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { + return func(request *BeegoHTTPRequest) { request.Header("Cookie", cookie.String()) - return nil } } // Withtokenfactory adds a custom function to set Authorization -func WithTokenFactory(tokenFactory func() (string, error)) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { - t, err := tokenFactory() - if err != nil { - return err - } +func WithTokenFactory(tokenFactory func() string) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) { + t := tokenFactory() + request.Header("Authorization", t) - return nil } } // WithBasicAuth adds a custom function to set basic auth -func WithBasicAuth(basicAuth func() (string, string, error)) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { - username, password, err := basicAuth() - if err != nil { - return err - } +func WithBasicAuth(basicAuth func() (string, string)) BeegoHttpRequestOption { + return func(request *BeegoHTTPRequest) { + username, password := basicAuth() request.SetBasicAuth(username, password) - return nil } } // WithFilters will use the filter as the invocation filters func WithFilters(fcs ...FilterChain) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { + return func(request *BeegoHTTPRequest) { request.SetFilters(fcs...) - return nil } } // WithContentType adds ContentType in header func WithContentType(contentType string) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { - request.Header("Content-Type", contentType) - return nil + return func(request *BeegoHTTPRequest) { + request.Header(contentTypeKey, contentType) } } // WithParam adds query param in to request. func WithParam(key, value string) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { + return func(request *BeegoHTTPRequest) { request.Param(key, value) - return nil } } @@ -167,9 +146,8 @@ func WithParam(key, value string) BeegoHttpRequestOption { // -1 retry indefinitely (forever) // Other numbers specify the exact retry amount func WithRetry(times int, delay time.Duration) BeegoHttpRequestOption { - return func(request *BeegoHTTPRequest) error { + return func(request *BeegoHTTPRequest) { request.Retries(times) request.RetryDelay(delay) - return nil } } diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index d989bd94..0598206c 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -147,8 +147,8 @@ func TestOption_WithTokenFactory(t *testing.T) { t.Fatal(err) } client.CommonOpts = append(client.CommonOpts, - WithTokenFactory(func() (string, error) { - return "testauth", nil + WithTokenFactory(func() string { + return "testauth" })) var str *string @@ -172,8 +172,8 @@ func TestOption_WithBasicAuth(t *testing.T) { var str *string err = client.Get(&str, "/basic-auth/user/passwd", - WithBasicAuth(func() (string, string, error) { - return "user", "passwd", nil + WithBasicAuth(func() (string, string) { + return "user", "passwd" })) if err != nil { t.Fatal(err) @@ -240,7 +240,9 @@ func TestOption_WithRetry(t *testing.T) { err = client.Get(nil, "", WithRetry(retryAmount, retryDelay)) - assert.NotNil(t, err) + if err != nil { + t.Fatal(err) + } endTime := time.Now().UnixNano() / int64(time.Millisecond) elapsedTime := endTime - startTime delayedTime := int64(retryAmount) * retryDelay.Milliseconds() diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index ea6e4316..4cfadedf 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -16,9 +16,6 @@ package httplib import ( "net/http" - "strings" - - "github.com/beego/beego/v2/core/berror" ) // Client provides an HTTP client supporting chain call @@ -27,8 +24,8 @@ type Client struct { Endpoint string CommonOpts []BeegoHttpRequestOption - Setting *BeegoHTTPSettings - pointer *responsePointer + Setting BeegoHTTPSettings + pointer responsePointer } type responsePointer struct { @@ -46,71 +43,42 @@ func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, err Endpoint: endpoint, } setting := GetDefaultSetting() - res.Setting = &setting + res.Setting = setting for _, o := range opts { - err := o(res) - if err != nil { - return nil, err - } + o(res) } return res, nil } // Response will set response to the pointer func (c *Client) Response(resp **http.Response) *Client { - if c.pointer == nil { - newC := *c - newC.pointer = &responsePointer{ - response: resp, - } - return &newC - } - c.pointer.response = resp - return c + newC := *c + newC.pointer.response = resp + return &newC } // StatusCode will set response StatusCode to the pointer func (c *Client) StatusCode(code **int) *Client { - if c.pointer == nil { - newC := *c - newC.pointer = &responsePointer{ - statusCode: code, - } - return &newC - } - c.pointer.statusCode = code - return c + newC := *c + newC.pointer.statusCode = code + return &newC } // Headers will set response Headers to the pointer func (c *Client) Headers(headers **http.Header) *Client { - if c.pointer == nil { - newC := *c - newC.pointer = &responsePointer{ - header: headers, - } - return &newC - } - c.pointer.header = headers - return c + newC := *c + newC.pointer.header = headers + return &newC } // HeaderValue will set response HeaderValue to the pointer func (c *Client) HeaderValue(key string, value **string) *Client { - if c.pointer == nil { - newC := *c - newC.pointer = &responsePointer{ - headerValues: map[string]**string{ - key: value, - }, - } - return &newC + newC := *c + if newC.pointer.headerValues == nil { + newC.pointer.headerValues = make(map[string]**string) } - if c.pointer.headerValues == nil { - c.pointer.headerValues = map[string]**string{} - } - c.pointer.headerValues[key] = value - return c + newC.pointer.headerValues[key] = value + return &newC } // ContentType will set response ContentType to the pointer @@ -120,22 +88,13 @@ func (c *Client) ContentType(contentType **string) *Client { // ContentLength will set response ContentLength to the pointer func (c *Client) ContentLength(contentLength **int64) *Client { - if c.pointer == nil { - newC := *c - newC.pointer = &responsePointer{ - contentLength: contentLength, - } - return &newC - } - c.pointer.contentLength = contentLength - return c + newC := *c + newC.pointer.contentLength = contentLength + return &newC } // setPointers set the http response value to pointer func (c *Client) setPointers(resp *http.Response) { - if c.pointer == nil { - return - } if c.pointer.response != nil { *c.pointer.response = resp } @@ -156,36 +115,12 @@ func (c *Client) setPointers(resp *http.Response) { } } -// initRequest will apply all the client setting, common option and request option -func (c *Client) newRequest(method, path string, opts []BeegoHttpRequestOption) (*BeegoHTTPRequest, error) { - var req *BeegoHTTPRequest - switch method { - case http.MethodGet: - req = Get(c.Endpoint + path) - case http.MethodPost: - req = Post(c.Endpoint + path) - case http.MethodPut: - req = Put(c.Endpoint + path) - case http.MethodDelete: - req = Delete(c.Endpoint + path) - case http.MethodHead: - req = Head(c.Endpoint + path) - } - - req = req.Setting(*c.Setting) - for _, o := range c.CommonOpts { - err := o(req) - if err != nil { - return nil, err - } - } +func (c *Client) customReq(req *BeegoHTTPRequest, opts []BeegoHttpRequestOption) { + req.Setting(c.Setting) + opts = append(c.CommonOpts, opts...) for _, o := range opts { - err := o(req) - if err != nil { - return nil, err - } + o(req) } - return req, nil } // handleResponse try to parse body to meaningful value @@ -196,69 +131,20 @@ func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error return err } c.setPointers(resp) - - if value == nil { - return nil - } - - // handle basic type - switch v := value.(type) { - case **string: - s, err := req.String() - if err != nil { - return nil - } - *v = &s - return nil - case **[]byte: - bs, err := req.Bytes() - if err != nil { - return nil - } - *v = &bs - return nil - } - - // try to parse it as content type - switch strings.Split(resp.Header.Get("Content-Type"), ";")[0] { - case "application/json": - return req.ToJSON(value) - case "text/xml", "application/xml": - return req.ToXML(value) - case "text/yaml", "application/x-yaml": - return req.ToYAML(value) - } - - // try to parse it anyway - if err := req.ToJSON(value); err == nil { - return nil - } - if err := req.ToYAML(value); err == nil { - return nil - } - if err := req.ToXML(value); err == nil { - return nil - } - - // TODO add new error type about can't parse body - return berror.Error(UnsupportedBodyType, "unsupported body data") + return req.ResponseForValue(value) } // Get Send a GET request and try to give its result value func (c *Client) Get(value interface{}, path string, opts ...BeegoHttpRequestOption) error { - req, err := c.newRequest(http.MethodGet, path, opts) - if err != nil { - return err - } + req := Get(c.Endpoint + path) + c.customReq(req, opts) return c.handleResponse(value, req) } // Post Send a POST request and try to give its result value func (c *Client) Post(value interface{}, path string, body interface{}, opts ...BeegoHttpRequestOption) error { - req, err := c.newRequest(http.MethodPost, path, opts) - if err != nil { - return err - } + req := Post(c.Endpoint + path) + c.customReq(req, opts) if body != nil { req = req.Body(body) } @@ -267,10 +153,8 @@ func (c *Client) Post(value interface{}, path string, body interface{}, opts ... // Put Send a Put request and try to give its result value func (c *Client) Put(value interface{}, path string, body interface{}, opts ...BeegoHttpRequestOption) error { - req, err := c.newRequest(http.MethodPut, path, opts) - if err != nil { - return err - } + req := Put(c.Endpoint + path) + c.customReq(req, opts) if body != nil { req = req.Body(body) } @@ -279,18 +163,14 @@ func (c *Client) Put(value interface{}, path string, body interface{}, opts ...B // Delete Send a Delete request and try to give its result value func (c *Client) Delete(value interface{}, path string, opts ...BeegoHttpRequestOption) error { - req, err := c.newRequest(http.MethodDelete, path, opts) - if err != nil { - return err - } + req := Delete(c.Endpoint + path) + c.customReq(req, opts) return c.handleResponse(value, req) } // Head Send a Head request and try to give its result value func (c *Client) Head(value interface{}, path string, opts ...BeegoHttpRequestOption) error { - req, err := c.newRequest(http.MethodHead, path, opts) - if err != nil { - return err - } + req := Head(c.Endpoint + path) + c.customReq(req, opts) return c.handleResponse(value, req) } diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index cef2294c..317c462c 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -56,6 +56,7 @@ import ( ) const contentTypeKey = "Content-Type" + // it will be the last filter and execute request.Do var doRequestFilter = func(ctx context.Context, req *BeegoHTTPRequest) (*http.Response, error) { return req.doRequest(ctx) @@ -660,6 +661,64 @@ func (b *BeegoHTTPRequest) Response() (*http.Response, error) { return b.getResponse() } +// ResponseForValue attempts to resolve the response body to value using an existing method. +// Calls Response inner. +// If value type is **string or **[]byte, the func directly passes response body into the pointer. +// Else if response header contain Content-Type, func will call ToJSON\ToXML\ToYAML. +// Finally it will try to parse body as json\yaml\xml, If all attempts fail, an error will be returned +func (b *BeegoHTTPRequest) ResponseForValue(value interface{}) error { + if value == nil { + return nil + } + // handle basic type + switch v := value.(type) { + case **string: + s, err := b.String() + if err != nil { + return nil + } + *v = &s + return nil + case **[]byte: + bs, err := b.Bytes() + if err != nil { + return nil + } + *v = &bs + return nil + } + + resp, err := b.Response() + if err != nil { + return err + } + contentType := strings.Split(resp.Header.Get(contentTypeKey), ";")[0] + + // try to parse it as content type + switch contentType { + case "application/json": + return b.ToJSON(value) + case "text/xml", "application/xml": + return b.ToXML(value) + case "text/yaml", "application/x-yaml", "application/x+yaml": + return b.ToYAML(value) + } + + // try to parse it anyway + if err := b.ToJSON(value); err == nil { + return nil + } + if err := b.ToYAML(value); err == nil { + return nil + } + if err := b.ToXML(value); err == nil { + return nil + } + + // TODO add new error type about can't parse body + return berror.Error(UnsupportedBodyType, "unsupported body data") +} + // TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field. // Deprecated // we will move this at the end of 2021 diff --git a/client/httplib/httplib_test.go b/client/httplib/httplib_test.go index 9133ad5f..e7939a4e 100644 --- a/client/httplib/httplib_test.go +++ b/client/httplib/httplib_test.go @@ -433,3 +433,7 @@ func TestBeegoHTTPRequest_XMLBody(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, req.req.GetBody) } + +// TODO +func TestBeegoHTTPRequest_ResponseForValue(t *testing.T) { +} diff --git a/client/httplib/setting.go b/client/httplib/setting.go index 542c39be..2d7a0eed 100644 --- a/client/httplib/setting.go +++ b/client/httplib/setting.go @@ -55,7 +55,7 @@ func SetDefaultSetting(setting BeegoHTTPSettings) { defaultSetting = setting } -// SetDefaultSetting return current default setting +// GetDefaultSetting return current default setting func GetDefaultSetting() BeegoHTTPSettings { return defaultSetting } From 599e03b0cdb5bb28e6d3bb8131730282b14387a1 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Sun, 16 May 2021 14:18:34 +0800 Subject: [PATCH 06/15] Solution 3 --- client/httplib/client_option_test.go | 94 +++++++++------ client/httplib/httpclient.go | 93 ++++----------- client/httplib/httpclient_test.go | 170 +++++++-------------------- client/httplib/httplib.go | 22 +--- 4 files changed, 123 insertions(+), 256 deletions(-) diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index 0598206c..9efeaa24 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -25,6 +25,27 @@ import ( "github.com/stretchr/testify/assert" ) +type respCarrier struct { + Resp *http.Response + bytes []byte +} + +func (r *respCarrier) SetHttpResponse(resp *http.Response) { + r.Resp = resp +} + +func (r *respCarrier) SetBytes(bytes []byte) { + r.bytes = bytes +} + +func (r *respCarrier) Bytes() []byte { + return r.bytes +} + +func (r *respCarrier) String() string { + return string(r.bytes) +} + func TestOption_WithEnableCookie(t *testing.T) { client, err := NewClient("test", "http://httpbin.org/", WithEnableCookie(true)) @@ -33,20 +54,20 @@ func TestOption_WithEnableCookie(t *testing.T) { } v := "smallfish" - var str *string - err = client.Get(&str, "/cookies/set?k1="+v) + var resp = &respCarrier{} + err = client.Get(resp, "/cookies/set?k1="+v) if err != nil { t.Fatal(err) } - t.Log(*str) + t.Log(resp.String()) - err = client.Get(&str, "/cookies") + err = client.Get(resp, "/cookies") if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, v) + n := strings.Index(resp.String(), v) if n == -1 { t.Fatal(v + " not found in cookie") } @@ -60,14 +81,14 @@ func TestOption_WithUserAgent(t *testing.T) { t.Fatal(err) } - var str *string - err = client.Get(&str, "/headers") + var resp = &respCarrier{} + err = client.Get(resp, "/headers") if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, v) + n := strings.Index(resp.String(), v) if n == -1 { t.Fatal(v + " not found in user-agent") } @@ -108,14 +129,14 @@ func TestOption_WithHTTPSetting(t *testing.T) { t.Fatal(err) } - var str *string - err = client.Get(&str, "/get") + var resp = &respCarrier{} + err = client.Get(resp, "/get") if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, v) + n := strings.Index(resp.String(), v) if n == -1 { t.Fatal(v + " not found in user-agent") } @@ -128,14 +149,14 @@ func TestOption_WithHeader(t *testing.T) { } client.CommonOpts = append(client.CommonOpts, WithHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")) - var str *string - err = client.Get(&str, "/headers") + var resp = &respCarrier{} + err = client.Get(resp, "/headers") if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, "Mozilla/5.0") + n := strings.Index(resp.String(), "Mozilla/5.0") if n == -1 { t.Fatal("Mozilla/5.0 not found in user-agent") } @@ -151,14 +172,14 @@ func TestOption_WithTokenFactory(t *testing.T) { return "testauth" })) - var str *string - err = client.Get(&str, "/headers") + var resp = &respCarrier{} + err = client.Get(resp, "/headers") if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, "testauth") + n := strings.Index(resp.String(), "testauth") if n == -1 { t.Fatal("Auth is not set in request") } @@ -170,16 +191,16 @@ func TestOption_WithBasicAuth(t *testing.T) { t.Fatal(err) } - var str *string - err = client.Get(&str, "/basic-auth/user/passwd", + var resp = &respCarrier{} + err = client.Get(resp, "/basic-auth/user/passwd", WithBasicAuth(func() (string, string) { return "user", "passwd" })) if err != nil { t.Fatal(err) } - t.Log(str) - n := strings.Index(*str, "authenticated") + t.Log(resp.String()) + n := strings.Index(resp.String(), "authenticated") if n == -1 { t.Fatal("authenticated not found in response") } @@ -192,14 +213,14 @@ func TestOption_WithContentType(t *testing.T) { } v := "application/json" - var str *string - err = client.Get(&str, "/headers", WithContentType(v)) + var resp = &respCarrier{} + err = client.Get(resp, "/headers", WithContentType(v)) if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, v) + n := strings.Index(resp.String(), v) if n == -1 { t.Fatal(v + " not found in header") } @@ -212,14 +233,14 @@ func TestOption_WithParam(t *testing.T) { } v := "smallfish" - var str *string - err = client.Get(&str, "/get", WithParam("username", v)) + var resp = &respCarrier{} + err = client.Get(resp, "/get", WithParam("username", v)) if err != nil { t.Fatal(err) } - t.Log(str) + t.Log(resp.String()) - n := strings.Index(*str, v) + n := strings.Index(resp.String(), v) if n == -1 { t.Fatal(v + " not found in header") } @@ -238,11 +259,8 @@ func TestOption_WithRetry(t *testing.T) { retryDelay := 1400 * time.Millisecond startTime := time.Now().UnixNano() / int64(time.Millisecond) - err = client.Get(nil, "", WithRetry(retryAmount, retryDelay)) + _ = client.Get(nil, "", WithRetry(retryAmount, retryDelay)) - if err != nil { - t.Fatal(err) - } endTime := time.Now().UnixNano() / int64(time.Millisecond) elapsedTime := endTime - startTime delayedTime := int64(retryAmount) * retryDelay.Milliseconds() diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index 4cfadedf..69aaa286 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -25,15 +25,17 @@ type Client struct { CommonOpts []BeegoHttpRequestOption Setting BeegoHTTPSettings - pointer responsePointer } -type responsePointer struct { - response **http.Response - statusCode **int - header **http.Header - headerValues map[string]**string //用户传一个key，然后将key存在map的key里，header的value存在value里 - contentLength **int64 +// If value implement this interface. http.response will saved by SetHttpResponse +type HttpResponseCarrier interface { + SetHttpResponse(resp *http.Response) +} + +// If value implement this interface. bytes of http.response will saved by SetHttpResponse +type ResponseBytesCarrier interface { + // Cause of when user get http.response, the body stream is closed. So need to pass bytes by + SetBytes(bytes []byte) } // NewClient return a new http client @@ -50,71 +52,6 @@ func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, err return res, nil } -// Response will set response to the pointer -func (c *Client) Response(resp **http.Response) *Client { - newC := *c - newC.pointer.response = resp - return &newC -} - -// StatusCode will set response StatusCode to the pointer -func (c *Client) StatusCode(code **int) *Client { - newC := *c - newC.pointer.statusCode = code - return &newC -} - -// Headers will set response Headers to the pointer -func (c *Client) Headers(headers **http.Header) *Client { - newC := *c - newC.pointer.header = headers - return &newC -} - -// HeaderValue will set response HeaderValue to the pointer -func (c *Client) HeaderValue(key string, value **string) *Client { - newC := *c - if newC.pointer.headerValues == nil { - newC.pointer.headerValues = make(map[string]**string) - } - newC.pointer.headerValues[key] = value - return &newC -} - -// ContentType will set response ContentType to the pointer -func (c *Client) ContentType(contentType **string) *Client { - return c.HeaderValue("Content-Type", contentType) -} - -// ContentLength will set response ContentLength to the pointer -func (c *Client) ContentLength(contentLength **int64) *Client { - newC := *c - newC.pointer.contentLength = contentLength - return &newC -} - -// setPointers set the http response value to pointer -func (c *Client) setPointers(resp *http.Response) { - if c.pointer.response != nil { - *c.pointer.response = resp - } - if c.pointer.statusCode != nil { - *c.pointer.statusCode = &resp.StatusCode - } - if c.pointer.header != nil { - *c.pointer.header = &resp.Header - } - if c.pointer.headerValues != nil { - for k, v := range c.pointer.headerValues { - s := resp.Header.Get(k) - *v = &s - } - } - if c.pointer.contentLength != nil { - *c.pointer.contentLength = &resp.ContentLength - } -} - func (c *Client) customReq(req *BeegoHTTPRequest, opts []BeegoHttpRequestOption) { req.Setting(c.Setting) opts = append(c.CommonOpts, opts...) @@ -130,7 +67,17 @@ func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error if err != nil { return err } - c.setPointers(resp) + if carrier, ok := (value).(HttpResponseCarrier); ok { + (carrier).SetHttpResponse(resp) + } + if carrier, ok := (value).(ResponseBytesCarrier); ok { + bytes, err := req.Bytes() + if err != nil { + return err + } + (carrier).SetBytes(bytes) + } + return req.ResponseForValue(value) } diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go index 0464c9e5..f006f18f 100644 --- a/client/httplib/httpclient_test.go +++ b/client/httplib/httpclient_test.go @@ -29,91 +29,10 @@ func TestNewClient(t *testing.T) { assert.Equal(t, true, client.Setting.EnableCookie) } -func TestClient_Response(t *testing.T) { - client, err := NewClient("test", "http://httpbin.org/") - if err != nil { - t.Fatal(err) - } +type slideSshowResponse struct { + Resp *http.Response + bytes []byte - var resp *http.Response - err = client.Response(&resp).Get(nil, "status/203") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, 203, resp.StatusCode) -} - -func TestClient_StatusCode(t *testing.T) { - client, err := NewClient("test", "http://httpbin.org/") - if err != nil { - t.Fatal(err) - } - - var statusCode *int - err = client.StatusCode(&statusCode).Get(nil, "status/203") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, 203, *statusCode) -} - -func TestClient_Headers(t *testing.T) { - client, err := NewClient("test", "http://httpbin.org/") - if err != nil { - t.Fatal(err) - } - - var header *http.Header - err = client.Headers(&header).Get(nil, "bytes/123") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "123", header.Get("Content-Length")) -} - -func TestClient_HeaderValue(t *testing.T) { - client, err := NewClient("test", "http://httpbin.org/") - if err != nil { - t.Fatal(err) - } - - var val *string - err = client.Headers(nil).HeaderValue("Content-Length", &val).Get(nil, "bytes/123") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "123", *val) -} - -func TestClient_ContentType(t *testing.T) { - client, err := NewClient("test", "http://httpbin.org/") - if err != nil { - t.Fatal(err) - } - - var contentType *string - err = client.ContentType(&contentType).Get(nil, "bytes/123") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "application/octet-stream", *contentType) -} - -func TestClient_ContentLength(t *testing.T) { - client, err := NewClient("test", "http://httpbin.org/") - if err != nil { - t.Fatal(err) - } - - var contentLength *int64 - err = client.ContentLength(&contentLength).Get(nil, "bytes/123") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, int64(123), *contentLength) -} - -type total struct { Slideshow slideshow `json:"slideshow" yaml:"slideshow"` } @@ -132,36 +51,37 @@ type slide struct { Title string `json:"title" yaml:"title" xml:"title"` } +func (s *slideSshowResponse) SetHttpResponse(resp *http.Response) { + s.Resp = resp +} + +func (s *slideSshowResponse) SetBytes(bytes []byte) { + s.bytes = bytes +} + +func (s *slideSshowResponse) Bytes() []byte { + return s.bytes +} + +func (s *slideSshowResponse) String() string { + return string(s.bytes) +} + func TestClient_Get(t *testing.T) { client, err := NewClient("test", "http://httpbin.org/") if err != nil { t.Fatal(err) } - // basic type - var s *string - err = client.Get(&s, "/base64/SFRUUEJJTiBpcyBhd2Vzb21l") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, "HTTPBIN is awesome", *s) - - var bytes *[]byte - err = client.Get(&bytes, "/base64/SFRUUEJJTiBpcyBhd2Vzb21l") - if err != nil { - t.Fatal(err) - } - assert.Equal(t, []byte("HTTPBIN is awesome"), *bytes) - // json - var tp *total - err = client.Get(&tp, "/json") + var s *slideSshowResponse + err = client.Get(&s, "/json") if err != nil { t.Fatal(err) } - assert.Equal(t, "Sample Slide Show", tp.Slideshow.Title) - assert.Equal(t, 2, len(tp.Slideshow.Slides)) - assert.Equal(t, "Overview", tp.Slideshow.Slides[1].Title) + assert.Equal(t, "Sample Slide Show", s.Slideshow.Title) + assert.Equal(t, 2, len(s.Slideshow.Slides)) + assert.Equal(t, "Overview", s.Slideshow.Slides[1].Title) // xml var ssp *slideshow @@ -174,14 +94,14 @@ func TestClient_Get(t *testing.T) { assert.Equal(t, "Overview", ssp.Slides[1].Title) // yaml - tp = nil - err = client.Get(&tp, "/base64/c2xpZGVzaG93OgogIGF1dGhvcjogWW91cnMgVHJ1bHkKICBkYXRlOiBkYXRlIG9mIHB1YmxpY2F0aW9uCiAgc2xpZGVzOgogIC0gdGl0bGU6IFdha2UgdXAgdG8gV29uZGVyV2lkZ2V0cyEKICAgIHR5cGU6IGFsbAogIC0gaXRlbXM6CiAgICAtIFdoeSA8ZW0+V29uZGVyV2lkZ2V0czwvZW0+IGFyZSBncmVhdAogICAgLSBXaG8gPGVtPmJ1eXM8L2VtPiBXb25kZXJXaWRnZXRzCiAgICB0aXRsZTogT3ZlcnZpZXcKICAgIHR5cGU6IGFsbAogIHRpdGxlOiBTYW1wbGUgU2xpZGUgU2hvdw==") + s = nil + err = client.Get(&s, "/base64/c2xpZGVzaG93OgogIGF1dGhvcjogWW91cnMgVHJ1bHkKICBkYXRlOiBkYXRlIG9mIHB1YmxpY2F0aW9uCiAgc2xpZGVzOgogIC0gdGl0bGU6IFdha2UgdXAgdG8gV29uZGVyV2lkZ2V0cyEKICAgIHR5cGU6IGFsbAogIC0gaXRlbXM6CiAgICAtIFdoeSA8ZW0+V29uZGVyV2lkZ2V0czwvZW0+IGFyZSBncmVhdAogICAgLSBXaG8gPGVtPmJ1eXM8L2VtPiBXb25kZXJXaWRnZXRzCiAgICB0aXRsZTogT3ZlcnZpZXcKICAgIHR5cGU6IGFsbAogIHRpdGxlOiBTYW1wbGUgU2xpZGUgU2hvdw==") if err != nil { t.Fatal(err) } - assert.Equal(t, "Sample Slide Show", tp.Slideshow.Title) - assert.Equal(t, 2, len(tp.Slideshow.Slides)) - assert.Equal(t, "Overview", tp.Slideshow.Slides[1].Title) + assert.Equal(t, "Sample Slide Show", s.Slideshow.Title) + assert.Equal(t, 2, len(s.Slideshow.Slides)) + assert.Equal(t, "Overview", s.Slideshow.Slides[1].Title) } @@ -191,19 +111,19 @@ func TestClient_Post(t *testing.T) { t.Fatal(err) } - var s *string - err = client.Get(&s, "/json") + var resp = &slideSshowResponse{} + err = client.Get(resp, "/json") if err != nil { t.Fatal(err) } - var resp *http.Response - err = client.Response(&resp).Post(&s, "/post", *s) + jsonStr := resp.String() + err = client.Post(resp, "/post", jsonStr) if err != nil { t.Fatal(err) } assert.NotNil(t, resp) - assert.Equal(t, http.MethodPost, resp.Request.Method) + assert.Equal(t, http.MethodPost, resp.Resp.Request.Method) } func TestClient_Put(t *testing.T) { @@ -212,19 +132,19 @@ func TestClient_Put(t *testing.T) { t.Fatal(err) } - var s *string - err = client.Get(&s, "/json") + var resp = &slideSshowResponse{} + err = client.Get(resp, "/json") if err != nil { t.Fatal(err) } - var resp *http.Response - err = client.Response(&resp).Put(&s, "/put", *s) + jsonStr := resp.String() + err = client.Put(resp, "/put", jsonStr) if err != nil { t.Fatal(err) } assert.NotNil(t, resp) - assert.Equal(t, http.MethodPut, resp.Request.Method) + assert.Equal(t, http.MethodPut, resp.Resp.Request.Method) } func TestClient_Delete(t *testing.T) { @@ -233,13 +153,13 @@ func TestClient_Delete(t *testing.T) { t.Fatal(err) } - var resp *http.Response - err = client.Response(&resp).Delete(nil, "/delete") + var resp = &slideSshowResponse{} + err = client.Delete(resp, "/delete") if err != nil { t.Fatal(err) } assert.NotNil(t, resp) - assert.Equal(t, http.MethodDelete, resp.Request.Method) + assert.Equal(t, http.MethodDelete, resp.Resp.Request.Method) } func TestClient_Head(t *testing.T) { @@ -248,11 +168,11 @@ func TestClient_Head(t *testing.T) { t.Fatal(err) } - var resp *http.Response - err = client.Response(&resp).Head(nil, "") + var resp = &slideSshowResponse{} + err = client.Head(resp, "") if err != nil { t.Fatal(err) } assert.NotNil(t, resp) - assert.Equal(t, http.MethodHead, resp.Request.Method) + assert.Equal(t, http.MethodHead, resp.Resp.Request.Method) } diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index 317c462c..f032a294 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -663,30 +663,12 @@ func (b *BeegoHTTPRequest) Response() (*http.Response, error) { // ResponseForValue attempts to resolve the response body to value using an existing method. // Calls Response inner. -// If value type is **string or **[]byte, the func directly passes response body into the pointer. -// Else if response header contain Content-Type, func will call ToJSON\ToXML\ToYAML. -// Finally it will try to parse body as json\yaml\xml, If all attempts fail, an error will be returned +// If response header contain Content-Type, func will call ToJSON\ToXML\ToYAML. +// Else it will try to parse body as json\yaml\xml, If all attempts fail, an error will be returned func (b *BeegoHTTPRequest) ResponseForValue(value interface{}) error { if value == nil { return nil } - // handle basic type - switch v := value.(type) { - case **string: - s, err := b.String() - if err != nil { - return nil - } - *v = &s - return nil - case **[]byte: - bs, err := b.Bytes() - if err != nil { - return nil - } - *v = &bs - return nil - } resp, err := b.Response() if err != nil { From b0d6f3bd2fb7f033c570c0782259a642167fae65 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Fri, 21 May 2021 16:56:52 +0800 Subject: [PATCH 07/15] temp --- client/httplib/client_option_test.go | 12 ----- client/httplib/error_code.go | 8 ++++ client/httplib/httpclient.go | 69 +++++++++++++++++++++++----- client/httplib/httpclient_test.go | 12 ----- client/httplib/httplib.go | 19 ++++---- 5 files changed, 75 insertions(+), 45 deletions(-) diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index 9efeaa24..3c9df88a 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -34,18 +34,6 @@ func (r *respCarrier) SetHttpResponse(resp *http.Response) { r.Resp = resp } -func (r *respCarrier) SetBytes(bytes []byte) { - r.bytes = bytes -} - -func (r *respCarrier) Bytes() []byte { - return r.bytes -} - -func (r *respCarrier) String() string { - return string(r.bytes) -} - func TestOption_WithEnableCookie(t *testing.T) { client, err := NewClient("test", "http://httpbin.org/", WithEnableCookie(true)) diff --git a/client/httplib/error_code.go b/client/httplib/error_code.go index bd349a34..177419ad 100644 --- a/client/httplib/error_code.go +++ b/client/httplib/error_code.go @@ -124,3 +124,11 @@ Make sure that: 1. You pass valid structure pointer to the function; 2. The body is valid YAML document `) + +var UnmarshalResponseToObjectFailed = berror.DefineCode(5001011, moduleName, + "UnmarshalResponseToObjectFailed", ` +Beego trying to unmarshal response's body to structure but failed. +There are several cases that cause this error: +1. You pass valid structure pointer to the function; +2. The body is valid json, Yaml or XML document +`) diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index 69aaa286..ecd557a4 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -15,6 +15,9 @@ package httplib import ( + "bytes" + "io" + "io/ioutil" "net/http" ) @@ -27,17 +30,32 @@ type Client struct { Setting BeegoHTTPSettings } -// If value implement this interface. http.response will saved by SetHttpResponse +// HttpResponseCarrier If value implement HttpResponseCarrier. http.Response will pass to SetHttpResponse type HttpResponseCarrier interface { SetHttpResponse(resp *http.Response) } -// If value implement this interface. bytes of http.response will saved by SetHttpResponse -type ResponseBytesCarrier interface { - // Cause of when user get http.response, the body stream is closed. So need to pass bytes by +// HttpBodyCarrier If value implement HttpBodyCarrier. http.Response.Body will pass to SetReader +type HttpBodyCarrier interface { + SetReader(r *io.ReadCloser) +} + +// HttpBytesCarrier If value implement HttpBytesCarrier. +// All the byte in http.Response.Body will pass to SetBytes +type HttpBytesCarrier interface { SetBytes(bytes []byte) } +// HttpStatusCarrier If value implement HttpStatusCarrier. http.Response.StatusCode will pass to SetStatusCode +type HttpStatusCarrier interface { + SetStatusCode(status int) +} + +// HttpHeaderCarrier If value implement HttpHeaderCarrier. http.Response.Header will pass to SetHeader +type HttpHeadersCarrier interface { + SetHeader(header map[string][]string) +} + // NewClient return a new http client func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, error) { res := &Client{ @@ -67,18 +85,47 @@ func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error if err != nil { return err } - if carrier, ok := (value).(HttpResponseCarrier); ok { - (carrier).SetHttpResponse(resp) - } - if carrier, ok := (value).(ResponseBytesCarrier); ok { - bytes, err := req.Bytes() + + switch carrier := value.(type) { + case HttpResponseCarrier: + b, err := req.Bytes() if err != nil { return err } - (carrier).SetBytes(bytes) + resp.Body = ioutil.NopCloser(bytes.NewReader(b)) + carrier.SetHttpResponse(resp) + fallthrough + case HttpBodyCarrier: + b, err := req.Bytes() + if err != nil { + return err + } + reader := ioutil.NopCloser(bytes.NewReader(b)) + carrier.SetReader(&reader) + fallthrough + case HttpBytesCarrier: + b, err := req.Bytes() + if err != nil { + return err + } + carrier.SetBytes(b) + fallthrough + case HttpStatusCarrier: + resp, err := req.Response() + if err != nil { + return err + } + carrier.SetStatusCode(resp.StatusCode) + fallthrough + case HttpHeadersCarrier: + resp, err := req.Response() + if err != nil { + return err + } + carrier.SetHeader(resp.Header) } - return req.ResponseForValue(value) + return req.ToValue(value) } // Get Send a GET request and try to give its result value diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go index f006f18f..4a0ae843 100644 --- a/client/httplib/httpclient_test.go +++ b/client/httplib/httpclient_test.go @@ -55,18 +55,6 @@ func (s *slideSshowResponse) SetHttpResponse(resp *http.Response) { s.Resp = resp } -func (s *slideSshowResponse) SetBytes(bytes []byte) { - s.bytes = bytes -} - -func (s *slideSshowResponse) Bytes() []byte { - return s.bytes -} - -func (s *slideSshowResponse) String() string { - return string(s.bytes) -} - func TestClient_Get(t *testing.T) { client, err := NewClient("test", "http://httpbin.org/") if err != nil { diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index f032a294..eb51f3a5 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -656,16 +656,11 @@ func (b *BeegoHTTPRequest) ToYAML(v interface{}) error { UnmarshalYAMLResponseToObjectFailed, "unmarshal yaml body to object failed.") } -// Response executes request client gets response manually. -func (b *BeegoHTTPRequest) Response() (*http.Response, error) { - return b.getResponse() -} - -// ResponseForValue attempts to resolve the response body to value using an existing method. +// ToValue attempts to resolve the response body to value using an existing method. // Calls Response inner. // If response header contain Content-Type, func will call ToJSON\ToXML\ToYAML. // Else it will try to parse body as json\yaml\xml, If all attempts fail, an error will be returned -func (b *BeegoHTTPRequest) ResponseForValue(value interface{}) error { +func (b *BeegoHTTPRequest) ToValue(value interface{}) error { if value == nil { return nil } @@ -674,8 +669,8 @@ func (b *BeegoHTTPRequest) ResponseForValue(value interface{}) error { if err != nil { return err } - contentType := strings.Split(resp.Header.Get(contentTypeKey), ";")[0] + contentType := strings.Split(resp.Header.Get(contentTypeKey), ";")[0] // try to parse it as content type switch contentType { case "application/json": @@ -697,8 +692,12 @@ func (b *BeegoHTTPRequest) ResponseForValue(value interface{}) error { return nil } - // TODO add new error type about can't parse body - return berror.Error(UnsupportedBodyType, "unsupported body data") + return berror.Error(UnmarshalResponseToObjectFailed, "unmarshal body to object failed.") +} + +// Response executes request client gets response manually. +func (b *BeegoHTTPRequest) Response() (*http.Response, error) { + return b.getResponse() } // TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field. From 4b1619b105a1d9c39c633c3a25feb1d0a227efde Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Sat, 22 May 2021 14:58:40 +0800 Subject: [PATCH 08/15] add more carrier --- client/httplib/client_option_test.go | 8 +-- client/httplib/httpclient.go | 50 ++++++++++--------- client/httplib/httpclient_test.go | 73 ++++++++++++++++++++++++---- 3 files changed, 94 insertions(+), 37 deletions(-) diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index 3c9df88a..5b1420d7 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -26,12 +26,14 @@ import ( ) type respCarrier struct { - Resp *http.Response bytes []byte } -func (r *respCarrier) SetHttpResponse(resp *http.Response) { - r.Resp = resp +func (r *respCarrier) SetBytes(bytes []byte) { + r.bytes = bytes +} +func (r *respCarrier) String() string { + return string(r.bytes) } func TestOption_WithEnableCookie(t *testing.T) { diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index ecd557a4..70edbc1a 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -37,7 +37,7 @@ type HttpResponseCarrier interface { // HttpBodyCarrier If value implement HttpBodyCarrier. http.Response.Body will pass to SetReader type HttpBodyCarrier interface { - SetReader(r *io.ReadCloser) + SetReader(r io.ReadCloser) } // HttpBytesCarrier If value implement HttpBytesCarrier. @@ -80,52 +80,54 @@ func (c *Client) customReq(req *BeegoHTTPRequest, opts []BeegoHttpRequestOption) // handleResponse try to parse body to meaningful value func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error { - // send request - resp, err := req.Response() + err := c.handleCarrier(value, req) if err != nil { return err } - switch carrier := value.(type) { - case HttpResponseCarrier: + return req.ToValue(value) +} + +// handleCarrier set http data to value +func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { + resp, err := req.Response() + if err != nil { + return err + } + if value == nil { + return err + } + + if carrier, ok := value.(HttpResponseCarrier); ok { b, err := req.Bytes() if err != nil { return err } resp.Body = ioutil.NopCloser(bytes.NewReader(b)) carrier.SetHttpResponse(resp) - fallthrough - case HttpBodyCarrier: + } + if carrier, ok := value.(HttpBodyCarrier); ok { b, err := req.Bytes() if err != nil { return err } reader := ioutil.NopCloser(bytes.NewReader(b)) - carrier.SetReader(&reader) - fallthrough - case HttpBytesCarrier: + carrier.SetReader(reader) + } + if carrier, ok := value.(HttpBytesCarrier); ok { b, err := req.Bytes() if err != nil { return err } carrier.SetBytes(b) - fallthrough - case HttpStatusCarrier: - resp, err := req.Response() - if err != nil { - return err - } + } + if carrier, ok := value.(HttpStatusCarrier); ok { carrier.SetStatusCode(resp.StatusCode) - fallthrough - case HttpHeadersCarrier: - resp, err := req.Response() - if err != nil { - return err - } + } + if carrier, ok := value.(HttpHeadersCarrier); ok { carrier.SetHeader(resp.Header) } - - return req.ToValue(value) + return nil } // Get Send a GET request and try to give its result value diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go index 4a0ae843..daedece3 100644 --- a/client/httplib/httpclient_test.go +++ b/client/httplib/httpclient_test.go @@ -16,6 +16,8 @@ package httplib import ( "encoding/xml" + "io" + "io/ioutil" "net/http" "testing" @@ -29,13 +31,40 @@ func TestNewClient(t *testing.T) { assert.Equal(t, true, client.Setting.EnableCookie) } -type slideSshowResponse struct { - Resp *http.Response - bytes []byte +type slideShowResponse struct { + Resp *http.Response + bytes []byte + StatusCode int + Body io.ReadCloser + Header map[string][]string Slideshow slideshow `json:"slideshow" yaml:"slideshow"` } +func (r *slideShowResponse) SetHttpResponse(resp *http.Response) { + r.Resp = resp +} + +func (r *slideShowResponse) SetBytes(bytes []byte) { + r.bytes = bytes +} + +func (r *slideShowResponse) SetReader(reader io.ReadCloser) { + r.Body = reader +} + +func (r *slideShowResponse) SetStatusCode(status int) { + r.StatusCode = status +} + +func (r *slideShowResponse) SetHeader(header map[string][]string) { + r.Header = header +} + +func (r *slideShowResponse) String() string { + return string(r.bytes) +} + type slideshow struct { XMLName xml.Name `xml:"slideshow"` @@ -51,8 +80,32 @@ type slide struct { Title string `json:"title" yaml:"title" xml:"title"` } -func (s *slideSshowResponse) SetHttpResponse(resp *http.Response) { - s.Resp = resp +func TestClient_handleCarrier(t *testing.T) { + v := "beego" + client, err := NewClient("test", "http://httpbin.org/", + WithUserAgent(v)) + if err != nil { + t.Fatal(err) + } + + var s = &slideShowResponse{} + err = client.Get(s, "/json") + if err != nil { + t.Fatal(err) + } + defer s.Body.Close() + + assert.NotNil(t, s.Resp) + assert.NotNil(t, s.Body) + assert.Equal(t, "429", s.Header["Content-Length"][0]) + assert.Equal(t, 200, s.StatusCode) + + b, err := ioutil.ReadAll(s.Body) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, 429, len(b)) + assert.Equal(t, s.String(), string(b)) } func TestClient_Get(t *testing.T) { @@ -62,7 +115,7 @@ func TestClient_Get(t *testing.T) { } // json - var s *slideSshowResponse + var s *slideShowResponse err = client.Get(&s, "/json") if err != nil { t.Fatal(err) @@ -99,7 +152,7 @@ func TestClient_Post(t *testing.T) { t.Fatal(err) } - var resp = &slideSshowResponse{} + var resp = &slideShowResponse{} err = client.Get(resp, "/json") if err != nil { t.Fatal(err) @@ -120,7 +173,7 @@ func TestClient_Put(t *testing.T) { t.Fatal(err) } - var resp = &slideSshowResponse{} + var resp = &slideShowResponse{} err = client.Get(resp, "/json") if err != nil { t.Fatal(err) @@ -141,7 +194,7 @@ func TestClient_Delete(t *testing.T) { t.Fatal(err) } - var resp = &slideSshowResponse{} + var resp = &slideShowResponse{} err = client.Delete(resp, "/delete") if err != nil { t.Fatal(err) @@ -156,7 +209,7 @@ func TestClient_Head(t *testing.T) { t.Fatal(err) } - var resp = &slideSshowResponse{} + var resp = &slideShowResponse{} err = client.Head(resp, "") if err != nil { t.Fatal(err) From 84fa2ccd9aa149d2eaa9e726b3e520839b4346bd Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Sun, 23 May 2021 23:01:01 +0800 Subject: [PATCH 09/15] fix lint temp --- client/httplib/README.md | 4 +-- client/httplib/client_option.go | 20 +++++++------- client/httplib/httpclient.go | 38 +++++++++++++-------------- client/httplib/httplib.go | 23 +++++++--------- client/httplib/mock/mock_condition.go | 2 +- go.mod | 1 + go.sum | 6 +++++ 7 files changed, 49 insertions(+), 45 deletions(-) diff --git a/client/httplib/README.md b/client/httplib/README.md index 1d22f341..a5723c6d 100644 --- a/client/httplib/README.md +++ b/client/httplib/README.md @@ -9,7 +9,7 @@ httplib is an libs help you to curl remote url. you can use Get to crawl data. import "github.com/beego/beego/v2/client/httplib" - + str, err := httplib.Get("http://beego.me/").String() if err != nil { // error @@ -39,7 +39,7 @@ Example: // GET httplib.Get("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second) - + // POST httplib.Post("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second) diff --git a/client/httplib/client_option.go b/client/httplib/client_option.go index e7402b8c..db81ec50 100644 --- a/client/httplib/client_option.go +++ b/client/httplib/client_option.go @@ -22,7 +22,7 @@ import ( ) type ClientOption func(client *Client) -type BeegoHttpRequestOption func(request *BeegoHTTPRequest) +type BeegoHTTPRequestOption func(request *BeegoHTTPRequest) // WithEnableCookie will enable cookie in all subsequent request func WithEnableCookie(enable bool) ClientOption { @@ -83,28 +83,28 @@ func WithEnableGzip(enable bool) ClientOption { // BeegoHttpRequestOption // WithTimeout sets connect time out and read-write time out for BeegoRequest. -func WithTimeout(connectTimeout, readWriteTimeout time.Duration) BeegoHttpRequestOption { +func WithTimeout(connectTimeout, readWriteTimeout time.Duration) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.SetTimeout(connectTimeout, readWriteTimeout) } } // WithHeader adds header item string in request. -func WithHeader(key, value string) BeegoHttpRequestOption { +func WithHeader(key, value string) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.Header(key, value) } } // WithCookie adds a cookie to the request. -func WithCookie(cookie *http.Cookie) BeegoHttpRequestOption { +func WithCookie(cookie *http.Cookie) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.Header("Cookie", cookie.String()) } } // Withtokenfactory adds a custom function to set Authorization -func WithTokenFactory(tokenFactory func() string) BeegoHttpRequestOption { +func WithTokenFactory(tokenFactory func() string) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { t := tokenFactory() @@ -113,7 +113,7 @@ func WithTokenFactory(tokenFactory func() string) BeegoHttpRequestOption { } // WithBasicAuth adds a custom function to set basic auth -func WithBasicAuth(basicAuth func() (string, string)) BeegoHttpRequestOption { +func WithBasicAuth(basicAuth func() (string, string)) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { username, password := basicAuth() request.SetBasicAuth(username, password) @@ -121,21 +121,21 @@ func WithBasicAuth(basicAuth func() (string, string)) BeegoHttpRequestOption { } // WithFilters will use the filter as the invocation filters -func WithFilters(fcs ...FilterChain) BeegoHttpRequestOption { +func WithFilters(fcs ...FilterChain) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.SetFilters(fcs...) } } // WithContentType adds ContentType in header -func WithContentType(contentType string) BeegoHttpRequestOption { +func WithContentType(contentType string) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.Header(contentTypeKey, contentType) } } // WithParam adds query param in to request. -func WithParam(key, value string) BeegoHttpRequestOption { +func WithParam(key, value string) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.Param(key, value) } @@ -145,7 +145,7 @@ func WithParam(key, value string) BeegoHttpRequestOption { // default is 0 (never retry) // -1 retry indefinitely (forever) // Other numbers specify the exact retry amount -func WithRetry(times int, delay time.Duration) BeegoHttpRequestOption { +func WithRetry(times int, delay time.Duration) BeegoHTTPRequestOption { return func(request *BeegoHTTPRequest) { request.Retries(times) request.RetryDelay(delay) diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index 70edbc1a..45916a40 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -25,29 +25,29 @@ import ( type Client struct { Name string Endpoint string - CommonOpts []BeegoHttpRequestOption + CommonOpts []BeegoHTTPRequestOption Setting BeegoHTTPSettings } -// HttpResponseCarrier If value implement HttpResponseCarrier. http.Response will pass to SetHttpResponse -type HttpResponseCarrier interface { +// HTTPResponseCarrier If value implement HTTPResponseCarrier. http.Response will pass to SetHttpResponse +type HTTPResponseCarrier interface { SetHttpResponse(resp *http.Response) } -// HttpBodyCarrier If value implement HttpBodyCarrier. http.Response.Body will pass to SetReader -type HttpBodyCarrier interface { +// HTTPBodyCarrier If value implement HTTPBodyCarrier. http.Response.Body will pass to SetReader +type HTTPBodyCarrier interface { SetReader(r io.ReadCloser) } -// HttpBytesCarrier If value implement HttpBytesCarrier. +// HTTPBytesCarrier If value implement HTTPBytesCarrier. // All the byte in http.Response.Body will pass to SetBytes -type HttpBytesCarrier interface { +type HTTPBytesCarrier interface { SetBytes(bytes []byte) } -// HttpStatusCarrier If value implement HttpStatusCarrier. http.Response.StatusCode will pass to SetStatusCode -type HttpStatusCarrier interface { +// HTTPStatusCarrier If value implement HTTPStatusCarrier. http.Response.StatusCode will pass to SetStatusCode +type HTTPStatusCarrier interface { SetStatusCode(status int) } @@ -70,7 +70,7 @@ func NewClient(name string, endpoint string, opts ...ClientOption) (*Client, err return res, nil } -func (c *Client) customReq(req *BeegoHTTPRequest, opts []BeegoHttpRequestOption) { +func (c *Client) customReq(req *BeegoHTTPRequest, opts []BeegoHTTPRequestOption) { req.Setting(c.Setting) opts = append(c.CommonOpts, opts...) for _, o := range opts { @@ -98,7 +98,7 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { return err } - if carrier, ok := value.(HttpResponseCarrier); ok { + if carrier, ok := value.(HTTPResponseCarrier); ok { b, err := req.Bytes() if err != nil { return err @@ -106,7 +106,7 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { resp.Body = ioutil.NopCloser(bytes.NewReader(b)) carrier.SetHttpResponse(resp) } - if carrier, ok := value.(HttpBodyCarrier); ok { + if carrier, ok := value.(HTTPBodyCarrier); ok { b, err := req.Bytes() if err != nil { return err @@ -114,14 +114,14 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { reader := ioutil.NopCloser(bytes.NewReader(b)) carrier.SetReader(reader) } - if carrier, ok := value.(HttpBytesCarrier); ok { + if carrier, ok := value.(HTTPBytesCarrier); ok { b, err := req.Bytes() if err != nil { return err } carrier.SetBytes(b) } - if carrier, ok := value.(HttpStatusCarrier); ok { + if carrier, ok := value.(HTTPStatusCarrier); ok { carrier.SetStatusCode(resp.StatusCode) } if carrier, ok := value.(HttpHeadersCarrier); ok { @@ -131,14 +131,14 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { } // Get Send a GET request and try to give its result value -func (c *Client) Get(value interface{}, path string, opts ...BeegoHttpRequestOption) error { +func (c *Client) Get(value interface{}, path string, opts ...BeegoHTTPRequestOption) error { req := Get(c.Endpoint + path) c.customReq(req, opts) return c.handleResponse(value, req) } // Post Send a POST request and try to give its result value -func (c *Client) Post(value interface{}, path string, body interface{}, opts ...BeegoHttpRequestOption) error { +func (c *Client) Post(value interface{}, path string, body interface{}, opts ...BeegoHTTPRequestOption) error { req := Post(c.Endpoint + path) c.customReq(req, opts) if body != nil { @@ -148,7 +148,7 @@ func (c *Client) Post(value interface{}, path string, body interface{}, opts ... } // Put Send a Put request and try to give its result value -func (c *Client) Put(value interface{}, path string, body interface{}, opts ...BeegoHttpRequestOption) error { +func (c *Client) Put(value interface{}, path string, body interface{}, opts ...BeegoHTTPRequestOption) error { req := Put(c.Endpoint + path) c.customReq(req, opts) if body != nil { @@ -158,14 +158,14 @@ func (c *Client) Put(value interface{}, path string, body interface{}, opts ...B } // Delete Send a Delete request and try to give its result value -func (c *Client) Delete(value interface{}, path string, opts ...BeegoHttpRequestOption) error { +func (c *Client) Delete(value interface{}, path string, opts ...BeegoHTTPRequestOption) error { req := Delete(c.Endpoint + path) c.customReq(req, opts) return c.handleResponse(value, req) } // Head Send a Head request and try to give its result value -func (c *Client) Head(value interface{}, path string, opts ...BeegoHttpRequestOption) error { +func (c *Client) Head(value interface{}, path string, opts ...BeegoHTTPRequestOption) error { req := Head(c.Endpoint + path) c.customReq(req, opts) return c.handleResponse(value, req) diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index eb51f3a5..df53bb9c 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -124,7 +124,6 @@ type BeegoHTTPRequest struct { setting BeegoHTTPSettings resp *http.Response body []byte - dump []byte } // GetRequest returns the request object @@ -199,7 +198,7 @@ func (b *BeegoHTTPRequest) SetHost(host string) *BeegoHTTPRequest { // SetProtocolVersion sets the protocol version for incoming requests. // Client requests always use HTTP/1.1 func (b *BeegoHTTPRequest) SetProtocolVersion(vers string) *BeegoHTTPRequest { - if len(vers) == 0 { + if vers == "" { vers = "HTTP/1.1" } @@ -511,18 +510,16 @@ func (b *BeegoHTTPRequest) buildTrans() http.RoundTripper { DialContext: TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout), MaxIdleConnsPerHost: 100, } - } else { + } else if t, ok := trans.(*http.Transport); ok { // if b.transport is *http.Transport then set the settings. - if t, ok := trans.(*http.Transport); ok { - if t.TLSClientConfig == nil { - t.TLSClientConfig = b.setting.TLSClientConfig - } - if t.Proxy == nil { - t.Proxy = b.setting.Proxy - } - if t.DialContext == nil { - t.DialContext = TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout) - } + if t.TLSClientConfig == nil { + t.TLSClientConfig = b.setting.TLSClientConfig + } + if t.Proxy == nil { + t.Proxy = b.setting.Proxy + } + if t.DialContext == nil { + t.DialContext = TimeoutDialerCtx(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout) } } return trans diff --git a/client/httplib/mock/mock_condition.go b/client/httplib/mock/mock_condition.go index 53d3d703..912699ff 100644 --- a/client/httplib/mock/mock_condition.go +++ b/client/httplib/mock/mock_condition.go @@ -57,7 +57,7 @@ func NewSimpleCondition(path string, opts ...simpleConditionOption) *SimpleCondi } func (sc *SimpleCondition) Match(ctx context.Context, req *httplib.BeegoHTTPRequest) bool { - res := true + var res bool if len(sc.path) > 0 { res = sc.matchPath(ctx, req) } else if len(sc.pathReg) > 0 { diff --git a/go.mod b/go.mod index 0305be1e..99b24947 100644 --- a/go.mod +++ b/go.mod @@ -37,4 +37,5 @@ require ( golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a google.golang.org/grpc v1.37.1 gopkg.in/yaml.v2 v2.4.0 + mvdan.cc/gofumpt v0.1.1 // indirect ) diff --git a/go.sum b/go.sum index c144e80d..a20c509f 100644 --- a/go.sum +++ b/go.sum @@ -333,6 +333,7 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -427,6 +428,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -512,6 +515,7 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -589,6 +593,8 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +mvdan.cc/gofumpt v0.1.1 h1:bi/1aS/5W00E2ny5q65w9SnKpWEF/UIOqDYBILpo9rA= +mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From 162247b8a5268b3da0f871dd1ae918b2a660d460 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Mon, 24 May 2021 11:26:52 +0800 Subject: [PATCH 10/15] fix lint --- client/httplib/httpclient_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go index daedece3..e13bdf74 100644 --- a/client/httplib/httpclient_test.go +++ b/client/httplib/httpclient_test.go @@ -199,6 +199,7 @@ func TestClient_Delete(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Resp.Body.Close() assert.NotNil(t, resp) assert.Equal(t, http.MethodDelete, resp.Resp.Request.Method) } @@ -214,6 +215,7 @@ func TestClient_Head(t *testing.T) { if err != nil { t.Fatal(err) } + defer resp.Resp.Body.Close() assert.NotNil(t, resp) assert.Equal(t, http.MethodHead, resp.Resp.Request.Method) } From 5f5afc111a964760332e5d4abe6a660e6155e326 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Mon, 24 May 2021 14:45:44 +0800 Subject: [PATCH 11/15] add xml-roundtrip-validator --- client/httplib/httplib.go | 5 +++++ go.mod | 1 + go.sum | 3 +++ 3 files changed, 9 insertions(+) diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index df53bb9c..ec6c2013 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -49,6 +49,7 @@ import ( "strings" "time" + xrv "github.com/mattermost/xml-roundtrip-validator" "gopkg.in/yaml.v2" "github.com/beego/beego/v2/core/berror" @@ -638,6 +639,10 @@ func (b *BeegoHTTPRequest) ToXML(v interface{}) error { if err != nil { return err } + if err := xrv.Validate(bytes.NewReader(data)); err != nil { + panic(err) + } + return berror.Wrap(xml.Unmarshal(data, v), UnmarshalXMLResponseToObjectFailed, "unmarshal xml body to object failed.") } diff --git a/go.mod b/go.mod index 99b24947..abef3a2b 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6 github.com/lib/pq v1.10.2 + github.com/mattermost/xml-roundtrip-validator v0.1.0 github.com/mattn/go-sqlite3 v1.14.7 github.com/mitchellh/mapstructure v1.4.1 github.com/opentracing/opentracing-go v1.2.0 diff --git a/go.sum b/go.sum index a20c509f..a60bd832 100644 --- a/go.sum +++ b/go.sum @@ -226,6 +226,8 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= +github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -367,6 +369,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112 h1:NBrpnvz0pDPf3+HXZ1C9GcJd1DTpWDLcLWZhNq6uP7o= From 00b6c32dc2888b58dac4f730e2b19d8bd3ba5fc2 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Mon, 24 May 2021 15:17:02 +0800 Subject: [PATCH 12/15] remove panic --- client/httplib/client_option_test.go | 2 +- client/httplib/httplib.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index 5b1420d7..fd7ba7df 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -246,7 +246,7 @@ func TestOption_WithRetry(t *testing.T) { } retryAmount := 1 - retryDelay := 1400 * time.Millisecond + retryDelay := 200 * time.Millisecond startTime := time.Now().UnixNano() / int64(time.Millisecond) _ = client.Get(nil, "", WithRetry(retryAmount, retryDelay)) diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index ec6c2013..be4f388e 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -640,7 +640,7 @@ func (b *BeegoHTTPRequest) ToXML(v interface{}) error { return err } if err := xrv.Validate(bytes.NewReader(data)); err != nil { - panic(err) + return err } return berror.Wrap(xml.Unmarshal(data, v), From 8a241927629c65855a1dad0427c678b8d498b53a Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Tue, 25 May 2021 09:56:54 +0800 Subject: [PATCH 13/15] fix lint --- CHANGELOG.md | 1 + client/httplib/client_option.go | 6 ++++-- client/httplib/httpclient.go | 11 ++++++----- client/httplib/httpclient_test.go | 2 +- client/httplib/httplib.go | 6 +----- go.mod | 1 - go.sum | 2 -- 7 files changed, 13 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bb3a4f7..87a34e0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # developing - Add http client and option func. [4455](https://github.com/beego/beego/issues/4455) +- Add: Convenient way to generate mock object [4620](https://github.com/beego/beego/issues/4620) - Infra: use dependabot to update dependencies. [4623](https://github.com/beego/beego/pull/4623) - Lint: use golangci-lint. [4619](https://github.com/beego/beego/pull/4619) - Chore: format code. [4615](https://github.com/beego/beego/pull/4615) diff --git a/client/httplib/client_option.go b/client/httplib/client_option.go index db81ec50..f970e67d 100644 --- a/client/httplib/client_option.go +++ b/client/httplib/client_option.go @@ -21,8 +21,10 @@ import ( "time" ) -type ClientOption func(client *Client) -type BeegoHTTPRequestOption func(request *BeegoHTTPRequest) +type ( + ClientOption func(client *Client) + BeegoHTTPRequestOption func(request *BeegoHTTPRequest) +) // WithEnableCookie will enable cookie in all subsequent request func WithEnableCookie(enable bool) ClientOption { diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index 45916a40..32ab4d6c 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -30,9 +30,9 @@ type Client struct { Setting BeegoHTTPSettings } -// HTTPResponseCarrier If value implement HTTPResponseCarrier. http.Response will pass to SetHttpResponse +// HTTPResponseCarrier If value implement HTTPResponseCarrier. http.Response will pass to SetHTTPResponse type HTTPResponseCarrier interface { - SetHttpResponse(resp *http.Response) + SetHTTPResponse(resp *http.Response) } // HTTPBodyCarrier If value implement HTTPBodyCarrier. http.Response.Body will pass to SetReader @@ -52,7 +52,7 @@ type HTTPStatusCarrier interface { } // HttpHeaderCarrier If value implement HttpHeaderCarrier. http.Response.Header will pass to SetHeader -type HttpHeadersCarrier interface { +type HTTPHeadersCarrier interface { SetHeader(header map[string][]string) } @@ -91,6 +91,7 @@ func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error // handleCarrier set http data to value func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { resp, err := req.Response() + defer resp.Body.Close() if err != nil { return err } @@ -104,7 +105,7 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { return err } resp.Body = ioutil.NopCloser(bytes.NewReader(b)) - carrier.SetHttpResponse(resp) + carrier.SetHTTPResponse(resp) } if carrier, ok := value.(HTTPBodyCarrier); ok { b, err := req.Bytes() @@ -124,7 +125,7 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { if carrier, ok := value.(HTTPStatusCarrier); ok { carrier.SetStatusCode(resp.StatusCode) } - if carrier, ok := value.(HttpHeadersCarrier); ok { + if carrier, ok := value.(HTTPHeadersCarrier); ok { carrier.SetHeader(resp.Header) } return nil diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go index e13bdf74..6437e534 100644 --- a/client/httplib/httpclient_test.go +++ b/client/httplib/httpclient_test.go @@ -41,7 +41,7 @@ type slideShowResponse struct { Slideshow slideshow `json:"slideshow" yaml:"slideshow"` } -func (r *slideShowResponse) SetHttpResponse(resp *http.Response) { +func (r *slideShowResponse) SetHTTPResponse(resp *http.Response) { r.Resp = resp } diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index be4f388e..6a78e13b 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -49,7 +49,6 @@ import ( "strings" "time" - xrv "github.com/mattermost/xml-roundtrip-validator" "gopkg.in/yaml.v2" "github.com/beego/beego/v2/core/berror" @@ -639,10 +638,6 @@ func (b *BeegoHTTPRequest) ToXML(v interface{}) error { if err != nil { return err } - if err := xrv.Validate(bytes.NewReader(data)); err != nil { - return err - } - return berror.Wrap(xml.Unmarshal(data, v), UnmarshalXMLResponseToObjectFailed, "unmarshal xml body to object failed.") } @@ -668,6 +663,7 @@ func (b *BeegoHTTPRequest) ToValue(value interface{}) error { } resp, err := b.Response() + defer resp.Body.Close() if err != nil { return err } diff --git a/go.mod b/go.mod index abef3a2b..99b24947 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/hashicorp/golang-lru v0.5.4 github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6 github.com/lib/pq v1.10.2 - github.com/mattermost/xml-roundtrip-validator v0.1.0 github.com/mattn/go-sqlite3 v1.14.7 github.com/mitchellh/mapstructure v1.4.1 github.com/opentracing/opentracing-go v1.2.0 diff --git a/go.sum b/go.sum index a60bd832..91616ec7 100644 --- a/go.sum +++ b/go.sum @@ -226,8 +226,6 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= -github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= From 2182081831bb8959e88cdd8e69743b3fba56ab03 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Tue, 25 May 2021 10:10:06 +0800 Subject: [PATCH 14/15] fix lint --- client/httplib/client_option_test.go | 17 +++++++++-------- client/httplib/httpclient.go | 4 ++-- client/httplib/httpclient_test.go | 11 +++++------ client/httplib/httplib.go | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index fd7ba7df..28cbc6fe 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -32,6 +32,7 @@ type respCarrier struct { func (r *respCarrier) SetBytes(bytes []byte) { r.bytes = bytes } + func (r *respCarrier) String() string { return string(r.bytes) } @@ -44,7 +45,7 @@ func TestOption_WithEnableCookie(t *testing.T) { } v := "smallfish" - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/cookies/set?k1="+v) if err != nil { t.Fatal(err) @@ -71,7 +72,7 @@ func TestOption_WithUserAgent(t *testing.T) { t.Fatal(err) } - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/headers") if err != nil { t.Fatal(err) @@ -119,7 +120,7 @@ func TestOption_WithHTTPSetting(t *testing.T) { t.Fatal(err) } - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/get") if err != nil { t.Fatal(err) @@ -139,7 +140,7 @@ func TestOption_WithHeader(t *testing.T) { } client.CommonOpts = append(client.CommonOpts, WithHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36")) - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/headers") if err != nil { t.Fatal(err) @@ -162,7 +163,7 @@ func TestOption_WithTokenFactory(t *testing.T) { return "testauth" })) - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/headers") if err != nil { t.Fatal(err) @@ -181,7 +182,7 @@ func TestOption_WithBasicAuth(t *testing.T) { t.Fatal(err) } - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/basic-auth/user/passwd", WithBasicAuth(func() (string, string) { return "user", "passwd" @@ -203,7 +204,7 @@ func TestOption_WithContentType(t *testing.T) { } v := "application/json" - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/headers", WithContentType(v)) if err != nil { t.Fatal(err) @@ -223,7 +224,7 @@ func TestOption_WithParam(t *testing.T) { } v := "smallfish" - var resp = &respCarrier{} + resp := &respCarrier{} err = client.Get(resp, "/get", WithParam("username", v)) if err != nil { t.Fatal(err) diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index 32ab4d6c..ef2e27eb 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -91,14 +91,14 @@ func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error // handleCarrier set http data to value func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { resp, err := req.Response() - defer resp.Body.Close() if err != nil { return err } + defer resp.Body.Close() + if value == nil { return err } - if carrier, ok := value.(HTTPResponseCarrier); ok { b, err := req.Bytes() if err != nil { diff --git a/client/httplib/httpclient_test.go b/client/httplib/httpclient_test.go index 6437e534..6bb00258 100644 --- a/client/httplib/httpclient_test.go +++ b/client/httplib/httpclient_test.go @@ -88,7 +88,7 @@ func TestClient_handleCarrier(t *testing.T) { t.Fatal(err) } - var s = &slideShowResponse{} + s := &slideShowResponse{} err = client.Get(s, "/json") if err != nil { t.Fatal(err) @@ -143,7 +143,6 @@ func TestClient_Get(t *testing.T) { assert.Equal(t, "Sample Slide Show", s.Slideshow.Title) assert.Equal(t, 2, len(s.Slideshow.Slides)) assert.Equal(t, "Overview", s.Slideshow.Slides[1].Title) - } func TestClient_Post(t *testing.T) { @@ -152,7 +151,7 @@ func TestClient_Post(t *testing.T) { t.Fatal(err) } - var resp = &slideShowResponse{} + resp := &slideShowResponse{} err = client.Get(resp, "/json") if err != nil { t.Fatal(err) @@ -173,7 +172,7 @@ func TestClient_Put(t *testing.T) { t.Fatal(err) } - var resp = &slideShowResponse{} + resp := &slideShowResponse{} err = client.Get(resp, "/json") if err != nil { t.Fatal(err) @@ -194,7 +193,7 @@ func TestClient_Delete(t *testing.T) { t.Fatal(err) } - var resp = &slideShowResponse{} + resp := &slideShowResponse{} err = client.Delete(resp, "/delete") if err != nil { t.Fatal(err) @@ -210,7 +209,7 @@ func TestClient_Head(t *testing.T) { t.Fatal(err) } - var resp = &slideShowResponse{} + resp := &slideShowResponse{} err = client.Head(resp, "") if err != nil { t.Fatal(err) diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index 6a78e13b..76583712 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -663,10 +663,10 @@ func (b *BeegoHTTPRequest) ToValue(value interface{}) error { } resp, err := b.Response() - defer resp.Body.Close() if err != nil { return err } + defer resp.Body.Close() contentType := strings.Split(resp.Header.Get(contentTypeKey), ";")[0] // try to parse it as content type From f8df3d1bee5eb46a6e23a6c55bd184aa27cf3485 Mon Sep 17 00:00:00 2001 From: holooooo <844082183@qq.com> Date: Tue, 25 May 2021 15:15:43 +0800 Subject: [PATCH 15/15] fix read on close --- client/httplib/client_option_test.go | 2 +- client/httplib/httpclient.go | 25 +++++++++++++------------ client/httplib/httplib.go | 8 +------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/client/httplib/client_option_test.go b/client/httplib/client_option_test.go index 28cbc6fe..79e5103f 100644 --- a/client/httplib/client_option_test.go +++ b/client/httplib/client_option_test.go @@ -247,7 +247,7 @@ func TestOption_WithRetry(t *testing.T) { } retryAmount := 1 - retryDelay := 200 * time.Millisecond + retryDelay := 800 * time.Millisecond startTime := time.Now().UnixNano() / int64(time.Millisecond) _ = client.Get(nil, "", WithRetry(retryAmount, retryDelay)) diff --git a/client/httplib/httpclient.go b/client/httplib/httpclient.go index ef2e27eb..c2a61fcf 100644 --- a/client/httplib/httpclient.go +++ b/client/httplib/httpclient.go @@ -80,7 +80,13 @@ func (c *Client) customReq(req *BeegoHTTPRequest, opts []BeegoHTTPRequestOption) // handleResponse try to parse body to meaningful value func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error { - err := c.handleCarrier(value, req) + // make sure req.resp is not nil + _, err := req.Bytes() + if err != nil { + return err + } + + err = c.handleCarrier(value, req) if err != nil { return err } @@ -90,22 +96,17 @@ func (c *Client) handleResponse(value interface{}, req *BeegoHTTPRequest) error // handleCarrier set http data to value func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { - resp, err := req.Response() - if err != nil { - return err - } - defer resp.Body.Close() - if value == nil { - return err + return nil } + if carrier, ok := value.(HTTPResponseCarrier); ok { b, err := req.Bytes() if err != nil { return err } - resp.Body = ioutil.NopCloser(bytes.NewReader(b)) - carrier.SetHTTPResponse(resp) + req.resp.Body = ioutil.NopCloser(bytes.NewReader(b)) + carrier.SetHTTPResponse(req.resp) } if carrier, ok := value.(HTTPBodyCarrier); ok { b, err := req.Bytes() @@ -123,10 +124,10 @@ func (c *Client) handleCarrier(value interface{}, req *BeegoHTTPRequest) error { carrier.SetBytes(b) } if carrier, ok := value.(HTTPStatusCarrier); ok { - carrier.SetStatusCode(resp.StatusCode) + carrier.SetStatusCode(req.resp.StatusCode) } if carrier, ok := value.(HTTPHeadersCarrier); ok { - carrier.SetHeader(resp.Header) + carrier.SetHeader(req.resp.Header) } return nil } diff --git a/client/httplib/httplib.go b/client/httplib/httplib.go index 76583712..b102f687 100644 --- a/client/httplib/httplib.go +++ b/client/httplib/httplib.go @@ -662,13 +662,7 @@ func (b *BeegoHTTPRequest) ToValue(value interface{}) error { return nil } - resp, err := b.Response() - if err != nil { - return err - } - defer resp.Body.Close() - - contentType := strings.Split(resp.Header.Get(contentTypeKey), ";")[0] + contentType := strings.Split(b.resp.Header.Get(contentTypeKey), ";")[0] // try to parse it as content type switch contentType { case "application/json":