From c8c25549e73c20788ed9fe222cd5fd6b8bf016d7 Mon Sep 17 00:00:00 2001 From: wangguoliang Date: Thu, 7 Sep 2017 19:01:34 +0800 Subject: [PATCH 1/3] should use time.Since instead of time.Now().Sub Signed-off-by: wgliang --- cache/memory.go | 2 +- context/param/options.go | 2 +- context/param/parsers_test.go | 2 +- orm/orm_log.go | 2 +- parser.go | 2 +- toolbox/profile.go | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cache/memory.go b/cache/memory.go index 57e868cf..784dbecb 100644 --- a/cache/memory.go +++ b/cache/memory.go @@ -38,7 +38,7 @@ func (mi *MemoryItem) isExpire() bool { if mi.lifespan == 0 { return false } - return time.Now().Sub(mi.createdTime) > mi.lifespan + return time.Since(mi.createdTime) > mi.lifespan } // MemoryCache is Memory cache adapter. diff --git a/context/param/options.go b/context/param/options.go index 58bdc3d0..3d5ba013 100644 --- a/context/param/options.go +++ b/context/param/options.go @@ -7,7 +7,7 @@ import ( // MethodParamOption defines a func which apply options on a MethodParam type MethodParamOption func(*MethodParam) -// IsRequired indicates that this param is required and can not be ommited from the http request +// IsRequired indicates that this param is required and can not be omitted from the http request var IsRequired MethodParamOption = func(p *MethodParam) { p.required = true } diff --git a/context/param/parsers_test.go b/context/param/parsers_test.go index b946ba08..7065a28e 100644 --- a/context/param/parsers_test.go +++ b/context/param/parsers_test.go @@ -75,7 +75,7 @@ func checkParser(def testDefinition, t *testing.T, methodParam ...MethodParam) { } convResult, err := safeConvert(reflect.ValueOf(result), toType) if err != nil { - t.Errorf("Convertion error for %v. from value: %v, toType: %v, error: %v", def.strValue, result, toType, err) + t.Errorf("Conversion error for %v. from value: %v, toType: %v, error: %v", def.strValue, result, toType, err) return } if !reflect.DeepEqual(convResult.Interface(), def.expectedValue) { diff --git a/orm/orm_log.go b/orm/orm_log.go index 26c73f9e..915bfd2d 100644 --- a/orm/orm_log.go +++ b/orm/orm_log.go @@ -36,7 +36,7 @@ func NewLog(out io.Writer) *Log { } func debugLogQueies(alias *alias, operaton, query string, t time.Time, err error, args ...interface{}) { - sub := time.Now().Sub(t) / 1e5 + sub := time.Since(t) / 1e5 elsp := float64(int(sub)) / 10.0 flag := " OK" if err != nil { diff --git a/parser.go b/parser.go index f787fd5c..1933c6c6 100644 --- a/parser.go +++ b/parser.go @@ -227,7 +227,7 @@ func parseComment(lines []*ast.Comment) (pc *parsedComment, err error) { } // direct copy from bee\g_docs.go -// analisys params return []string +// analysis params return []string // @Param query form string true "The email for login" // [query form string true "The email for login"] func getparams(str string) []string { diff --git a/toolbox/profile.go b/toolbox/profile.go index 06e40ede..944ec645 100644 --- a/toolbox/profile.go +++ b/toolbox/profile.go @@ -106,7 +106,7 @@ func printGC(memStats *runtime.MemStats, gcstats *debug.GCStats, w io.Writer) { if gcstats.NumGC > 0 { lastPause := gcstats.Pause[0] - elapsed := time.Now().Sub(startTime) + elapsed := time.Since(startTime) overhead := float64(gcstats.PauseTotal) / float64(elapsed) * 100 allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds() @@ -123,7 +123,7 @@ func printGC(memStats *runtime.MemStats, gcstats *debug.GCStats, w io.Writer) { toS(gcstats.PauseQuantiles[99])) } else { // while GC has disabled - elapsed := time.Now().Sub(startTime) + elapsed := time.Since(startTime) allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds() fmt.Fprintf(w, "Alloc:%s Sys:%s Alloc(Rate):%s/s\n", From a7354d2d084003e4122d6e69f7e5ab594fd117b2 Mon Sep 17 00:00:00 2001 From: astaxie Date: Sat, 9 Sep 2017 06:29:38 +0800 Subject: [PATCH 2/3] Revert "should use time.Since instead of time.Now().Sub" --- cache/memory.go | 2 +- context/param/options.go | 2 +- context/param/parsers_test.go | 2 +- orm/orm_log.go | 2 +- parser.go | 2 +- toolbox/profile.go | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cache/memory.go b/cache/memory.go index 784dbecb..57e868cf 100644 --- a/cache/memory.go +++ b/cache/memory.go @@ -38,7 +38,7 @@ func (mi *MemoryItem) isExpire() bool { if mi.lifespan == 0 { return false } - return time.Since(mi.createdTime) > mi.lifespan + return time.Now().Sub(mi.createdTime) > mi.lifespan } // MemoryCache is Memory cache adapter. diff --git a/context/param/options.go b/context/param/options.go index 3d5ba013..58bdc3d0 100644 --- a/context/param/options.go +++ b/context/param/options.go @@ -7,7 +7,7 @@ import ( // MethodParamOption defines a func which apply options on a MethodParam type MethodParamOption func(*MethodParam) -// IsRequired indicates that this param is required and can not be omitted from the http request +// IsRequired indicates that this param is required and can not be ommited from the http request var IsRequired MethodParamOption = func(p *MethodParam) { p.required = true } diff --git a/context/param/parsers_test.go b/context/param/parsers_test.go index 7065a28e..b946ba08 100644 --- a/context/param/parsers_test.go +++ b/context/param/parsers_test.go @@ -75,7 +75,7 @@ func checkParser(def testDefinition, t *testing.T, methodParam ...MethodParam) { } convResult, err := safeConvert(reflect.ValueOf(result), toType) if err != nil { - t.Errorf("Conversion error for %v. from value: %v, toType: %v, error: %v", def.strValue, result, toType, err) + t.Errorf("Convertion error for %v. from value: %v, toType: %v, error: %v", def.strValue, result, toType, err) return } if !reflect.DeepEqual(convResult.Interface(), def.expectedValue) { diff --git a/orm/orm_log.go b/orm/orm_log.go index 915bfd2d..26c73f9e 100644 --- a/orm/orm_log.go +++ b/orm/orm_log.go @@ -36,7 +36,7 @@ func NewLog(out io.Writer) *Log { } func debugLogQueies(alias *alias, operaton, query string, t time.Time, err error, args ...interface{}) { - sub := time.Since(t) / 1e5 + sub := time.Now().Sub(t) / 1e5 elsp := float64(int(sub)) / 10.0 flag := " OK" if err != nil { diff --git a/parser.go b/parser.go index 1933c6c6..f787fd5c 100644 --- a/parser.go +++ b/parser.go @@ -227,7 +227,7 @@ func parseComment(lines []*ast.Comment) (pc *parsedComment, err error) { } // direct copy from bee\g_docs.go -// analysis params return []string +// analisys params return []string // @Param query form string true "The email for login" // [query form string true "The email for login"] func getparams(str string) []string { diff --git a/toolbox/profile.go b/toolbox/profile.go index 944ec645..06e40ede 100644 --- a/toolbox/profile.go +++ b/toolbox/profile.go @@ -106,7 +106,7 @@ func printGC(memStats *runtime.MemStats, gcstats *debug.GCStats, w io.Writer) { if gcstats.NumGC > 0 { lastPause := gcstats.Pause[0] - elapsed := time.Since(startTime) + elapsed := time.Now().Sub(startTime) overhead := float64(gcstats.PauseTotal) / float64(elapsed) * 100 allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds() @@ -123,7 +123,7 @@ func printGC(memStats *runtime.MemStats, gcstats *debug.GCStats, w io.Writer) { toS(gcstats.PauseQuantiles[99])) } else { // while GC has disabled - elapsed := time.Since(startTime) + elapsed := time.Now().Sub(startTime) allocatedRate := float64(memStats.TotalAlloc) / elapsed.Seconds() fmt.Fprintf(w, "Alloc:%s Sys:%s Alloc(Rate):%s/s\n", From 4921014c6415fdfa105c92149b3ffb9001430a2c Mon Sep 17 00:00:00 2001 From: Waleed Gadelkareem Date: Wed, 13 Sep 2017 02:03:46 +0200 Subject: [PATCH 3/3] Add JSON or Apache access log formatting option to config: AccessLogsFormat = JSON_FORMAT or APACHE_FORMAT ref #2738 --- admin_test.go | 1 + config.go | 2 ++ logs/accesslog.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++ router.go | 68 +++++++++++++++++++++++++++------------------- 4 files changed, 113 insertions(+), 27 deletions(-) create mode 100644 logs/accesslog.go diff --git a/admin_test.go b/admin_test.go index 2348792e..a99da6fc 100644 --- a/admin_test.go +++ b/admin_test.go @@ -67,6 +67,7 @@ func oldMap() map[string]interface{} { m["BConfig.WebConfig.Session.SessionDomain"] = BConfig.WebConfig.Session.SessionDomain m["BConfig.WebConfig.Session.SessionDisableHTTPOnly"] = BConfig.WebConfig.Session.SessionDisableHTTPOnly m["BConfig.Log.AccessLogs"] = BConfig.Log.AccessLogs + m["BConfig.Log.AccessLogsFormat"] = BConfig.Log.AccessLogsFormat m["BConfig.Log.FileLineNum"] = BConfig.Log.FileLineNum m["BConfig.Log.Outputs"] = BConfig.Log.Outputs return m diff --git a/config.go b/config.go index e6e99570..d344ec6a 100644 --- a/config.go +++ b/config.go @@ -104,6 +104,7 @@ type SessionConfig struct { // LogConfig holds Log related config type LogConfig struct { AccessLogs bool + AccessLogsFormat string //access log format: JSON_FORMAT, APACHE_FORMAT or empty string FileLineNum bool Outputs map[string]string // Store Adaptor : config } @@ -240,6 +241,7 @@ func newBConfig() *Config { }, Log: LogConfig{ AccessLogs: false, + AccessLogsFormat: "", FileLineNum: true, Outputs: map[string]string{"console": ""}, }, diff --git a/logs/accesslog.go b/logs/accesslog.go new file mode 100644 index 00000000..1245e69b --- /dev/null +++ b/logs/accesslog.go @@ -0,0 +1,69 @@ +// Copyright 2014 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logs + +import ( + "bytes" + "encoding/json" + "time" + "fmt" +) + +const ( + ApacheFormatPattern = "%s - - [%s] \"%s %d %d\" %f %s %s\n" + ApacheFormat = "APACHE_FORMAT" + JsonFormat = "JSON_FORMAT" +) + +type AccessLogRecord struct { + RemoteAddr string `json:"remote_addr"` + RequestTime time.Time `json:"request_time"` + RequestMethod string `json:"request_method"` + Request string `json:"request"` + ServerProtocol string `json:"server_protocol"` + Host string `json:"host"` + Status int `json:"status"` + BodyBytesSent int64 `json:"body_bytes_sent"` + ElapsedTime time.Duration `json:"elapsed_time"` + HttpReferrer string `json:"http_referrer"` + HttpUserAgent string `json:"http_user_agent"` + RemoteUser string `json:"remote_user"` +} + +func (r *AccessLogRecord) JSON() ([]byte, error) { + buffer := &bytes.Buffer{} + encoder := json.NewEncoder(buffer) + encoder.SetEscapeHTML(false) + err := encoder.Encode(r) + return buffer.Bytes(), err +} + +func AccessLog(r *AccessLogRecord, format string) { + var msg string + + if format == ApacheFormat { + timeFormatted := r.RequestTime.Format("02/Jan/2006 03:04:05") + msg = fmt.Sprintf(ApacheFormatPattern, r.RemoteAddr, timeFormatted, r.Request, r.Status, r.BodyBytesSent, + r.ElapsedTime.Seconds(), r.HttpReferrer, r.HttpUserAgent) + } else { + jsonData, err := r.JSON() + if err != nil { + msg = fmt.Sprintf(`{"Error": "%s"}`, err) + } else { + msg = string(jsonData) + } + } + beeLogger.Debug(msg) +} diff --git a/router.go b/router.go index e5a4e80d..146e9af6 100644 --- a/router.go +++ b/router.go @@ -43,7 +43,7 @@ const ( ) const ( - routerTypeBeego = iota + routerTypeBeego = iota routerTypeRESTFul routerTypeHandler ) @@ -845,17 +845,20 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) } Admin: - //admin module record QPS +//admin module record QPS + + statusCode := context.ResponseWriter.Status + if statusCode == 0 { + statusCode = 200 + } + if BConfig.Listen.EnableAdmin { timeDur := time.Since(startTime) pattern := "" if routerInfo != nil { pattern = routerInfo.pattern } - statusCode := context.ResponseWriter.Status - if statusCode == 0 { - statusCode = 200 - } + if FilterMonitorFunc(r.Method, r.URL.Path, timeDur, pattern, statusCode) { if runRouter != nil { go toolbox.StatisticsMap.AddStatistics(r.Method, r.URL.Path, runRouter.Name(), timeDur) @@ -869,36 +872,47 @@ Admin: timeDur := time.Since(startTime) var devInfo string - statusCode := context.ResponseWriter.Status - if statusCode == 0 { - statusCode = 200 - } - iswin := (runtime.GOOS == "windows") statusColor := logs.ColorByStatus(iswin, statusCode) methodColor := logs.ColorByMethod(iswin, r.Method) resetColor := logs.ColorByMethod(iswin, "") - - if findRouter { - if routerInfo != nil { - devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s r:%s", context.Input.IP(), statusColor, statusCode, - resetColor, timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path, - routerInfo.pattern) + if BConfig.Log.AccessLogsFormat != "" { + record := &logs.AccessLogRecord{ + RemoteAddr: context.Input.IP(), + RequestTime: startTime, + RequestMethod: r.Method, + Request: fmt.Sprintf("%s %s %s", r.Method, r.RequestURI, r.Proto), + ServerProtocol: r.Proto, + Host: r.Host, + Status: statusCode, + ElapsedTime: timeDur, + HttpReferrer: r.Header.Get("Referer"), + HttpUserAgent: r.Header.Get("User-Agent"), + RemoteUser: r.Header.Get("Remote-User"), + BodyBytesSent: 0, //@todo this one is missing! + } + logs.AccessLog(record, BConfig.Log.AccessLogsFormat) + }else { + if findRouter { + if routerInfo != nil { + devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s r:%s", context.Input.IP(), statusColor, statusCode, + resetColor, timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path, + routerInfo.pattern) + } else { + devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s", context.Input.IP(), statusColor, statusCode, resetColor, + timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path) + } } else { devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s", context.Input.IP(), statusColor, statusCode, resetColor, - timeDur.String(), "match", methodColor, r.Method, resetColor, r.URL.Path) + timeDur.String(), "nomatch", methodColor, r.Method, resetColor, r.URL.Path) + } + if iswin { + logs.W32Debug(devInfo) + } else { + logs.Debug(devInfo) } - } else { - devInfo = fmt.Sprintf("|%15s|%s %3d %s|%13s|%8s|%s %-7s %s %-3s", context.Input.IP(), statusColor, statusCode, resetColor, - timeDur.String(), "nomatch", methodColor, r.Method, resetColor, r.URL.Path) - } - if iswin { - logs.W32Debug(devInfo) - } else { - logs.Debug(devInfo) } } - // Call WriteHeader if status code has been set changed if context.Output.Status != 0 { context.ResponseWriter.WriteHeader(context.Output.Status)