From 1f2f0b30f482904991bc9fb872b1cab31ec26623 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 22 Sep 2015 22:02:56 +0800 Subject: [PATCH 01/27] mem zip file refactor and test --- memzipfile.go | 97 ++++++++++++++++++++++++++-------------------- memzipfile_test.go | 66 +++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 42 deletions(-) create mode 100644 memzipfile_test.go diff --git a/memzipfile.go b/memzipfile.go index 594f1060..1e5239f5 100644 --- a/memzipfile.go +++ b/memzipfile.go @@ -15,6 +15,7 @@ package beego import ( + "bufio" "bytes" "compress/flate" "compress/gzip" @@ -49,57 +50,26 @@ func openMemZipFile(path string, zip string) (*memFile, error) { modTime := osFileInfo.ModTime() fileSize := osFileInfo.Size() + mapKey := zip + ":" + path lock.RLock() - cfi, ok := menFileInfoMap[zip+":"+path] + cfi, ok := menFileInfoMap[mapKey] lock.RUnlock() if !(ok && cfi.ModTime() == modTime && cfi.fileSize == fileSize) { - var content []byte - if zip == "gzip" { - var zipBuf bytes.Buffer - gzipWriter, e := gzip.NewWriterLevel(&zipBuf, gzip.BestCompression) - if e != nil { - return nil, e - } - _, e = io.Copy(gzipWriter, osFile) - gzipWriter.Close() - if e != nil { - return nil, e - } - content, e = ioutil.ReadAll(&zipBuf) - if e != nil { - return nil, e - } - } else if zip == "deflate" { - var zipBuf bytes.Buffer - deflateWriter, e := flate.NewWriter(&zipBuf, flate.BestCompression) - if e != nil { - return nil, e - } - _, e = io.Copy(deflateWriter, osFile) - deflateWriter.Close() - if e != nil { - return nil, e - } - content, e = ioutil.ReadAll(&zipBuf) - if e != nil { - return nil, e - } - } else { - content, e = ioutil.ReadAll(osFile) - if e != nil { - return nil, e - } - } - - cfi = &memFileInfo{osFileInfo, modTime, content, int64(len(content)), fileSize} lock.Lock() defer lock.Unlock() - menFileInfoMap[zip+":"+path] = cfi + if cfi, ok = menFileInfoMap[mapKey]; !ok { + cfi, e = newMenFileInfo(osFile, osFileInfo, zip) + if e != nil { + return nil, e + } + menFileInfoMap[mapKey] = cfi + } } + return &memFile{fi: cfi, offset: 0}, nil } -// MemFileInfo contains a compressed file bytes and file information. +// memFileInfo contains a compressed file bytes and file information. // it implements os.FileInfo interface. type memFileInfo struct { os.FileInfo @@ -109,6 +79,49 @@ type memFileInfo struct { fileSize int64 } +// newMenFileInfo return a memFileInfo from file by zip type +func newMenFileInfo(file *os.File, fileInfo os.FileInfo, zip string) (*memFileInfo, error) { + var content []byte + var zipBuf bytes.Buffer + var fileWriter io.Writer + var err error + + switch zip { + case "gzip": + fileWriter, err = gzip.NewWriterLevel(&zipBuf, gzip.BestCompression) + case "deflate": + fileWriter, err = flate.NewWriter(&zipBuf, flate.BestCompression) + default: + fileWriter = bufio.NewWriter(&zipBuf) + } + if err != nil { + return nil, err + } + + _, err = io.Copy(fileWriter, file) + if err != nil { + return nil, err + } + + switch fileWriter.(type) { + case io.WriteCloser: + fileWriter.(io.WriteCloser).Close() + } + + content, err = ioutil.ReadAll(&zipBuf) + if err != nil { + return nil, err + } + + return &memFileInfo{ + FileInfo: fileInfo, + modTime: fileInfo.ModTime(), + content: content, + contentSize: int64(len(content)), + fileSize: fileInfo.Size(), + }, nil +} + // Name returns the compressed filename. func (fi *memFileInfo) Name() string { return fi.Name() diff --git a/memzipfile_test.go b/memzipfile_test.go new file mode 100644 index 00000000..a8c67ead --- /dev/null +++ b/memzipfile_test.go @@ -0,0 +1,66 @@ +package beego + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "io" + "io/ioutil" + "os" + "testing" +) + +const licenseFile = "./LICENSE" + +func TestOpenMemZipFile_1(t *testing.T) { + mf, err := openMemZipFile(licenseFile, "") + if err != nil { + t.Fail() + } + file, _ := os.Open(licenseFile) + content, _ := ioutil.ReadAll(file) + assetMenFileAndContent(mf, content, t) +} + +func assetMenFileAndContent(mf *memFile, content []byte, t *testing.T) { + if mf.fi.contentSize != int64(len(content)) { + t.Log("content size not same") + t.Fail() + } + for i, v := range content { + if v != mf.fi.content[i] { + t.Log("content not same") + t.Fail() + } + } + if len(menFileInfoMap) == 0 { + t.Log("men map is empty") + t.Fail() + } +} +func TestOpenMemZipFile_2(t *testing.T) { + mf, err := openMemZipFile(licenseFile, "gzip") + if err != nil { + t.Fail() + } + file, _ := os.Open(licenseFile) + var zipBuf bytes.Buffer + fileWriter, _ := gzip.NewWriterLevel(&zipBuf, gzip.BestCompression) + io.Copy(fileWriter, file) + fileWriter.Close() + content, _ := ioutil.ReadAll(&zipBuf) + assetMenFileAndContent(mf, content, t) +} +func TestOpenMemZipFile_3(t *testing.T) { + mf, err := openMemZipFile(licenseFile, "deflate") + if err != nil { + t.Fail() + } + file, _ := os.Open(licenseFile) + var zipBuf bytes.Buffer + fileWriter, _ := flate.NewWriter(&zipBuf, flate.BestCompression) + io.Copy(fileWriter, file) + fileWriter.Close() + content, _ := ioutil.ReadAll(&zipBuf) + assetMenFileAndContent(mf, content, t) +} From 3872c48349f3a26a4b7f0458f4cce779d85970f1 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 10:55:47 +0800 Subject: [PATCH 02/27] accept encoding refactor and bug fixed --- acceptencoder/acceptencoder.go | 118 ++++++++++++++++++++++++++++ acceptencoder/acceptencoder_test.go | 42 ++++++++++ context/output.go | 36 ++------- memzipfile.go | 42 +--------- staticfile.go | 11 ++- 5 files changed, 174 insertions(+), 75 deletions(-) create mode 100644 acceptencoder/acceptencoder.go create mode 100644 acceptencoder/acceptencoder_test.go diff --git a/acceptencoder/acceptencoder.go b/acceptencoder/acceptencoder.go new file mode 100644 index 00000000..70fd6183 --- /dev/null +++ b/acceptencoder/acceptencoder.go @@ -0,0 +1,118 @@ +// Copyright 2015 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package acceptencoder + +import ( + "compress/flate" + "compress/gzip" + "io" + "net/http" + "os" + "strconv" + "strings" + + "gopkg.in/bufio.v1" +) + +type q struct { + name string + value float64 +} + +// WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) + +func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { + return writeLevel(encoding, writer, file, flate.BestCompression) +} + +// WriteBody reads writes content to writer by the specific encoding(gzip/deflate) + +func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { + return writeLevel(encoding, writer, bufio.NewBuffer(content), flate.BestSpeed) +} + +// writeLevel reads from reader,writes to writer by specific encoding and compress level +// the compress level is defined by deflate package + +func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { + var outputWriter io.Writer + var err error + switch encoding { + case "gzip": + outputWriter, err = gzip.NewWriterLevel(writer, level) + case "deflate": + outputWriter, err = flate.NewWriter(writer, level) + default: + outputWriter = writer.(io.Writer) + } + if err != nil { + return false, "", err + } + if _, err = io.Copy(outputWriter, reader); err != nil { + return false, "", err + } + switch outputWriter.(type) { + case io.WriteCloser: + outputWriter.(io.WriteCloser).Close() + } + return encoding != "", encoding, nil +} + +// ParseEncoding will extract the right encoding for response +// the Accept-Encoding's sec is here: +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3 + +func ParseEncoding(r *http.Request) string { + if r == nil { + return "" + } + return parseEncoding(r) +} + + +func parseEncoding(r *http.Request) string { + acceptEncoding := r.Header.Get("Accept-Encoding") + if acceptEncoding == "" { + return "" + } + var lastQ q + for _, v := range strings.Split(acceptEncoding, ",") { + v = strings.TrimSpace(v) + if v == "" { + continue + } + vs := strings.Split(v, ";") + if len(vs) == 1 { + lastQ = q{vs[0], 1} + break + } + if len(vs) == 2 { + f, _ := strconv.ParseFloat(strings.Replace(vs[1], "q=", "", -1), 64) + if f == 0 { + continue + } + if f > lastQ.value { + lastQ = q{vs[0], f} + } + } + } + if lastQ.name == "*" { + return "gzip" + } + if lastQ.name == "identity" { + return "" + } + return lastQ.name +} diff --git a/acceptencoder/acceptencoder_test.go b/acceptencoder/acceptencoder_test.go new file mode 100644 index 00000000..7b73b587 --- /dev/null +++ b/acceptencoder/acceptencoder_test.go @@ -0,0 +1,42 @@ +// Copyright 2015 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package acceptencoder + +import "testing" + +func Test_ExtractEncoding(t *testing.T) { + if parseEncoding("gzip,deflate") != "gzip" { + t.Fail() + } + if parseEncoding("deflate,gzip") != "deflate" { + t.Fail() + } + + if parseEncoding("gzip;q=.5,deflate") != "deflate" { + t.Fail() + } + + if parseEncoding("gzip;q=0,deflate") != "deflate" { + t.Fail() + } + + if parseEncoding("deflate;q=0.5,gzip;q=0.5,identity") != "identity" { + t.Fail() + } + + if parseEncoding("*") != "gzip" { + t.Fail() + } +} diff --git a/context/output.go b/context/output.go index d132e392..a72c6f74 100644 --- a/context/output.go +++ b/context/output.go @@ -15,9 +15,8 @@ package context import ( + "beego/acceptencoder" "bytes" - "compress/flate" - "compress/gzip" "encoding/json" "encoding/xml" "errors" @@ -55,25 +54,12 @@ func (output *BeegoOutput) Header(key, val string) { // it sends out response body directly. func (output *BeegoOutput) Body(content []byte) { outputWriter := output.Context.ResponseWriter.(io.Writer) - if output.EnableGzip == true && output.Context.Input.Header("Accept-Encoding") != "" { - splitted := strings.SplitN(output.Context.Input.Header("Accept-Encoding"), ",", -1) - encodings := make([]string, len(splitted)) - - for i, val := range splitted { - encodings[i] = strings.TrimSpace(val) - } - for _, val := range encodings { - if val == "gzip" { - output.Header("Content-Encoding", "gzip") - outputWriter, _ = gzip.NewWriterLevel(output.Context.ResponseWriter, gzip.BestSpeed) - - break - } else if val == "deflate" { - output.Header("Content-Encoding", "deflate") - outputWriter, _ = flate.NewWriter(output.Context.ResponseWriter, flate.BestSpeed) - break - } - } + var encoding string + if output.EnableGzip { + encoding = acceptencoder.ParseEncoding(output.Context.Input.Request) + } + if b, n, _ := acceptencoder.WriteBody(encoding, outputWriter, content); b { + output.Header("Content-Encoding", n) } else { output.Header("Content-Length", strconv.Itoa(len(content))) } @@ -84,14 +70,6 @@ func (output *BeegoOutput) Body(content []byte) { output.Context.ResponseWriter.WriteHeader(output.Status) output.Status = 0 } - - outputWriter.Write(content) - switch outputWriter.(type) { - case *gzip.Writer: - outputWriter.(*gzip.Writer).Close() - case *flate.Writer: - outputWriter.(*flate.Writer).Close() - } } // Cookie sets cookie value via given key. diff --git a/memzipfile.go b/memzipfile.go index 376a1ae9..12b63eae 100644 --- a/memzipfile.go +++ b/memzipfile.go @@ -15,16 +15,12 @@ package beego import ( - "bufio" + "beego/acceptencoder" "bytes" - "compress/flate" - "compress/gzip" "errors" "io" "io/ioutil" - "net/http" "os" - "strings" "sync" "time" ) @@ -83,36 +79,17 @@ type memFileInfo struct { func newMenFileInfo(file *os.File, fileInfo os.FileInfo, zip string) (*memFileInfo, error) { var content []byte var zipBuf bytes.Buffer - var fileWriter io.Writer var err error - switch zip { - case "gzip": - fileWriter, err = gzip.NewWriterLevel(&zipBuf, gzip.BestCompression) - case "deflate": - fileWriter, err = flate.NewWriter(&zipBuf, flate.BestCompression) - default: - fileWriter = bufio.NewWriter(&zipBuf) - } + _, _, err = acceptencoder.WriteFile(zip, &zipBuf, file) if err != nil { return nil, err } - _, err = io.Copy(fileWriter, file) - if err != nil { - return nil, err - } - - switch fileWriter.(type) { - case io.WriteCloser: - fileWriter.(io.WriteCloser).Close() - } - content, err = ioutil.ReadAll(&zipBuf) if err != nil { return nil, err } - return &memFileInfo{ FileInfo: fileInfo, modTime: fileInfo.ModTime(), @@ -210,18 +187,3 @@ func (f *memFile) Seek(offset int64, whence int) (ret int64, err error) { f.offset = offset return f.offset, nil } - -// getAcceptEncodingZip returns accept encoding format in http header. -// zip is first, then deflate if both accepted. -// If no accepted, return empty string. -func getAcceptEncodingZip(r *http.Request) string { - ss := r.Header.Get("Accept-Encoding") - ss = strings.ToLower(ss) - if strings.Contains(ss, "gzip") { - return "gzip" - } else if strings.Contains(ss, "deflate") { - return "deflate" - } else { - return "" - } -} diff --git a/staticfile.go b/staticfile.go index de530b0c..20f7c519 100644 --- a/staticfile.go +++ b/staticfile.go @@ -15,6 +15,7 @@ package beego import ( + "beego/acceptencoder" "net/http" "os" "path" @@ -111,7 +112,7 @@ func serverStaticRouter(ctx *context.Context) { //to compress file var contentEncoding string if EnableGzip { - contentEncoding = getAcceptEncodingZip(ctx.Request) + contentEncoding = acceptencoder.ParseEncoding(ctx.Request) } memZipFile, err := openMemZipFile(filePath, contentEncoding) @@ -123,12 +124,10 @@ func serverStaticRouter(ctx *context.Context) { return } - if contentEncoding == "gzip" { - ctx.Output.Header("Content-Encoding", "gzip") - } else if contentEncoding == "deflate" { - ctx.Output.Header("Content-Encoding", "deflate") - } else { + if contentEncoding == "" { ctx.Output.Header("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)) + } else { + ctx.Output.Header("Content-Encoding", contentEncoding) } http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, fileInfo.ModTime(), memZipFile) From 0bc70e88f03c87072680af971ad876a5e9e1f835 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 11:00:29 +0800 Subject: [PATCH 03/27] ignore the other compress method --- acceptencoder/acceptencoder.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/acceptencoder/acceptencoder.go b/acceptencoder/acceptencoder.go index 70fd6183..65b5f2db 100644 --- a/acceptencoder/acceptencoder.go +++ b/acceptencoder/acceptencoder.go @@ -26,11 +26,6 @@ import ( "gopkg.in/bufio.v1" ) -type q struct { - name string - value float64 -} - // WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { @@ -55,6 +50,9 @@ func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) case "deflate": outputWriter, err = flate.NewWriter(writer, level) default: + // all the other compress methods will ignore + // such as the deprecated compress and chrome-only sdch + encoding = "" outputWriter = writer.(io.Writer) } if err != nil { @@ -81,6 +79,10 @@ func ParseEncoding(r *http.Request) string { return parseEncoding(r) } +type q struct { + name string + value float64 +} func parseEncoding(r *http.Request) string { acceptEncoding := r.Header.Get("Accept-Encoding") From 891be34fc657e11aa475a6cf8d856e76adb15038 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 11:20:13 +0800 Subject: [PATCH 04/27] encoding should be specify --- acceptencoder/acceptencoder.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/acceptencoder/acceptencoder.go b/acceptencoder/acceptencoder.go index 65b5f2db..10d4ca31 100644 --- a/acceptencoder/acceptencoder.go +++ b/acceptencoder/acceptencoder.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "gopkg.in/bufio.v1" + "bytes" ) // WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) @@ -35,7 +35,7 @@ func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, // WriteBody reads writes content to writer by the specific encoding(gzip/deflate) func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { - return writeLevel(encoding, writer, bufio.NewBuffer(content), flate.BestSpeed) + return writeLevel(encoding, writer, bytes.NewBuffer(content), flate.BestSpeed) } // writeLevel reads from reader,writes to writer by specific encoding and compress level @@ -84,6 +84,15 @@ type q struct { value float64 } +var ( + encodingMap = map[string]string{ // all the other compress methods will ignore + "gzip": "gzip", + "deflate": "deflate", + "*": "gzip", // * means any compress will accept,we prefer gzip + "identity": "", // identity means none-compress + } +) + func parseEncoding(r *http.Request) string { acceptEncoding := r.Header.Get("Accept-Encoding") if acceptEncoding == "" { @@ -110,11 +119,5 @@ func parseEncoding(r *http.Request) string { } } } - if lastQ.name == "*" { - return "gzip" - } - if lastQ.name == "identity" { - return "" - } - return lastQ.name + return encodingMap[lastQ.name] } From 8f42193610c6f772835f49efec58abdfe99a69ed Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 11:32:18 +0800 Subject: [PATCH 05/27] better compress func design --- acceptencoder/acceptencoder.go | 40 +++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/acceptencoder/acceptencoder.go b/acceptencoder/acceptencoder.go index 10d4ca31..a79ede94 100644 --- a/acceptencoder/acceptencoder.go +++ b/acceptencoder/acceptencoder.go @@ -44,16 +44,11 @@ func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { var outputWriter io.Writer var err error - switch encoding { - case "gzip": - outputWriter, err = gzip.NewWriterLevel(writer, level) - case "deflate": - outputWriter, err = flate.NewWriter(writer, level) - default: - // all the other compress methods will ignore - // such as the deprecated compress and chrome-only sdch + if cf, ok := encodingCompressMap[encoding]; ok { + outputWriter, err = cf(writer, level) + } else { encoding = "" - outputWriter = writer.(io.Writer) + outputWriter, err = noneCompress(writer, level) } if err != nil { return false, "", err @@ -83,13 +78,24 @@ type q struct { name string value float64 } +type compressFunc func(io.Writer, int) (io.Writer, error) + +func noneCompress(wr io.Writer, level int) (io.Writer, error) { + return wr, nil +} +func gzipCompress(wr io.Writer, level int) (io.Writer, error) { + return gzip.NewWriterLevel(wr, level) +} +func deflateCompress(wr io.Writer, level int) (io.Writer, error) { + return flate.NewWriter(wr, level) +} var ( - encodingMap = map[string]string{ // all the other compress methods will ignore - "gzip": "gzip", - "deflate": "deflate", - "*": "gzip", // * means any compress will accept,we prefer gzip - "identity": "", // identity means none-compress + encodingCompressMap = map[string]compressFunc{ // all the other compress methods will ignore + "gzip": gzipCompress, + "deflate": deflateCompress, + "*": gzipCompress, // * means any compress will accept,we prefer gzip + "identity": noneCompress, // identity means none-compress } ) @@ -119,5 +125,9 @@ func parseEncoding(r *http.Request) string { } } } - return encodingMap[lastQ.name] + if _, ok := encodingCompressMap[lastQ.name]; ok { + return lastQ.name + } else { + return "" + } } From 83ec39d02efb06d098b6f7becedb1be34a6fd3ce Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 11:47:10 +0800 Subject: [PATCH 06/27] refactor max age cookies setting --- context/output.go | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/context/output.go b/context/output.go index a72c6f74..6ffc5072 100644 --- a/context/output.go +++ b/context/output.go @@ -80,25 +80,21 @@ func (output *BeegoOutput) Cookie(name string, value string, others ...interface //fix cookie not work in IE if len(others) > 0 { + var maxAge int64 + switch v := others[0].(type) { case int: - if v > 0 { - fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(v)*time.Second).UTC().Format(time.RFC1123), v) - } else if v <= 0 { - fmt.Fprintf(&b, "; Max-Age=0") - } - case int64: - if v > 0 { - fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(v)*time.Second).UTC().Format(time.RFC1123), v) - } else if v <= 0 { - fmt.Fprintf(&b, "; Max-Age=0") - } + maxAge = int64(v) case int32: - if v > 0 { - fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(v)*time.Second).UTC().Format(time.RFC1123), v) - } else if v <= 0 { - fmt.Fprintf(&b, "; Max-Age=0") - } + maxAge = int64(v) + case int64: + maxAge = v + } + + if maxAge > 0 { + fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(maxAge)*time.Second).UTC().Format(time.RFC1123), maxAge) + } else if maxAge <= 0 { + fmt.Fprintf(&b, "; Max-Age=0") } } From 07c93cd32c4fa411526176523fd9a0a70a793603 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 11:59:32 +0800 Subject: [PATCH 07/27] mem zip file test ,add license --- memzipfile_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/memzipfile_test.go b/memzipfile_test.go index a8c67ead..82511634 100644 --- a/memzipfile_test.go +++ b/memzipfile_test.go @@ -1,3 +1,17 @@ +// Copyright 2015 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package beego import ( From 8603127c81d126b6aaf4f5f568626d41566efef9 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 13:10:42 +0800 Subject: [PATCH 08/27] beego package file path rewrite --- context/output.go | 3 ++- memzipfile.go | 3 ++- staticfile.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/context/output.go b/context/output.go index 6ffc5072..60912313 100644 --- a/context/output.go +++ b/context/output.go @@ -15,7 +15,6 @@ package context import ( - "beego/acceptencoder" "bytes" "encoding/json" "encoding/xml" @@ -29,6 +28,8 @@ import ( "strconv" "strings" "time" + + "github.com/astaxie/beego/acceptencoder" ) // BeegoOutput does work for sending response header. diff --git a/memzipfile.go b/memzipfile.go index 12b63eae..79e20a6c 100644 --- a/memzipfile.go +++ b/memzipfile.go @@ -15,7 +15,6 @@ package beego import ( - "beego/acceptencoder" "bytes" "errors" "io" @@ -23,6 +22,8 @@ import ( "os" "sync" "time" + + "github.com/astaxie/beego/acceptencoder" ) var ( diff --git a/staticfile.go b/staticfile.go index 20f7c519..ec6a74ab 100644 --- a/staticfile.go +++ b/staticfile.go @@ -15,7 +15,6 @@ package beego import ( - "beego/acceptencoder" "net/http" "os" "path" @@ -23,6 +22,7 @@ import ( "strconv" "strings" + "github.com/astaxie/beego/acceptencoder" "github.com/astaxie/beego/context" "github.com/astaxie/beego/utils" ) From 7964f7f163114217e6a4e0e7581f76ab725d20f3 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 13:16:16 +0800 Subject: [PATCH 09/27] test bug fixed --- acceptencoder/acceptencoder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptencoder/acceptencoder_test.go b/acceptencoder/acceptencoder_test.go index 7b73b587..39786626 100644 --- a/acceptencoder/acceptencoder_test.go +++ b/acceptencoder/acceptencoder_test.go @@ -32,7 +32,7 @@ func Test_ExtractEncoding(t *testing.T) { t.Fail() } - if parseEncoding("deflate;q=0.5,gzip;q=0.5,identity") != "identity" { + if parseEncoding("deflate;q=0.5,gzip;q=0.5,identity") != "" { t.Fail() } From 7ccf049a50433cf75a2065797fd3eaec3899eb61 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 13:27:33 +0800 Subject: [PATCH 10/27] bug fixed --- acceptencoder/acceptencoder_test.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/acceptencoder/acceptencoder_test.go b/acceptencoder/acceptencoder_test.go index 39786626..31cbc283 100644 --- a/acceptencoder/acceptencoder_test.go +++ b/acceptencoder/acceptencoder_test.go @@ -14,29 +14,32 @@ package acceptencoder -import "testing" +import ( + "net/http" + "testing" +) func Test_ExtractEncoding(t *testing.T) { - if parseEncoding("gzip,deflate") != "gzip" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip,deflate"}}}) != "gzip" { t.Fail() } - if parseEncoding("deflate,gzip") != "deflate" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate,gzip"}}}) != "deflate" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=.5,deflate"}}}) != "deflate" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=.5,deflate;q=0.3"}}}) != "gzip" { t.Fail() } - if parseEncoding("gzip;q=.5,deflate") != "deflate" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=0,deflate"}}}) != "deflate" { t.Fail() } - - if parseEncoding("gzip;q=0,deflate") != "deflate" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate;q=0.5,gzip;q=0.5,identity"}}}) != "identity" { t.Fail() } - - if parseEncoding("deflate;q=0.5,gzip;q=0.5,identity") != "" { - t.Fail() - } - - if parseEncoding("*") != "gzip" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"*"}}}) != "*" { t.Fail() } } From 39e29caf9b3deb52c30ed095d566746fe55ab4d6 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 13:42:10 +0800 Subject: [PATCH 11/27] refactor to fix encoder type bug --- acceptencoder/acceptencoder.go | 24 ++++++++++++++---------- acceptencoder/acceptencoder_test.go | 4 ++-- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/acceptencoder/acceptencoder.go b/acceptencoder/acceptencoder.go index a79ede94..974e7468 100644 --- a/acceptencoder/acceptencoder.go +++ b/acceptencoder/acceptencoder.go @@ -44,8 +44,8 @@ func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { var outputWriter io.Writer var err error - if cf, ok := encodingCompressMap[encoding]; ok { - outputWriter, err = cf(writer, level) + if cf, ok := encoderMap[encoding]; ok { + outputWriter, err = cf.encode(writer, level) } else { encoding = "" outputWriter, err = noneCompress(writer, level) @@ -78,7 +78,6 @@ type q struct { name string value float64 } -type compressFunc func(io.Writer, int) (io.Writer, error) func noneCompress(wr io.Writer, level int) (io.Writer, error) { return wr, nil @@ -90,12 +89,17 @@ func deflateCompress(wr io.Writer, level int) (io.Writer, error) { return flate.NewWriter(wr, level) } +type acceptEncoder struct { + name string + encode func(io.Writer, int) (io.Writer, error) +} + var ( - encodingCompressMap = map[string]compressFunc{ // all the other compress methods will ignore - "gzip": gzipCompress, - "deflate": deflateCompress, - "*": gzipCompress, // * means any compress will accept,we prefer gzip - "identity": noneCompress, // identity means none-compress + encoderMap = map[string]acceptEncoder{ // all the other compress methods will ignore + "gzip": acceptEncoder{"gzip", gzipCompress}, + "deflate": acceptEncoder{"deflate", deflateCompress}, + "*": acceptEncoder{"gzip", gzipCompress}, // * means any compress will accept,we prefer gzip + "identity": acceptEncoder{"", noneCompress}, // identity means none-compress } ) @@ -125,8 +129,8 @@ func parseEncoding(r *http.Request) string { } } } - if _, ok := encodingCompressMap[lastQ.name]; ok { - return lastQ.name + if cf, ok := encoderMap[lastQ.name]; ok { + return cf.name } else { return "" } diff --git a/acceptencoder/acceptencoder_test.go b/acceptencoder/acceptencoder_test.go index 31cbc283..6022e553 100644 --- a/acceptencoder/acceptencoder_test.go +++ b/acceptencoder/acceptencoder_test.go @@ -36,10 +36,10 @@ func Test_ExtractEncoding(t *testing.T) { if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=0,deflate"}}}) != "deflate" { t.Fail() } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate;q=0.5,gzip;q=0.5,identity"}}}) != "identity" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate;q=0.5,gzip;q=0.5,identity"}}}) != "" { t.Fail() } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"*"}}}) != "*" { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"*"}}}) != "gzip" { t.Fail() } } From 7ef9b3d55bda195b05fb9f545b05d1aaeded6452 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Tue, 10 Nov 2015 14:07:26 +0800 Subject: [PATCH 12/27] runnable typo fixed --- router.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/router.go b/router.go index 1e8144bf..7952663e 100644 --- a/router.go +++ b/router.go @@ -717,18 +717,18 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) if doFilter(BeforeExec) { goto Admin } - isRunable := false + isRunnable := false if routerInfo != nil { if routerInfo.routerType == routerTypeRESTFul { if _, ok := routerInfo.methods[r.Method]; ok { - isRunable = true + isRunnable = true routerInfo.runFunction(context) } else { exception("405", context) goto Admin } } else if routerInfo.routerType == routerTypeHandler { - isRunable = true + isRunnable = true routerInfo.handler.ServeHTTP(rw, r) } else { runrouter = routerInfo.controllerType @@ -750,7 +750,7 @@ func (p *ControllerRegister) ServeHTTP(rw http.ResponseWriter, r *http.Request) } // also defined runrouter & runMethod from filter - if !isRunable { + if !isRunnable { //Invoke the request handler vc := reflect.New(runrouter) execController, ok := vc.Interface().(ControllerInterface) From f457ea0fe939d821e9ecdc1cafa8916ddfd5d3cb Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 13:47:36 +0800 Subject: [PATCH 13/27] refactor encoder package --- context/acceptencoder.go | 137 ++++++++++++++++++++++++++++++++++ context/acceptencoder_test.go | 45 +++++++++++ 2 files changed, 182 insertions(+) create mode 100644 context/acceptencoder.go create mode 100644 context/acceptencoder_test.go diff --git a/context/acceptencoder.go b/context/acceptencoder.go new file mode 100644 index 00000000..9985bf36 --- /dev/null +++ b/context/acceptencoder.go @@ -0,0 +1,137 @@ +// Copyright 2015 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package context + +import ( + "compress/flate" + "compress/gzip" + "io" + "net/http" + "os" + "strconv" + "strings" + + "bytes" +) + +// WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) + +func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { + return writeLevel(encoding, writer, file, flate.BestCompression) +} + +// WriteBody reads writes content to writer by the specific encoding(gzip/deflate) + +func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { + return writeLevel(encoding, writer, bytes.NewBuffer(content), flate.BestSpeed) +} + +// writeLevel reads from reader,writes to writer by specific encoding and compress level +// the compress level is defined by deflate package + +func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { + var outputWriter io.Writer + var err error + if cf, ok := encoderMap[encoding]; ok { + outputWriter, err = cf.encode(writer, level) + } else { + encoding = "" + outputWriter, err = noneCompress(writer, level) + } + if err != nil { + return false, "", err + } + if _, err = io.Copy(outputWriter, reader); err != nil { + return false, "", err + } + switch outputWriter.(type) { + case io.WriteCloser: + outputWriter.(io.WriteCloser).Close() + } + return encoding != "", encoding, nil +} + +// ParseEncoding will extract the right encoding for response +// the Accept-Encoding's sec is here: +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3 + +func ParseEncoding(r *http.Request) string { + if r == nil { + return "" + } + return parseEncoding(r) +} + +type q struct { + name string + value float64 +} + +func noneCompress(wr io.Writer, level int) (io.Writer, error) { + return wr, nil +} +func gzipCompress(wr io.Writer, level int) (io.Writer, error) { + return gzip.NewWriterLevel(wr, level) +} +func deflateCompress(wr io.Writer, level int) (io.Writer, error) { + return flate.NewWriter(wr, level) +} + +type acceptEncoder struct { + name string + encode func(io.Writer, int) (io.Writer, error) +} + +var ( + encoderMap = map[string]acceptEncoder{ // all the other compress methods will ignore + "gzip": acceptEncoder{"gzip", gzipCompress}, + "deflate": acceptEncoder{"deflate", deflateCompress}, + "*": acceptEncoder{"gzip", gzipCompress}, // * means any compress will accept,we prefer gzip + "identity": acceptEncoder{"", noneCompress}, // identity means none-compress + } +) + +func parseEncoding(r *http.Request) string { + acceptEncoding := r.Header.Get("Accept-Encoding") + if acceptEncoding == "" { + return "" + } + var lastQ q + for _, v := range strings.Split(acceptEncoding, ",") { + v = strings.TrimSpace(v) + if v == "" { + continue + } + vs := strings.Split(v, ";") + if len(vs) == 1 { + lastQ = q{vs[0], 1} + break + } + if len(vs) == 2 { + f, _ := strconv.ParseFloat(strings.Replace(vs[1], "q=", "", -1), 64) + if f == 0 { + continue + } + if f > lastQ.value { + lastQ = q{vs[0], f} + } + } + } + if cf, ok := encoderMap[lastQ.name]; ok { + return cf.name + } else { + return "" + } +} diff --git a/context/acceptencoder_test.go b/context/acceptencoder_test.go new file mode 100644 index 00000000..147313c5 --- /dev/null +++ b/context/acceptencoder_test.go @@ -0,0 +1,45 @@ +// Copyright 2015 beego Author. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package context + +import ( + "net/http" + "testing" +) + +func Test_ExtractEncoding(t *testing.T) { + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip,deflate"}}}) != "gzip" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate,gzip"}}}) != "deflate" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=.5,deflate"}}}) != "deflate" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=.5,deflate;q=0.3"}}}) != "gzip" { + t.Fail() + } + + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=0,deflate"}}}) != "deflate" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate;q=0.5,gzip;q=0.5,identity"}}}) != "" { + t.Fail() + } + if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"*"}}}) != "gzip" { + t.Fail() + } +} From e4c6e5d2e1ee9d9de4f6d07a7470f0e727dddc10 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 13:47:47 +0800 Subject: [PATCH 14/27] change package --- context/output.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/context/output.go b/context/output.go index 60912313..454d132e 100644 --- a/context/output.go +++ b/context/output.go @@ -28,8 +28,6 @@ import ( "strconv" "strings" "time" - - "github.com/astaxie/beego/acceptencoder" ) // BeegoOutput does work for sending response header. @@ -57,9 +55,9 @@ func (output *BeegoOutput) Body(content []byte) { outputWriter := output.Context.ResponseWriter.(io.Writer) var encoding string if output.EnableGzip { - encoding = acceptencoder.ParseEncoding(output.Context.Input.Request) + encoding = ParseEncoding(output.Context.Input.Request) } - if b, n, _ := acceptencoder.WriteBody(encoding, outputWriter, content); b { + if b, n, _ := WriteBody(encoding, outputWriter, content); b { output.Header("Content-Encoding", n) } else { output.Header("Content-Length", strconv.Itoa(len(content))) From 15b0b9b66dacb4f72c4939c0c58c08c16a6c659e Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 16:21:04 +0800 Subject: [PATCH 15/27] delete old static file code --- memzipfile.go | 190 --------------------------------------------- memzipfile_test.go | 80 ------------------- 2 files changed, 270 deletions(-) delete mode 100644 memzipfile.go delete mode 100644 memzipfile_test.go diff --git a/memzipfile.go b/memzipfile.go deleted file mode 100644 index 79e20a6c..00000000 --- a/memzipfile.go +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright 2014 beego Author. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package beego - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "os" - "sync" - "time" - - "github.com/astaxie/beego/acceptencoder" -) - -var ( - menFileInfoMap = make(map[string]*memFileInfo) - lock sync.RWMutex -) - -// openMemZipFile returns MemFile object with a compressed static file. -// it's used for serve static file if gzip enable. -func openMemZipFile(path string, zip string) (*memFile, error) { - osFile, e := os.Open(path) - if e != nil { - return nil, e - } - defer osFile.Close() - - osFileInfo, e := osFile.Stat() - if e != nil { - return nil, e - } - - modTime := osFileInfo.ModTime() - fileSize := osFileInfo.Size() - mapKey := zip + ":" + path - lock.RLock() - cfi, ok := menFileInfoMap[mapKey] - lock.RUnlock() - if !(ok && cfi.ModTime() == modTime && cfi.fileSize == fileSize) { - lock.Lock() - defer lock.Unlock() - if cfi, ok = menFileInfoMap[mapKey]; !ok { - cfi, e = newMenFileInfo(osFile, osFileInfo, zip) - if e != nil { - return nil, e - } - menFileInfoMap[mapKey] = cfi - } - } - - return &memFile{fi: cfi, offset: 0}, nil -} - -// memFileInfo contains a compressed file bytes and file information. -// it implements os.FileInfo interface. -type memFileInfo struct { - os.FileInfo - modTime time.Time - content []byte - contentSize int64 - fileSize int64 -} - -// newMenFileInfo return a memFileInfo from file by zip type -func newMenFileInfo(file *os.File, fileInfo os.FileInfo, zip string) (*memFileInfo, error) { - var content []byte - var zipBuf bytes.Buffer - var err error - - _, _, err = acceptencoder.WriteFile(zip, &zipBuf, file) - if err != nil { - return nil, err - } - - content, err = ioutil.ReadAll(&zipBuf) - if err != nil { - return nil, err - } - return &memFileInfo{ - FileInfo: fileInfo, - modTime: fileInfo.ModTime(), - content: content, - contentSize: int64(len(content)), - fileSize: fileInfo.Size(), - }, nil -} - -// Name returns the compressed filename. -func (fi *memFileInfo) Name() string { - return fi.Name() -} - -// Size returns the raw file content size, not compressed size. -func (fi *memFileInfo) Size() int64 { - return fi.contentSize -} - -// Mode returns file mode. -func (fi *memFileInfo) Mode() os.FileMode { - return fi.Mode() -} - -// ModTime returns the last modified time of raw file. -func (fi *memFileInfo) ModTime() time.Time { - return fi.modTime -} - -// IsDir returns the compressing file is a directory or not. -func (fi *memFileInfo) IsDir() bool { - return fi.IsDir() -} - -// return nil. implement the os.FileInfo interface method. -func (fi *memFileInfo) Sys() interface{} { - return nil -} - -// memFile contains MemFileInfo and bytes offset when reading. -// it implements io.Reader,io.ReadCloser and io.Seeker. -type memFile struct { - fi *memFileInfo - offset int64 -} - -// Close memfile. -func (f *memFile) Close() error { - return nil -} - -// Get os.FileInfo of memfile. -func (f *memFile) Stat() (os.FileInfo, error) { - return f.fi, nil -} - -// read os.FileInfo of files in directory of memfile. -// it returns empty slice. -func (f *memFile) Readdir(count int) ([]os.FileInfo, error) { - infos := []os.FileInfo{} - - return infos, nil -} - -// Read bytes from the compressed file bytes. -func (f *memFile) Read(p []byte) (n int, err error) { - if len(f.fi.content)-int(f.offset) >= len(p) { - n = len(p) - } else { - n = len(f.fi.content) - int(f.offset) - err = io.EOF - } - copy(p, f.fi.content[f.offset:f.offset+int64(n)]) - f.offset += int64(n) - return -} - -var errWhence = errors.New("Seek: invalid whence") -var errOffset = errors.New("Seek: invalid offset") - -// Read bytes from the compressed file bytes by seeker. -func (f *memFile) Seek(offset int64, whence int) (ret int64, err error) { - switch whence { - default: - return 0, errWhence - case os.SEEK_SET: - case os.SEEK_CUR: - offset += f.offset - case os.SEEK_END: - offset += int64(len(f.fi.content)) - } - if offset < 0 || int(offset) > len(f.fi.content) { - return 0, errOffset - } - f.offset = offset - return f.offset, nil -} diff --git a/memzipfile_test.go b/memzipfile_test.go deleted file mode 100644 index 82511634..00000000 --- a/memzipfile_test.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2015 beego Author. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package beego - -import ( - "bytes" - "compress/flate" - "compress/gzip" - "io" - "io/ioutil" - "os" - "testing" -) - -const licenseFile = "./LICENSE" - -func TestOpenMemZipFile_1(t *testing.T) { - mf, err := openMemZipFile(licenseFile, "") - if err != nil { - t.Fail() - } - file, _ := os.Open(licenseFile) - content, _ := ioutil.ReadAll(file) - assetMenFileAndContent(mf, content, t) -} - -func assetMenFileAndContent(mf *memFile, content []byte, t *testing.T) { - if mf.fi.contentSize != int64(len(content)) { - t.Log("content size not same") - t.Fail() - } - for i, v := range content { - if v != mf.fi.content[i] { - t.Log("content not same") - t.Fail() - } - } - if len(menFileInfoMap) == 0 { - t.Log("men map is empty") - t.Fail() - } -} -func TestOpenMemZipFile_2(t *testing.T) { - mf, err := openMemZipFile(licenseFile, "gzip") - if err != nil { - t.Fail() - } - file, _ := os.Open(licenseFile) - var zipBuf bytes.Buffer - fileWriter, _ := gzip.NewWriterLevel(&zipBuf, gzip.BestCompression) - io.Copy(fileWriter, file) - fileWriter.Close() - content, _ := ioutil.ReadAll(&zipBuf) - assetMenFileAndContent(mf, content, t) -} -func TestOpenMemZipFile_3(t *testing.T) { - mf, err := openMemZipFile(licenseFile, "deflate") - if err != nil { - t.Fail() - } - file, _ := os.Open(licenseFile) - var zipBuf bytes.Buffer - fileWriter, _ := flate.NewWriter(&zipBuf, flate.BestCompression) - io.Copy(fileWriter, file) - fileWriter.Close() - content, _ := ioutil.ReadAll(&zipBuf) - assetMenFileAndContent(mf, content, t) -} From 82c50b972d684e30b4f99e4f6a046cca30a766ec Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 16:22:05 +0800 Subject: [PATCH 16/27] new test file --- staticfile_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 staticfile_test.go diff --git a/staticfile_test.go b/staticfile_test.go new file mode 100644 index 00000000..e635fcc6 --- /dev/null +++ b/staticfile_test.go @@ -0,0 +1,71 @@ +package beego + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "io" + "io/ioutil" + "os" + "testing" +) + +const licenseFile = "./LICENSE" + +func testOpenFile(encoding string, content []byte, t *testing.T) { + fi, _ := os.Stat(licenseFile) + b, n, sch, err := openFile(licenseFile, fi, encoding) + if err != nil { + t.Log(err) + t.Fail() + } + + t.Log("open static file encoding "+n, b) + + assetOpenFileAndContent(sch, content, t) +} +func TestOpenStaticFile_1(t *testing.T) { + file, _ := os.Open(licenseFile) + content, _ := ioutil.ReadAll(file) + testOpenFile("", content, t) +} + +func TestOpenStaticFileGzip_1(t *testing.T) { + file, _ := os.Open(licenseFile) + var zipBuf bytes.Buffer + fileWriter, _ := gzip.NewWriterLevel(&zipBuf, gzip.BestCompression) + io.Copy(fileWriter, file) + fileWriter.Close() + content, _ := ioutil.ReadAll(&zipBuf) + + testOpenFile("gzip", content, t) +} +func TestOpenStaticFileDeflate_1(t *testing.T) { + file, _ := os.Open(licenseFile) + var zipBuf bytes.Buffer + fileWriter, _ := flate.NewWriter(&zipBuf, flate.BestCompression) + io.Copy(fileWriter, file) + fileWriter.Close() + content, _ := ioutil.ReadAll(&zipBuf) + + testOpenFile("deflate", content, t) +} + +func assetOpenFileAndContent(sch *serveContentHolder, content []byte, t *testing.T) { + t.Log(sch.size, len(content)) + if sch.size != int64(len(content)) { + t.Log("static content file size not same") + t.Fail() + } + bs, _ := ioutil.ReadAll(sch) + for i, v := range content { + if v != bs[i] { + t.Log("content not same") + t.Fail() + } + } + if len(staticFileMap) == 0 { + t.Log("men map is empty") + t.Fail() + } +} From d2c60619fa857c99a3ab9416b08719b321e583e7 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 16:22:40 +0800 Subject: [PATCH 17/27] new static file support code --- staticfile.go | 228 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 139 insertions(+), 89 deletions(-) diff --git a/staticfile.go b/staticfile.go index ec6a74ab..cbe2b26a 100644 --- a/staticfile.go +++ b/staticfile.go @@ -15,123 +15,173 @@ package beego import ( + "bytes" "net/http" "os" "path" "path/filepath" "strconv" "strings" + "sync" + + "errors" + + "time" - "github.com/astaxie/beego/acceptencoder" "github.com/astaxie/beego/context" - "github.com/astaxie/beego/utils" ) +var notStaticRequestErr = errors.New("request not a static file request") + func serverStaticRouter(ctx *context.Context) { if ctx.Input.Method() != "GET" && ctx.Input.Method() != "HEAD" { return } - requestPath := filepath.Clean(ctx.Input.Request.URL.Path) + forbidden, filePath, fileInfo, err := lookupFile(ctx) + if err == notStaticRequestErr { + return + } + + if forbidden { + exception("403", ctx) + return + } + + if filePath == "" || fileInfo == nil { + if RunMode == "dev" { + Warn("Can't find/open the file:", filePath, err) + } + http.NotFound(ctx.ResponseWriter, ctx.Request) + return + } + if fileInfo.IsDir() { + //serveFile will list dir + http.ServeFile(ctx.ResponseWriter, ctx.Request, filePath) + return + } + + var enableCompress = EnableGzip && isStaticCompress(filePath) + var acceptEncoding string + if enableCompress { + acceptEncoding = context.ParseEncoding(ctx.Request) + } + b, n, sch, err := openFile(filePath, fileInfo, acceptEncoding) + if err != nil { + if RunMode == "dev" { + Warn("Can't compress the file:", filePath, err) + } + http.NotFound(ctx.ResponseWriter, ctx.Request) + return + } + + if b { + ctx.Output.Header("Content-Encoding", n) + } else { + ctx.Output.Header("Content-Length", strconv.FormatInt(sch.size, 10)) + } + + http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, sch.modTime, sch) + return + +} + +type serveContentHolder struct { + *strings.Reader + modTime time.Time + size int64 + encoding string +} + +var ( + staticFileMap = make(map[string]*serveContentHolder) + mapLock sync.Mutex +) + +func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, error) { + mapKey := acceptEncoding + ":" + filePath + mapFile, ok := staticFileMap[mapKey] + if ok { + if mapFile.modTime == fi.ModTime() && mapFile.size == fi.Size() { + return mapFile.encoding != "", mapFile.encoding, mapFile, nil + } + } + mapLock.Lock() + delete(staticFileMap, mapKey) + defer mapLock.Unlock() + if mapFile, ok = staticFileMap[mapKey]; !ok { + file, err := os.Open(filePath) + if err != nil { + return false, "", nil, err + } + defer file.Close() + var bufferWriter bytes.Buffer + _, n, err := context.WriteFile(acceptEncoding, &bufferWriter, file) + if err != nil { + return false, "", nil, err + } + mapFile = &serveContentHolder{Reader: strings.NewReader(string(bufferWriter.Bytes())), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n} + staticFileMap[mapKey] = mapFile + } + + return mapFile.encoding != "", mapFile.encoding, mapFile, nil +} + +func isStaticCompress(filePath string) bool { + for _, statExtension := range StaticExtensionsToGzip { + if strings.HasSuffix(strings.ToLower(filePath), strings.ToLower(statExtension)) { + return true + } + } + return false +} + +func searchFile(ctx *context.Context) (string, os.FileInfo, error) { + requestPath := filepath.Clean(ctx.Input.Request.URL.Path) // special processing : favicon.ico/robots.txt can be in any static dir if requestPath == "/favicon.ico" || requestPath == "/robots.txt" { file := path.Join(".", requestPath) - if utils.FileExists(file) { - http.ServeFile(ctx.ResponseWriter, ctx.Request, file) - return + if fi, _ := os.Stat(file); fi != nil { + return file, fi, nil } - for _, staticDir := range StaticDir { - file := path.Join(staticDir, requestPath) - if utils.FileExists(file) { - http.ServeFile(ctx.ResponseWriter, ctx.Request, file) - return + filePath := path.Join(staticDir, requestPath) + if fi, _ := os.Stat(filePath); fi != nil { + return filePath, fi, nil } } - - http.NotFound(ctx.ResponseWriter, ctx.Request) - return + return "", nil, errors.New(requestPath + " file not find") } for prefix, staticDir := range StaticDir { if len(prefix) == 0 { continue } - if strings.HasPrefix(requestPath, prefix) { - if len(requestPath) > len(prefix) && requestPath[len(prefix)] != '/' { - continue - } - filePath := path.Join(staticDir, requestPath[len(prefix):]) - fileInfo, err := os.Stat(filePath) - if err != nil { - if RunMode == "dev" { - Warn("Can't find the file:", filePath, err) - } - http.NotFound(ctx.ResponseWriter, ctx.Request) - return - } - //if the request is dir and DirectoryIndex is false then - if fileInfo.IsDir() { - if !DirectoryIndex { - exception("403", ctx) - return - } - if ctx.Input.Request.URL.Path[len(ctx.Input.Request.URL.Path)-1] != '/' { - http.Redirect(ctx.ResponseWriter, ctx.Request, ctx.Input.Request.URL.Path+"/", 302) - return - } - } - - if strings.HasSuffix(requestPath, "/index.html") { - fileReader, err := os.Open(filePath) - if err != nil { - if RunMode == "dev" { - Warn("Can't open the file:", filePath, err) - } - http.NotFound(ctx.ResponseWriter, ctx.Request) - return - } - defer fileReader.Close() - http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, fileInfo.ModTime(), fileReader) - return - } - - isStaticFileToCompress := false - for _, statExtension := range StaticExtensionsToGzip { - if strings.HasSuffix(strings.ToLower(filePath), strings.ToLower(statExtension)) { - isStaticFileToCompress = true - break - } - } - - if !isStaticFileToCompress { - http.ServeFile(ctx.ResponseWriter, ctx.Request, filePath) - return - } - - //to compress file - var contentEncoding string - if EnableGzip { - contentEncoding = acceptencoder.ParseEncoding(ctx.Request) - } - - memZipFile, err := openMemZipFile(filePath, contentEncoding) - if err != nil { - if RunMode == "dev" { - Warn("Can't compress the file:", filePath, err) - } - http.NotFound(ctx.ResponseWriter, ctx.Request) - return - } - - if contentEncoding == "" { - ctx.Output.Header("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)) - } else { - ctx.Output.Header("Content-Encoding", contentEncoding) - } - - http.ServeContent(ctx.ResponseWriter, ctx.Request, filePath, fileInfo.ModTime(), memZipFile) - return + if !strings.Contains(requestPath, prefix) { + continue + } + if len(requestPath) > len(prefix) && requestPath[len(prefix)] != '/' { + continue + } + filePath := path.Join(staticDir, requestPath[len(prefix):]) + if fi, err := os.Stat(filePath); fi != nil { + return filePath, fi, err } } + return "", nil, notStaticRequestErr +} + +func lookupFile(ctx *context.Context) (bool, string, os.FileInfo, error) { + fp, fi, err := searchFile(ctx) + if fp == "" || fi == nil { + return false, "", nil, err + } + if !fi.IsDir() { + return false, fp, fi, err + } + ifp := filepath.Join(fp, "index.html") + if ifi, _ := os.Stat(ifp); ifi != nil && ifi.Mode().IsRegular() { + return false, ifp, ifi, err + } + return !DirectoryIndex, fp, fi, err } From b25a1355f9e49b771f7aee321ed9365c1c52cdfa Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 16:27:41 +0800 Subject: [PATCH 18/27] remove old code --- acceptencoder/acceptencoder.go | 137 ---------------------------- acceptencoder/acceptencoder_test.go | 45 --------- 2 files changed, 182 deletions(-) delete mode 100644 acceptencoder/acceptencoder.go delete mode 100644 acceptencoder/acceptencoder_test.go diff --git a/acceptencoder/acceptencoder.go b/acceptencoder/acceptencoder.go deleted file mode 100644 index 974e7468..00000000 --- a/acceptencoder/acceptencoder.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2015 beego Author. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package acceptencoder - -import ( - "compress/flate" - "compress/gzip" - "io" - "net/http" - "os" - "strconv" - "strings" - - "bytes" -) - -// WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) - -func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { - return writeLevel(encoding, writer, file, flate.BestCompression) -} - -// WriteBody reads writes content to writer by the specific encoding(gzip/deflate) - -func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { - return writeLevel(encoding, writer, bytes.NewBuffer(content), flate.BestSpeed) -} - -// writeLevel reads from reader,writes to writer by specific encoding and compress level -// the compress level is defined by deflate package - -func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { - var outputWriter io.Writer - var err error - if cf, ok := encoderMap[encoding]; ok { - outputWriter, err = cf.encode(writer, level) - } else { - encoding = "" - outputWriter, err = noneCompress(writer, level) - } - if err != nil { - return false, "", err - } - if _, err = io.Copy(outputWriter, reader); err != nil { - return false, "", err - } - switch outputWriter.(type) { - case io.WriteCloser: - outputWriter.(io.WriteCloser).Close() - } - return encoding != "", encoding, nil -} - -// ParseEncoding will extract the right encoding for response -// the Accept-Encoding's sec is here: -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3 - -func ParseEncoding(r *http.Request) string { - if r == nil { - return "" - } - return parseEncoding(r) -} - -type q struct { - name string - value float64 -} - -func noneCompress(wr io.Writer, level int) (io.Writer, error) { - return wr, nil -} -func gzipCompress(wr io.Writer, level int) (io.Writer, error) { - return gzip.NewWriterLevel(wr, level) -} -func deflateCompress(wr io.Writer, level int) (io.Writer, error) { - return flate.NewWriter(wr, level) -} - -type acceptEncoder struct { - name string - encode func(io.Writer, int) (io.Writer, error) -} - -var ( - encoderMap = map[string]acceptEncoder{ // all the other compress methods will ignore - "gzip": acceptEncoder{"gzip", gzipCompress}, - "deflate": acceptEncoder{"deflate", deflateCompress}, - "*": acceptEncoder{"gzip", gzipCompress}, // * means any compress will accept,we prefer gzip - "identity": acceptEncoder{"", noneCompress}, // identity means none-compress - } -) - -func parseEncoding(r *http.Request) string { - acceptEncoding := r.Header.Get("Accept-Encoding") - if acceptEncoding == "" { - return "" - } - var lastQ q - for _, v := range strings.Split(acceptEncoding, ",") { - v = strings.TrimSpace(v) - if v == "" { - continue - } - vs := strings.Split(v, ";") - if len(vs) == 1 { - lastQ = q{vs[0], 1} - break - } - if len(vs) == 2 { - f, _ := strconv.ParseFloat(strings.Replace(vs[1], "q=", "", -1), 64) - if f == 0 { - continue - } - if f > lastQ.value { - lastQ = q{vs[0], f} - } - } - } - if cf, ok := encoderMap[lastQ.name]; ok { - return cf.name - } else { - return "" - } -} diff --git a/acceptencoder/acceptencoder_test.go b/acceptencoder/acceptencoder_test.go deleted file mode 100644 index 6022e553..00000000 --- a/acceptencoder/acceptencoder_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2015 beego Author. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package acceptencoder - -import ( - "net/http" - "testing" -) - -func Test_ExtractEncoding(t *testing.T) { - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip,deflate"}}}) != "gzip" { - t.Fail() - } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate,gzip"}}}) != "deflate" { - t.Fail() - } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=.5,deflate"}}}) != "deflate" { - t.Fail() - } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=.5,deflate;q=0.3"}}}) != "gzip" { - t.Fail() - } - - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"gzip;q=0,deflate"}}}) != "deflate" { - t.Fail() - } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"deflate;q=0.5,gzip;q=0.5,identity"}}}) != "" { - t.Fail() - } - if parseEncoding(&http.Request{Header: map[string][]string{"Accept-Encoding": []string{"*"}}}) != "gzip" { - t.Fail() - } -} From 1200b7c34785cd3167049751a4236623dd4d5c90 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Wed, 11 Nov 2015 18:06:18 +0800 Subject: [PATCH 19/27] method refactor --- context/acceptencoder.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 9985bf36..2d568fc0 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -23,28 +23,34 @@ import ( "strconv" "strings" - "bytes" + "fmt" + "io/ioutil" ) // WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { - return writeLevel(encoding, writer, file, flate.BestCompression) + content, err := ioutil.ReadAll(file) + if err != nil { + return false, "", err + } + return writeLevel(encoding, writer, content, flate.BestCompression) } // WriteBody reads writes content to writer by the specific encoding(gzip/deflate) func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { - return writeLevel(encoding, writer, bytes.NewBuffer(content), flate.BestSpeed) + return writeLevel(encoding, writer, content, flate.BestSpeed) } // writeLevel reads from reader,writes to writer by specific encoding and compress level // the compress level is defined by deflate package -func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { +func writeLevel(encoding string, writer io.Writer, content []byte, level int) (bool, string, error) { var outputWriter io.Writer var err error if cf, ok := encoderMap[encoding]; ok { + fmt.Println("write level", encoding) outputWriter, err = cf.encode(writer, level) } else { encoding = "" @@ -53,9 +59,7 @@ func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) if err != nil { return false, "", err } - if _, err = io.Copy(outputWriter, reader); err != nil { - return false, "", err - } + outputWriter.Write(content) switch outputWriter.(type) { case io.WriteCloser: outputWriter.(io.WriteCloser).Close() From a9881388f7e42d893c386b59c94c66fb6c4d1e74 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 12 Nov 2015 10:08:57 +0800 Subject: [PATCH 20/27] accept encoder header setting fixed --- context/acceptencoder.go | 2 -- context/output.go | 7 ++++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 2d568fc0..c2a79fa1 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -23,7 +23,6 @@ import ( "strconv" "strings" - "fmt" "io/ioutil" ) @@ -50,7 +49,6 @@ func writeLevel(encoding string, writer io.Writer, content []byte, level int) (b var outputWriter io.Writer var err error if cf, ok := encoderMap[encoding]; ok { - fmt.Println("write level", encoding) outputWriter, err = cf.encode(writer, level) } else { encoding = "" diff --git a/context/output.go b/context/output.go index 454d132e..4d902487 100644 --- a/context/output.go +++ b/context/output.go @@ -52,23 +52,24 @@ func (output *BeegoOutput) Header(key, val string) { // if EnableGzip, compress content string. // it sends out response body directly. func (output *BeegoOutput) Body(content []byte) { - outputWriter := output.Context.ResponseWriter.(io.Writer) var encoding string + var buf = &bytes.Buffer{} if output.EnableGzip { encoding = ParseEncoding(output.Context.Input.Request) } - if b, n, _ := WriteBody(encoding, outputWriter, content); b { + if b, n, _ := WriteBody(encoding, buf, content); b { output.Header("Content-Encoding", n) } else { output.Header("Content-Length", strconv.Itoa(len(content))) } - // Write status code if it has been set manually // Set it to 0 afterwards to prevent "multiple response.WriteHeader calls" if output.Status != 0 { output.Context.ResponseWriter.WriteHeader(output.Status) output.Status = 0 } + + io.Copy(output.Context.ResponseWriter, buf) } // Cookie sets cookie value via given key. From f8db8ae9c35ba27a8ad20e3f08fd80e42dd0fece Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 12 Nov 2015 10:48:36 +0800 Subject: [PATCH 21/27] add some comments --- staticfile.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/staticfile.go b/staticfile.go index cbe2b26a..894d5c8d 100644 --- a/staticfile.go +++ b/staticfile.go @@ -127,6 +127,8 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str return mapFile.encoding != "", mapFile.encoding, mapFile, nil } +// isStaticCompress detect static files + func isStaticCompress(filePath string) bool { for _, statExtension := range StaticExtensionsToGzip { if strings.HasSuffix(strings.ToLower(filePath), strings.ToLower(statExtension)) { @@ -136,6 +138,8 @@ func isStaticCompress(filePath string) bool { return false } +// searchFile search the file by url path +// if none the static file prefix matches ,return notStaticRequestErr func searchFile(ctx *context.Context) (string, os.FileInfo, error) { requestPath := filepath.Clean(ctx.Input.Request.URL.Path) // special processing : favicon.ico/robots.txt can be in any static dir @@ -171,6 +175,10 @@ func searchFile(ctx *context.Context) (string, os.FileInfo, error) { return "", nil, notStaticRequestErr } +// lookupFile find the file to serve +// if the file is dir ,search the index.html as default file( MUST NOT A DIR also) +// if the index.html not exist or is a dir, give a forbidden response depending on DirectoryIndex + func lookupFile(ctx *context.Context) (bool, string, os.FileInfo, error) { fp, fi, err := searchFile(ctx) if fp == "" || fi == nil { From 214030fad4b4601fdaf0c17e0f4c548f20e40fbf Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 12 Nov 2015 11:44:29 +0800 Subject: [PATCH 22/27] bytes reader replace string reader --- context/acceptencoder.go | 15 +++++---------- staticfile.go | 4 ++-- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index c2a79fa1..c252cf6b 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -15,6 +15,7 @@ package context import ( + "bytes" "compress/flate" "compress/gzip" "io" @@ -22,30 +23,24 @@ import ( "os" "strconv" "strings" - - "io/ioutil" ) // WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { - content, err := ioutil.ReadAll(file) - if err != nil { - return false, "", err - } - return writeLevel(encoding, writer, content, flate.BestCompression) + return writeLevel(encoding, writer, file, flate.BestCompression) } // WriteBody reads writes content to writer by the specific encoding(gzip/deflate) func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { - return writeLevel(encoding, writer, content, flate.BestSpeed) + return writeLevel(encoding, writer, bytes.NewReader(content), flate.BestSpeed) } // writeLevel reads from reader,writes to writer by specific encoding and compress level // the compress level is defined by deflate package -func writeLevel(encoding string, writer io.Writer, content []byte, level int) (bool, string, error) { +func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { var outputWriter io.Writer var err error if cf, ok := encoderMap[encoding]; ok { @@ -57,7 +52,7 @@ func writeLevel(encoding string, writer io.Writer, content []byte, level int) (b if err != nil { return false, "", err } - outputWriter.Write(content) + io.Copy(outputWriter, reader) switch outputWriter.(type) { case io.WriteCloser: outputWriter.(io.WriteCloser).Close() diff --git a/staticfile.go b/staticfile.go index 894d5c8d..5dae81ac 100644 --- a/staticfile.go +++ b/staticfile.go @@ -87,7 +87,7 @@ func serverStaticRouter(ctx *context.Context) { } type serveContentHolder struct { - *strings.Reader + *bytes.Reader modTime time.Time size int64 encoding string @@ -120,7 +120,7 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str if err != nil { return false, "", nil, err } - mapFile = &serveContentHolder{Reader: strings.NewReader(string(bufferWriter.Bytes())), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n} + mapFile = &serveContentHolder{Reader: bytes.NewReader(bufferWriter.Bytes()), modTime: fi.ModTime(), size: int64(bufferWriter.Len()), encoding: n} staticFileMap[mapKey] = mapFile } From 46fbeaadad21ae1361ff8b71ccc9e7facf2fae9e Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 12 Nov 2015 12:03:53 +0800 Subject: [PATCH 23/27] refactor accept encoder ,simplify the struct --- context/acceptencoder.go | 61 +++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index c252cf6b..887459e1 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -25,6 +25,26 @@ import ( "strings" ) +type acceptEncoder struct { + name string + encode func(io.Writer, int) (io.Writer, error) +} + +var ( + noneCompressEncoder = acceptEncoder{"", func(wr io.Writer, level int) (io.Writer, error) { return wr, nil }} + gzipCompressEncoder = acceptEncoder{"gzip", func(wr io.Writer, level int) (io.Writer, error) { return gzip.NewWriterLevel(wr, level) }} + deflateCompressEncoder = acceptEncoder{"deflate", func(wr io.Writer, level int) (io.Writer, error) { return flate.NewWriter(wr, level) }} +) + +var ( + encoderMap = map[string]acceptEncoder{ // all the other compress methods will ignore + "gzip": gzipCompressEncoder, + "deflate": deflateCompressEncoder, + "*": gzipCompressEncoder, // * means any compress will accept,we prefer gzip + "identity": noneCompressEncoder, // identity means none-compress + } +) + // WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { @@ -43,16 +63,23 @@ func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { var outputWriter io.Writer var err error + var ce = noneCompressEncoder + if cf, ok := encoderMap[encoding]; ok { - outputWriter, err = cf.encode(writer, level) - } else { - encoding = "" - outputWriter, err = noneCompress(writer, level) + ce = cf } + encoding = ce.name + outputWriter, err = ce.encode(writer, level) + if err != nil { return false, "", err } - io.Copy(outputWriter, reader) + + _, err = io.Copy(outputWriter, reader) + if err != nil { + return false, "", err + } + switch outputWriter.(type) { case io.WriteCloser: outputWriter.(io.WriteCloser).Close() @@ -76,30 +103,6 @@ type q struct { value float64 } -func noneCompress(wr io.Writer, level int) (io.Writer, error) { - return wr, nil -} -func gzipCompress(wr io.Writer, level int) (io.Writer, error) { - return gzip.NewWriterLevel(wr, level) -} -func deflateCompress(wr io.Writer, level int) (io.Writer, error) { - return flate.NewWriter(wr, level) -} - -type acceptEncoder struct { - name string - encode func(io.Writer, int) (io.Writer, error) -} - -var ( - encoderMap = map[string]acceptEncoder{ // all the other compress methods will ignore - "gzip": acceptEncoder{"gzip", gzipCompress}, - "deflate": acceptEncoder{"deflate", deflateCompress}, - "*": acceptEncoder{"gzip", gzipCompress}, // * means any compress will accept,we prefer gzip - "identity": acceptEncoder{"", noneCompress}, // identity means none-compress - } -) - func parseEncoding(r *http.Request) string { acceptEncoding := r.Header.Get("Accept-Encoding") if acceptEncoding == "" { From bc2195b07f0a1d8d8c6bfabd0308a3de4abd8dca Mon Sep 17 00:00:00 2001 From: JessonChan Date: Thu, 12 Nov 2015 16:59:07 +0800 Subject: [PATCH 24/27] code simplify --- context/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/context/output.go b/context/output.go index 4d902487..f0d66f36 100644 --- a/context/output.go +++ b/context/output.go @@ -93,7 +93,7 @@ func (output *BeegoOutput) Cookie(name string, value string, others ...interface if maxAge > 0 { fmt.Fprintf(&b, "; Expires=%s; Max-Age=%d", time.Now().Add(time.Duration(maxAge)*time.Second).UTC().Format(time.RFC1123), maxAge) - } else if maxAge <= 0 { + } else { fmt.Fprintf(&b, "; Max-Age=0") } } From d963bb79bd20c74495132d89b9dd84c051377c87 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Mon, 16 Nov 2015 10:31:27 +0800 Subject: [PATCH 25/27] avoid map-lock delete --- staticfile.go | 1 - 1 file changed, 1 deletion(-) diff --git a/staticfile.go b/staticfile.go index 5dae81ac..80162e74 100644 --- a/staticfile.go +++ b/staticfile.go @@ -107,7 +107,6 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str } } mapLock.Lock() - delete(staticFileMap, mapKey) defer mapLock.Unlock() if mapFile, ok = staticFileMap[mapKey]; !ok { file, err := os.Open(filePath) From 5d01afe3a62e965e806a03f5289504148f11b3fc Mon Sep 17 00:00:00 2001 From: JessonChan Date: Mon, 16 Nov 2015 14:05:05 +0800 Subject: [PATCH 26/27] isOk to check whether the file is latest --- staticfile.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/staticfile.go b/staticfile.go index 80162e74..cb57b133 100644 --- a/staticfile.go +++ b/staticfile.go @@ -100,15 +100,13 @@ var ( func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, string, *serveContentHolder, error) { mapKey := acceptEncoding + ":" + filePath - mapFile, ok := staticFileMap[mapKey] - if ok { - if mapFile.modTime == fi.ModTime() && mapFile.size == fi.Size() { - return mapFile.encoding != "", mapFile.encoding, mapFile, nil - } + mapFile, _ := staticFileMap[mapKey] + if isOk(mapFile, fi) { + return mapFile.encoding != "", mapFile.encoding, mapFile, nil } mapLock.Lock() defer mapLock.Unlock() - if mapFile, ok = staticFileMap[mapKey]; !ok { + if mapFile, _ = staticFileMap[mapKey]; !isOk(mapFile, fi) { file, err := os.Open(filePath) if err != nil { return false, "", nil, err @@ -126,6 +124,13 @@ func openFile(filePath string, fi os.FileInfo, acceptEncoding string) (bool, str return mapFile.encoding != "", mapFile.encoding, mapFile, nil } +func isOk(s *serveContentHolder, fi os.FileInfo) bool { + if s == nil { + return false + } + return s.modTime == fi.ModTime() && s.size == fi.Size() +} + // isStaticCompress detect static files func isStaticCompress(filePath string) bool { From 9170b91075fe0019df7861fd77b8332d0ba0c988 Mon Sep 17 00:00:00 2001 From: JessonChan Date: Sat, 21 Nov 2015 08:46:19 +0800 Subject: [PATCH 27/27] go style format (remove the blank after comments) --- context/acceptencoder.go | 4 ---- staticfile.go | 2 -- 2 files changed, 6 deletions(-) diff --git a/context/acceptencoder.go b/context/acceptencoder.go index 887459e1..1bd2cc3d 100644 --- a/context/acceptencoder.go +++ b/context/acceptencoder.go @@ -46,20 +46,17 @@ var ( ) // WriteFile reads from file and writes to writer by the specific encoding(gzip/deflate) - func WriteFile(encoding string, writer io.Writer, file *os.File) (bool, string, error) { return writeLevel(encoding, writer, file, flate.BestCompression) } // WriteBody reads writes content to writer by the specific encoding(gzip/deflate) - func WriteBody(encoding string, writer io.Writer, content []byte) (bool, string, error) { return writeLevel(encoding, writer, bytes.NewReader(content), flate.BestSpeed) } // writeLevel reads from reader,writes to writer by specific encoding and compress level // the compress level is defined by deflate package - func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) (bool, string, error) { var outputWriter io.Writer var err error @@ -90,7 +87,6 @@ func writeLevel(encoding string, writer io.Writer, reader io.Reader, level int) // ParseEncoding will extract the right encoding for response // the Accept-Encoding's sec is here: // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3 - func ParseEncoding(r *http.Request) string { if r == nil { return "" diff --git a/staticfile.go b/staticfile.go index cb57b133..e1d88b87 100644 --- a/staticfile.go +++ b/staticfile.go @@ -132,7 +132,6 @@ func isOk(s *serveContentHolder, fi os.FileInfo) bool { } // isStaticCompress detect static files - func isStaticCompress(filePath string) bool { for _, statExtension := range StaticExtensionsToGzip { if strings.HasSuffix(strings.ToLower(filePath), strings.ToLower(statExtension)) { @@ -182,7 +181,6 @@ func searchFile(ctx *context.Context) (string, os.FileInfo, error) { // lookupFile find the file to serve // if the file is dir ,search the index.html as default file( MUST NOT A DIR also) // if the index.html not exist or is a dir, give a forbidden response depending on DirectoryIndex - func lookupFile(ctx *context.Context) (bool, string, os.FileInfo, error) { fp, fi, err := searchFile(ctx) if fp == "" || fi == nil {