From 97a373f5e7cda54fa490a3a6993a0b839ff18b01 Mon Sep 17 00:00:00 2001 From: Unknwon Date: Sat, 23 Aug 2014 21:13:55 +0800 Subject: [PATCH] Fix #362, update code with upstream --- modules/base/markdown.go | 2 +- modules/httplib/httplib.go | 296 ++++++++++++++++++++++---------- modules/httplib/httplib_test.go | 196 +++++++++++++++++++-- 3 files changed, 387 insertions(+), 107 deletions(-) mode change 100755 => 100644 modules/httplib/httplib.go mode change 100755 => 100644 modules/httplib/httplib_test.go diff --git a/modules/base/markdown.go b/modules/base/markdown.go index 0825decb44..b8bd33e113 100644 --- a/modules/base/markdown.go +++ b/modules/base/markdown.go @@ -39,7 +39,7 @@ func isLink(link []byte) bool { func IsMarkdownFile(name string) bool { name = strings.ToLower(name) switch filepath.Ext(name) { - case ".md", ".markdown", ".mdown": + case ".md", ".markdown", ".mdown", ".mkd": return true } return false diff --git a/modules/httplib/httplib.go b/modules/httplib/httplib.go old mode 100755 new mode 100644 index 35870c1f1c..5f592168dc --- a/modules/httplib/httplib.go +++ b/modules/httplib/httplib.go @@ -5,6 +5,8 @@ package httplib +// NOTE: last sync c07b1d8 on Aug 23, 2014. + import ( "bytes" "crypto/tls" @@ -12,91 +14,143 @@ import ( "encoding/xml" "io" "io/ioutil" + "mime/multipart" "net" "net/http" + "net/http/cookiejar" "net/http/httputil" "net/url" "os" "strings" + "sync" "time" ) -var defaultUserAgent = "gogsServer" +var defaultSetting = BeegoHttpSettings{false, "beegoServer", 60 * time.Second, 60 * time.Second, nil, nil, nil, false} +var defaultCookieJar http.CookieJar +var settingMutex sync.Mutex + +// createDefaultCookie creates a global cookiejar to store cookies. +func createDefaultCookie() { + settingMutex.Lock() + defer settingMutex.Unlock() + defaultCookieJar, _ = cookiejar.New(nil) +} + +// Overwrite default settings +func SetDefaultSetting(setting BeegoHttpSettings) { + settingMutex.Lock() + defer settingMutex.Unlock() + defaultSetting = setting + if defaultSetting.ConnectTimeout == 0 { + defaultSetting.ConnectTimeout = 60 * time.Second + } + if defaultSetting.ReadWriteTimeout == 0 { + defaultSetting.ReadWriteTimeout = 60 * time.Second + } +} + +// return *BeegoHttpRequest with specific method +func newBeegoRequest(url, method string) *BeegoHttpRequest { + var resp http.Response + req := http.Request{ + Method: method, + Header: make(http.Header), + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + return &BeegoHttpRequest{url, &req, map[string]string{}, map[string]string{}, defaultSetting, &resp, nil} +} // Get returns *BeegoHttpRequest with GET method. func Get(url string) *BeegoHttpRequest { - var req http.Request - req.Method = "GET" - req.Header = http.Header{} - req.Header.Set("User-Agent", defaultUserAgent) - return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} + return newBeegoRequest(url, "GET") } // Post returns *BeegoHttpRequest with POST method. func Post(url string) *BeegoHttpRequest { - var req http.Request - req.Method = "POST" - req.Header = http.Header{} - req.Header.Set("User-Agent", defaultUserAgent) - return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} + return newBeegoRequest(url, "POST") } // Put returns *BeegoHttpRequest with PUT method. func Put(url string) *BeegoHttpRequest { - var req http.Request - req.Method = "PUT" - req.Header = http.Header{} - req.Header.Set("User-Agent", defaultUserAgent) - return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} + return newBeegoRequest(url, "PUT") } -// Delete returns *BeegoHttpRequest DELETE GET method. +// Delete returns *BeegoHttpRequest DELETE method. func Delete(url string) *BeegoHttpRequest { - var req http.Request - req.Method = "DELETE" - req.Header = http.Header{} - req.Header.Set("User-Agent", defaultUserAgent) - return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} + return newBeegoRequest(url, "DELETE") } // Head returns *BeegoHttpRequest with HEAD method. func Head(url string) *BeegoHttpRequest { - var req http.Request - req.Method = "HEAD" - req.Header = http.Header{} - req.Header.Set("User-Agent", defaultUserAgent) - return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} + return newBeegoRequest(url, "HEAD") +} + +// BeegoHttpSettings +type BeegoHttpSettings struct { + ShowDebug bool + UserAgent string + ConnectTimeout time.Duration + ReadWriteTimeout time.Duration + TlsClientConfig *tls.Config + Proxy func(*http.Request) (*url.URL, error) + Transport http.RoundTripper + EnableCookie bool } // BeegoHttpRequest provides more useful methods for requesting one url than http.Request. type BeegoHttpRequest struct { - url string - req *http.Request - params map[string]string - showdebug bool - connectTimeout time.Duration - readWriteTimeout time.Duration - tlsClientConfig *tls.Config - proxy func(*http.Request) (*url.URL, error) - transport http.RoundTripper + url string + req *http.Request + params map[string]string + files map[string]string + setting BeegoHttpSettings + resp *http.Response + body []byte +} + +// Change request settings +func (b *BeegoHttpRequest) Setting(setting BeegoHttpSettings) *BeegoHttpRequest { + b.setting = setting + return b +} + +// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password. +func (b *BeegoHttpRequest) SetBasicAuth(username, password string) *BeegoHttpRequest { + b.req.SetBasicAuth(username, password) + return b +} + +// SetEnableCookie sets enable/disable cookiejar +func (b *BeegoHttpRequest) SetEnableCookie(enable bool) *BeegoHttpRequest { + b.setting.EnableCookie = enable + return b +} + +// SetUserAgent sets User-Agent header field +func (b *BeegoHttpRequest) SetUserAgent(useragent string) *BeegoHttpRequest { + b.setting.UserAgent = useragent + return b } // Debug sets show debug or not when executing request. func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest { - b.showdebug = isdebug + b.setting.ShowDebug = isdebug return b } // SetTimeout sets connect time out and read-write time out for BeegoRequest. func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest { - b.connectTimeout = connectTimeout - b.readWriteTimeout = readWriteTimeout + b.setting.ConnectTimeout = connectTimeout + b.setting.ReadWriteTimeout = readWriteTimeout return b } // SetTLSClientConfig sets tls connection configurations if visiting https url. func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest { - b.tlsClientConfig = config + b.setting.TlsClientConfig = config return b } @@ -106,6 +160,23 @@ func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest { return b } +// Set the protocol version for incoming requests. +// Client requests always use HTTP/1.1. +func (b *BeegoHttpRequest) SetProtocolVersion(vers string) *BeegoHttpRequest { + if len(vers) == 0 { + vers = "HTTP/1.1" + } + + major, minor, ok := http.ParseHTTPVersion(vers) + if ok { + b.req.Proto = vers + b.req.ProtoMajor = major + b.req.ProtoMinor = minor + } + + return b +} + // SetCookie add cookie into request. func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest { b.req.Header.Add("Cookie", cookie.String()) @@ -114,7 +185,7 @@ func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest { // Set transport to func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest { - b.transport = transport + b.setting.Transport = transport return b } @@ -126,7 +197,7 @@ func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpR // return u, nil // } func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest { - b.proxy = proxy + b.setting.Proxy = proxy return b } @@ -137,6 +208,11 @@ func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest { return b } +func (b *BeegoHttpRequest) PostFile(formname, filename string) *BeegoHttpRequest { + b.files[formname] = filename + return b +} + // Body adds request raw body. // it supports string and []byte. func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest { @@ -154,6 +230,9 @@ func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest { } func (b *BeegoHttpRequest) getResponse() (*http.Response, error) { + if b.resp.StatusCode != 0 { + return b.resp, nil + } var paramBody string if len(b.params) > 0 { var buf bytes.Buffer @@ -174,21 +253,90 @@ func (b *BeegoHttpRequest) getResponse() (*http.Response, error) { b.url = b.url + "?" + paramBody } } else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 { - b.Header("Content-Type", "application/x-www-form-urlencoded") - b.Body(paramBody) + if len(b.files) > 0 { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + for formname, filename := range b.files { + fileWriter, err := bodyWriter.CreateFormFile(formname, filename) + if err != nil { + return nil, err + } + fh, err := os.Open(filename) + if err != nil { + return nil, err + } + //iocopy + _, err = io.Copy(fileWriter, fh) + fh.Close() + if err != nil { + return nil, err + } + } + for k, v := range b.params { + bodyWriter.WriteField(k, v) + } + contentType := bodyWriter.FormDataContentType() + bodyWriter.Close() + b.Header("Content-Type", contentType) + b.req.Body = ioutil.NopCloser(bodyBuf) + b.req.ContentLength = int64(bodyBuf.Len()) + } else { + b.Header("Content-Type", "application/x-www-form-urlencoded") + b.Body(paramBody) + } } url, err := url.Parse(b.url) - if url.Scheme == "" { - b.url = "http://" + b.url - url, err = url.Parse(b.url) - } if err != nil { return nil, err } b.req.URL = url - if b.showdebug { + + trans := b.setting.Transport + + if trans == nil { + // create default transport + trans = &http.Transport{ + TLSClientConfig: b.setting.TlsClientConfig, + Proxy: b.setting.Proxy, + Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout), + } + } else { + // if b.transport is *http.Transport then set the settings. + if t, ok := trans.(*http.Transport); ok { + if t.TLSClientConfig == nil { + t.TLSClientConfig = b.setting.TlsClientConfig + } + if t.Proxy == nil { + t.Proxy = b.setting.Proxy + } + if t.Dial == nil { + t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout) + } + } + } + + var jar http.CookieJar + if b.setting.EnableCookie { + if defaultCookieJar == nil { + createDefaultCookie() + } + jar = defaultCookieJar + } else { + jar = nil + } + + client := &http.Client{ + Transport: trans, + Jar: jar, + } + + if b.setting.UserAgent != "" { + b.req.Header.Set("User-Agent", b.setting.UserAgent) + } + + if b.setting.ShowDebug { dump, err := httputil.DumpRequest(b.req, true) if err != nil { println(err.Error()) @@ -196,38 +344,11 @@ func (b *BeegoHttpRequest) getResponse() (*http.Response, error) { println(string(dump)) } - trans := b.transport - - if trans == nil { - // create default transport - trans = &http.Transport{ - TLSClientConfig: b.tlsClientConfig, - Proxy: b.proxy, - Dial: TimeoutDialer(b.connectTimeout, b.readWriteTimeout), - } - } else { - // if b.transport is *http.Transport then set the settings. - if t, ok := trans.(*http.Transport); ok { - if t.TLSClientConfig == nil { - t.TLSClientConfig = b.tlsClientConfig - } - if t.Proxy == nil { - t.Proxy = b.proxy - } - if t.Dial == nil { - t.Dial = TimeoutDialer(b.connectTimeout, b.readWriteTimeout) - } - } - } - - client := &http.Client{ - Transport: trans, - } - resp, err := client.Do(b.req) if err != nil { return nil, err } + b.resp = resp return resp, nil } @@ -245,6 +366,9 @@ func (b *BeegoHttpRequest) String() (string, error) { // Bytes returns the body []byte in response. // it calls Response inner. func (b *BeegoHttpRequest) Bytes() ([]byte, error) { + if b.body != nil { + return b.body, nil + } resp, err := b.getResponse() if err != nil { return nil, err @@ -257,6 +381,7 @@ func (b *BeegoHttpRequest) Bytes() ([]byte, error) { if err != nil { return nil, err } + b.body = data return data, nil } @@ -278,10 +403,7 @@ func (b *BeegoHttpRequest) ToFile(filename string) error { } defer resp.Body.Close() _, err = io.Copy(f, resp.Body) - if err != nil { - return err - } - return nil + return err } // ToJson returns the map that marshals from the body bytes as json in response . @@ -292,24 +414,18 @@ func (b *BeegoHttpRequest) ToJson(v interface{}) error { return err } err = json.Unmarshal(data, v) - if err != nil { - return err - } - return nil + return err } // ToXml returns the map that marshals from the body bytes as xml in response . // it calls Response inner. -func (b *BeegoHttpRequest) ToXML(v interface{}) error { +func (b *BeegoHttpRequest) ToXml(v interface{}) error { data, err := b.Bytes() if err != nil { return err } err = xml.Unmarshal(data, v) - if err != nil { - return err - } - return nil + return err } // Response executes request client gets response mannually. diff --git a/modules/httplib/httplib_test.go b/modules/httplib/httplib_test.go old mode 100755 new mode 100644 index cc56dd8812..8a529c53a6 --- a/modules/httplib/httplib_test.go +++ b/modules/httplib/httplib_test.go @@ -1,32 +1,196 @@ +// Copyright 2013 The Beego Authors. All rights reserved. +// Copyright 2014 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + package httplib import ( "io/ioutil" + "os" + "strings" "testing" ) -func TestGetUrl(t *testing.T) { - resp, err := Get("http://beego.me/").Debug(true).Response() +func TestResponse(t *testing.T) { + req := Get("http://httpbin.org/get") + resp, err := req.Response() if err != nil { t.Fatal(err) } - if resp.Body == nil { - t.Fatal("body is nil") - } - data, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - if err != nil { - t.Fatal(err) - } - if len(data) == 0 { - t.Fatal("data is no") - } + t.Log(resp) +} - str, err := Get("http://beego.me/").String() +func TestGet(t *testing.T) { + req := Get("http://httpbin.org/get") + b, err := req.Bytes() if err != nil { t.Fatal(err) } - if len(str) == 0 { - t.Fatal("has no info") + t.Log(b) + + s, err := req.String() + if err != nil { + t.Fatal(err) + } + t.Log(s) + + if string(b) != s { + t.Fatal("request data not match") + } +} + +func TestSimplePost(t *testing.T) { + v := "smallfish" + req := Post("http://httpbin.org/post") + req.Param("username", v) + + str, err := req.String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(str, v) + if n == -1 { + t.Fatal(v + " not found in post") + } +} + +func TestPostFile(t *testing.T) { + v := "smallfish" + req := Post("http://httpbin.org/post") + req.Param("username", v) + req.PostFile("uploadfile", "httplib_test.go") + + str, err := req.String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(str, v) + if n == -1 { + t.Fatal(v + " not found in post") + } +} + +func TestSimplePut(t *testing.T) { + str, err := Put("http://httpbin.org/put").String() + if err != nil { + t.Fatal(err) + } + t.Log(str) +} + +func TestSimpleDelete(t *testing.T) { + str, err := Delete("http://httpbin.org/delete").String() + if err != nil { + t.Fatal(err) + } + t.Log(str) +} + +func TestWithCookie(t *testing.T) { + v := "smallfish" + str, err := Get("http://httpbin.org/cookies/set?k1=" + v).SetEnableCookie(true).String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + + str, err = Get("http://httpbin.org/cookies").SetEnableCookie(true).String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(str, v) + if n == -1 { + t.Fatal(v + " not found in cookie") + } +} + +func TestWithBasicAuth(t *testing.T) { + str, err := Get("http://httpbin.org/basic-auth/user/passwd").SetBasicAuth("user", "passwd").String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + n := strings.Index(str, "authenticated") + if n == -1 { + t.Fatal("authenticated not found in response") + } +} + +func TestWithUserAgent(t *testing.T) { + v := "beego" + str, err := Get("http://httpbin.org/headers").SetUserAgent(v).String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(str, v) + if n == -1 { + t.Fatal(v + " not found in user-agent") + } +} + +func TestWithSetting(t *testing.T) { + v := "beego" + var setting BeegoHttpSettings + setting.EnableCookie = true + setting.UserAgent = v + setting.Transport = nil + SetDefaultSetting(setting) + + str, err := Get("http://httpbin.org/get").String() + if err != nil { + t.Fatal(err) + } + t.Log(str) + + n := strings.Index(str, v) + if n == -1 { + t.Fatal(v + " not found in user-agent") + } +} + +func TestToJson(t *testing.T) { + req := Get("http://httpbin.org/ip") + resp, err := req.Response() + if err != nil { + t.Fatal(err) + } + t.Log(resp) + + // httpbin will return http remote addr + type Ip struct { + Origin string `json:"origin"` + } + var ip Ip + err = req.ToJson(&ip) + if err != nil { + t.Fatal(err) + } + t.Log(ip.Origin) + + if n := strings.Count(ip.Origin, "."); n != 3 { + t.Fatal("response is not valid ip") + } +} + +func TestToFile(t *testing.T) { + f := "beego_testfile" + req := Get("http://httpbin.org/ip") + err := req.ToFile(f) + if err != nil { + t.Fatal(err) + } + defer os.Remove(f) + b, err := ioutil.ReadFile(f) + if n := strings.Index(string(b), "origin"); n == -1 { + t.Fatal(err) } }