package utils import ( "fmt" "io" "io/ioutil" "net" "net/http" "os" "sync" "time" ) type FetchConfig struct { UserAgent string Mime string etags map[string]string etagsLock *sync.RWMutex EnableEtags bool } func NewFetchConfig() *FetchConfig { return &FetchConfig{ etags: make(map[string]string), etagsLock: &sync.RWMutex{}, Mime: "application/json", } } type HttpNotModified struct { File string } func (e HttpNotModified) Error() string { return fmt.Sprintf("HTTP 304 Not modified for %s", e.File) } type IdenticalEtag struct { File string Etag string } func (e IdenticalEtag) Error() string { return fmt.Sprintf("File %s is identical according to Etag: %s", e.File, e.Etag) } func (c *FetchConfig) FetchFile(file string) ([]byte, int, error) { var f io.Reader var err error status_code := -1 if len(file) > 8 && (file[0:7] == "http://" || file[0:8] == "https://") { // Copying base of DefaultTransport from https://golang.org/src/net/http/transport.go // There is a proposal for a Clone of tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, ProxyConnectHeader: map[string][]string{}, } // Keep User-Agent in proxy request tr.ProxyConnectHeader.Set("User-Agent", c.UserAgent) client := &http.Client{Transport: tr} req, err := http.NewRequest("GET", file, nil) req.Header.Set("User-Agent", c.UserAgent) if c.Mime != "" { req.Header.Set("Accept", c.Mime) } c.etagsLock.RLock() etag, ok := c.etags[file] c.etagsLock.RUnlock() if c.EnableEtags && ok { req.Header.Set("If-None-Match", etag) } proxyurl, err := http.ProxyFromEnvironment(req) if err != nil { return nil, -1, err } proxyreq := http.ProxyURL(proxyurl) tr.Proxy = proxyreq fhttp, err := client.Do(req) if err != nil { return nil, -1, err } if fhttp.Body != nil { defer fhttp.Body.Close() } defer client.CloseIdleConnections() if fhttp.StatusCode == 304 { return nil, fhttp.StatusCode, HttpNotModified{ File: file, } } if fhttp.StatusCode != 200 { c.etagsLock.Lock() delete(c.etags, file) c.etagsLock.Unlock() } newEtag := fhttp.Header.Get("ETag") if c.EnableEtags && newEtag != "" && newEtag == c.etags[file] { return nil, fhttp.StatusCode, IdenticalEtag{ File: file, Etag: newEtag, } } c.etagsLock.Lock() c.etags[file] = newEtag c.etagsLock.Unlock() f = fhttp.Body status_code = fhttp.StatusCode } else { f, err = os.Open(file) if err != nil { return nil, -1, err } } data, err := ioutil.ReadAll(f) if err != nil { return nil, -1, err } return data, status_code, nil }