diff --git a/cmd/gortr/gortr.go b/cmd/gortr/gortr.go index 2e19288..ef5b060 100644 --- a/cmd/gortr/gortr.go +++ b/cmd/gortr/gortr.go @@ -12,13 +12,6 @@ import ( "errors" "flag" "fmt" - rtr "github.com/cloudflare/gortr/lib" - "github.com/cloudflare/gortr/prefixfile" - "github.com/cloudflare/gortr/utils" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - log "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh" "io/ioutil" "net/http" "os" @@ -28,6 +21,14 @@ import ( "sync" "syscall" "time" + + rtr "github.com/cloudflare/gortr/lib" + "github.com/cloudflare/gortr/prefixfile" + "github.com/cloudflare/gortr/utils" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" ) const ( @@ -244,14 +245,12 @@ func (s *state) updateFile(file string) error { log.Debugf("Refreshing cache from %s", file) s.lastts = time.Now().UTC() - data, code, lastrefresh, err := s.fetchConfig.FetchFile(file) + data, code, err := s.fetchConfig.FetchFile(file) if err != nil { return err } - if lastrefresh { - LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9)) - } if code != -1 { + LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9)) RefreshStatusCode.WithLabelValues(file, fmt.Sprintf("%d", code)).Inc() } @@ -368,15 +367,14 @@ func (s *state) updateFile(file string) error { func (s *state) updateSlurm(file string) error { log.Debugf("Refreshing slurm from %v", file) - data, code, lastrefresh, err := s.fetchConfig.FetchFile(file) + data, code, err := s.fetchConfig.FetchFile(file) if err != nil { return err } - if lastrefresh { - LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9)) - } + if code != -1 { RefreshStatusCode.WithLabelValues(file, fmt.Sprintf("%d", code)).Inc() + LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9)) } buf := bytes.NewBuffer(data) diff --git a/cmd/rtrmon/rtrmon.go b/cmd/rtrmon/rtrmon.go index 67f1f1a..5100464 100644 --- a/cmd/rtrmon/rtrmon.go +++ b/cmd/rtrmon/rtrmon.go @@ -7,13 +7,6 @@ import ( "errors" "flag" "fmt" - rtr "github.com/cloudflare/gortr/lib" - "github.com/cloudflare/gortr/prefixfile" - "github.com/cloudflare/gortr/utils" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" - log "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh" "io/ioutil" "net" "net/http" @@ -22,6 +15,14 @@ import ( "runtime" "sync" "time" + + rtr "github.com/cloudflare/gortr/lib" + "github.com/cloudflare/gortr/prefixfile" + "github.com/cloudflare/gortr/utils" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" ) const ( @@ -264,7 +265,7 @@ func (c *Client) Start(id int, ch chan int) { } } else { log.Infof("%d: Fetching %s", c.id, c.Path) - data, _, _, err := c.FetchConfig.FetchFile(c.Path) + data, _, err := c.FetchConfig.FetchFile(c.Path) if err != nil { log.Error(err) continue diff --git a/utils/utils.go b/utils/utils.go index 22e6324..cb2fc7b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -7,8 +7,8 @@ import ( "net" "net/http" "os" - "time" "sync" + "time" ) type FetchConfig struct { @@ -22,9 +22,9 @@ type FetchConfig struct { func NewFetchConfig() *FetchConfig { return &FetchConfig{ - etags: make(map[string]string), + etags: make(map[string]string), etagsLock: &sync.RWMutex{}, - Mime: "application/json", + Mime: "application/json", } } @@ -45,11 +45,13 @@ func (e IdenticalEtag) Error() string { return fmt.Sprintf("File %s is identical according to Etag: %s", e.File, e.Etag) } -func (c *FetchConfig) FetchFile(file string) ([]byte, int, bool, error) { +func (c *FetchConfig) FetchFile(file string) ([]byte, int, error) { var f io.Reader var err error - if len(file) > 8 && (file[0:7] == "http://" || file[0:8] == "https://") { + status_code := -1 + + if len(file) > 8 && (file[0:7] == "http://" || file[0:8] == "https://") { // Copying base of DefaultTransport from https://golang.org/src/net/http/transport.go // There is a proposal for a Clone of tr := &http.Transport{ @@ -65,6 +67,7 @@ func (c *FetchConfig) FetchFile(file string) ([]byte, int, bool, error) { ExpectContinueTimeout: 1 * time.Second, ProxyConnectHeader: map[string][]string{}, } + // Keep User-Agent in proxy request tr.ProxyConnectHeader.Set("User-Agent", c.UserAgent) @@ -84,61 +87,57 @@ func (c *FetchConfig) FetchFile(file string) ([]byte, int, bool, error) { proxyurl, err := http.ProxyFromEnvironment(req) if err != nil { - return nil, -1, false, err + return nil, -1, err } proxyreq := http.ProxyURL(proxyurl) tr.Proxy = proxyreq - if err != nil { - return nil, -1, false, err - } - fhttp, err := client.Do(req) if err != nil { - return nil, -1, false, err + return nil, -1, err } if fhttp.Body != nil { defer fhttp.Body.Close() } defer client.CloseIdleConnections() - //RefreshStatusCode.WithLabelValues(file, fmt.Sprintf("%d", fhttp.StatusCode)).Inc() if fhttp.StatusCode == 304 { - //LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9)) - return nil, fhttp.StatusCode, true, HttpNotModified{ + return nil, fhttp.StatusCode, HttpNotModified{ File: file, } - } else if fhttp.StatusCode != 200 { + } + + if fhttp.StatusCode != 200 { c.etagsLock.Lock() delete(c.etags, file) c.etagsLock.Unlock() - return nil, fhttp.StatusCode, true, fmt.Errorf("HTTP %s", fhttp.Status) } - //LastRefresh.WithLabelValues(file).Set(float64(s.lastts.UnixNano() / 1e9)) - - f = fhttp.Body newEtag := fhttp.Header.Get("ETag") - - if !c.EnableEtags || newEtag == "" || newEtag != c.etags[file] { // check lock here - c.etagsLock.Lock() - c.etags[file] = newEtag - c.etagsLock.Unlock() - } else { - return nil, fhttp.StatusCode, true, IdenticalEtag{ + if c.EnableEtags && newEtag != "" && newEtag == c.etags[file] { + return nil, fhttp.StatusCode, IdenticalEtag{ File: file, Etag: newEtag, } } + + c.etagsLock.Lock() + c.etags[file] = newEtag + c.etagsLock.Unlock() + + f = fhttp.Body + status_code = fhttp.StatusCode } else { f, err = os.Open(file) if err != nil { - return nil, -1, false, err + return nil, -1, err } } + data, err := ioutil.ReadAll(f) if err != nil { - return nil, -1, false, err + return nil, -1, err } - return data, -1, false, nil + + return data, status_code, nil }