/* * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ package main import ( "context" "crypto/tls" "encoding/json" "fmt" "io" "log" "net/http" "net/url" "strconv" "time" dp "idio.link/go/depager/v2" ) func NewGitLabClient(ctx context.Context) *GitLabClient { t := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: false}, TLSHandshakeTimeout: 15 * time.Second, IdleConnTimeout: 30 * time.Minute, ResponseHeaderTimeout: 180 * time.Second, ExpectContinueTimeout: 10 * time.Second, } c := &http.Client{ Transport: t, Timeout: 30 * time.Minute, } baseURI, _ := url.Parse("https://gitlab.com/api/v4/") return &GitLabClient{ baseURI: baseURI, ctx: ctx, httpClient: c, retries: 3, retryWait: 10 * time.Second, } } type GitLabClient struct { ctx context.Context httpClient *http.Client baseURI *url.URL blockReqsBefore time.Time retries int retryWait time.Duration } func (c *GitLabClient) expandResource( resource *url.URL, ) *url.URL { ex, err := url.JoinPath(c.baseURI.String(), resource.EscapedPath()) if err != nil { panic(fmt.Sprintf("BUG: gitlab client: expand resource: failed to join path: '%s' + '%s': %v", c.baseURI, resource.RequestURI(), err)) } next, err := url.Parse(ex) if err != nil { panic(fmt.Sprintf("BUG: gitlab client: expand resource: failed to parse expanded resource '%s': %v", ex, err)) } return next } // retry when server rate limits are exceeded // See https://docs.gitlab.com/ee/api/rest/#status-codes func (c *GitLabClient) sendRequestWithRetry( req *http.Request, ) (resp *http.Response, err error) { for i := 0; i < c.retries+1; i++ { resp, err = c.httpClient.Do(req) if err != nil { err = fmt.Errorf("send request with retry: %w", err) return } if resp.StatusCode != http.StatusTooManyRequests { break } log.Printf("info: request throttled: %s '%s'", req.Method, req.RequestURI) // TODO Offer an async option in future, but this is // acceptable, for now. time.Sleep(c.retryWait / time.Millisecond) } if resp == nil { err = fmt.Errorf("send request with retry: unknown failure") return } return } func (c *GitLabClient) request( method string, resource *url.URL, body io.Reader, ) (respHead http.Header, respBody []byte, err error) { req, err := http.NewRequestWithContext( c.ctx, method, resource.String(), body, ) if err != nil { err = fmt.Errorf("request %v: %w", resource, err) return } resp, err := c.sendRequestWithRetry(req) if err != nil { err = fmt.Errorf("request '%s': %+v: %w", req.URL, req, err) return } defer resp.Body.Close() respHead = resp.Header respBody, err = io.ReadAll(resp.Body) if err != nil { err = fmt.Errorf("request %v: failed to read response body: %w", resource, err) return } err = c.ratelimitRequests(resp) if err != nil { err = fmt.Errorf("request %v: %w", resource, err) return } // Success response if 200 <= resp.StatusCode && resp.StatusCode <= 299 { return } err = fmt.Errorf("request %v: %s", resource, http.StatusText(resp.StatusCode)) return } /* Throttle requests. See * https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/rate-limiting * https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits We postpone subsequent requests by at least win/rem */ func (c *GitLabClient) ratelimitRequests( resp *http.Response, ) error { window := 60 * time.Second // window is not sent in headers remStr := resp.Header.Get("Rate-Limit-Remaining") remaining, err := strconv.ParseInt(remStr, 10, 64) if remStr != "" && err != nil { return fmt.Errorf("throttle requests: failed to parse header Rate-Limit-Remaining: %w", err) } if remaining != 0 && window != 0 { msDelay := int64(window/time.Millisecond) / remaining delay := time.Duration(msDelay) * time.Millisecond c.blockReqsBefore = time.Now().Add(delay) } return nil } func (c *GitLabClient) get( resource *url.URL, ) (http.Header, []byte, error) { return c.request(http.MethodGet, resource, nil) } func newGitLabSubclient[T any]( c *GitLabClient, resource *url.URL, ) *GitLabSubclient[T] { expanded := c.expandResource(resource) return &GitLabSubclient[T]{ GitLabClient: *c, uri: expanded, } } type GitLabSubclient[T any] struct { GitLabClient uri *url.URL } func (c *GitLabSubclient[T]) NextPage( offset uint64, ) (page dp.Page[T], cnt uint64, err error) { aggr := make([]T, 0, 32) head, body, err := c.get(c.uri) if err != nil { err = fmt.Errorf("gitlab client: next page: %w", err) return } cnt, err = strconv.ParseUint(head.Get("x-total"), 10, 64) if err != nil { err = fmt.Errorf("gitlab client: next page: parse header 'x-total': %w", err) return } err = json.Unmarshal(body, &aggr) if err != nil { err = fmt.Errorf("gitlab client: next page: unmarshal response: %w", err) return } if next := getLinkNext(head.Get("link")); next != "" { c.uri, err = url.Parse(next) if err != nil { err = fmt.Errorf("gitlab client: next page: unable to parse next link '%s': %w", next, err) return } } page = GitLabAggregate[T](aggr) return } type GitLabAggregate[T any] []T func (a GitLabAggregate[T]) Elems() []T { return a } type GitLabProject struct { Description string `json:"description"` PathWithNamespace string `json:"path_with_namespace"` RepoURL string `json:"http_url_to_repo"` WebURL string `json:"web_url"` DefaultBranch string `json:"default_branch"` } func FetchGitLabGroupProjects[T *GitLabProject]( c *GitLabClient, group string, ) dp.Pager[T] { resStrFmt := "/groups/%s/projects?simple=true" escaped := url.PathEscape(group) resStr := fmt.Sprintf(resStrFmt, escaped) resource, err := url.Parse(resStr) if err != nil { // should only occur in case of bugs panic(err) } return dp.NewPager[T]( newGitLabSubclient[T](c, resource), 100, ) }