migratetogogs/gitlab_client.go

/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */
package main

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"time"

	dp "idio.link/go/depager/v2"
)

func NewGitLabClient(ctx context.Context) *GitLabClient {
	t := &http.Transport{
		TLSClientConfig:       &tls.Config{InsecureSkipVerify: false},
		TLSHandshakeTimeout:   15 * time.Second,
		IdleConnTimeout:       30 * time.Minute,
		ResponseHeaderTimeout: 180 * time.Second,
		ExpectContinueTimeout: 10 * time.Second,
	}
	c := &http.Client{
		Transport: t,
		Timeout:   30 * time.Minute,
	}
	baseURI, _ := url.Parse("https://gitlab.com/api/v4/")

	return &GitLabClient{
		baseURI:    baseURI,
		ctx:        ctx,
		httpClient: c,
		retries:    3,
		retryWait:  10 * time.Second,
	}
}

type GitLabClient struct {
	ctx             context.Context
	httpClient      *http.Client
	baseURI         *url.URL
	blockReqsBefore time.Time
	retries         int
	retryWait       time.Duration
}

func (c *GitLabClient) expandResource(
	resource *url.URL,
) *url.URL {
	ex, err :=
		url.JoinPath(c.baseURI.String(), resource.EscapedPath())
	if err != nil {
		panic(fmt.Sprintf("BUG: gitlab client: expand resource: failed to join path: '%s' + '%s': %v", c.baseURI, resource.RequestURI(), err))
	}
	next, err := url.Parse(ex)
	if err != nil {
		panic(fmt.Sprintf("BUG: gitlab client: expand resource: failed to parse expanded resource '%s': %v", ex, err))
	}
	return next
}

// retry when server rate limits are exceeded
// See https://docs.gitlab.com/ee/api/rest/#status-codes
func (c *GitLabClient) sendRequestWithRetry(
	req *http.Request,
) (resp *http.Response, err error) {
	for i := 0; i < c.retries+1; i++ {
		resp, err = c.httpClient.Do(req)
		if err != nil {
			err = fmt.Errorf("send request with retry: %w", err)
			return
		}
		if resp.StatusCode != http.StatusTooManyRequests {
			break
		}
		log.Printf("info: request throttled: %s '%s'", req.Method, req.RequestURI)
		// TODO Offer an async option in future, but this is
		// acceptable, for now.
		time.Sleep(c.retryWait / time.Millisecond)
	}
	if resp == nil {
		err = fmt.Errorf("send request with retry: unknown failure")
		return
	}
	return
}

func (c *GitLabClient) request(
	method string,
	resource *url.URL,
	body io.Reader,
) (respHead http.Header, respBody []byte, err error) {
	req, err := http.NewRequestWithContext(
		c.ctx,
		method,
		resource.String(),
		body,
	)
	if err != nil {
		err = fmt.Errorf("request %v: %w", resource, err)
		return
	}

	resp, err := c.sendRequestWithRetry(req)
	if err != nil {
		err = fmt.Errorf("request '%s': %+v: %w", req.URL, req, err)
		return
	}
	defer resp.Body.Close()

	respHead = resp.Header
	respBody, err = io.ReadAll(resp.Body)
	if err != nil {
		err = fmt.Errorf("request %v: failed to read response body: %w", resource, err)
		return
	}

	err = c.ratelimitRequests(resp)
	if err != nil {
		err = fmt.Errorf("request %v: %w", resource, err)
		return
	}

	// Success response
	if 200 <= resp.StatusCode && resp.StatusCode <= 299 {
		return
	}

	err = fmt.Errorf("request %v: %s", resource, http.StatusText(resp.StatusCode))
	return
}

/*
Throttle requests. See

* https://gitlab.com/gitlab-com/runbooks/-/tree/master/docs/rate-limiting
* https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits

We postpone subsequent requests by at least win/rem
*/
func (c *GitLabClient) ratelimitRequests(
	resp *http.Response,
) error {
	window := 60 * time.Second // window is not sent in headers
	remStr := resp.Header.Get("Rate-Limit-Remaining")

	remaining, err := strconv.ParseInt(remStr, 10, 64)
	if remStr != "" && err != nil {
		return fmt.Errorf("throttle requests: failed to parse header Rate-Limit-Remaining: %w", err)
	}
	if remaining != 0 && window != 0 {
		msDelay := int64(window/time.Millisecond) / remaining
		delay := time.Duration(msDelay) * time.Millisecond
		c.blockReqsBefore = time.Now().Add(delay)
	}
	return nil
}

func (c *GitLabClient) get(
	resource *url.URL,
) (http.Header, []byte, error) {
	return c.request(http.MethodGet, resource, nil)
}

func newGitLabSubclient[T any](
	c *GitLabClient,
	resource *url.URL,
) *GitLabSubclient[T] {
	expanded := c.expandResource(resource)
	return &GitLabSubclient[T]{
		GitLabClient: *c,
		uri:          expanded,
	}
}

type GitLabSubclient[T any] struct {
	GitLabClient

	uri *url.URL
}

func (c *GitLabSubclient[T]) NextPage(
	offset uint64,
) (page dp.Page[T], cnt uint64, err error) {
	aggr := make([]T, 0, 32)
	head, body, err := c.get(c.uri)
	if err != nil {
		err = fmt.Errorf("gitlab client: next page: %w", err)
		return
	}
	cnt, err = strconv.ParseUint(head.Get("x-total"), 10, 64)
	if err != nil {
		err = fmt.Errorf("gitlab client: next page: parse header 'x-total': %w", err)
		return
	}
	err = json.Unmarshal(body, &aggr)
	if err != nil {
		err = fmt.Errorf("gitlab client: next page: unmarshal response: %w", err)
		return
	}
	if next := getLinkNext(head.Get("link")); next != "" {
		c.uri, err = url.Parse(next)
		if err != nil {
			err = fmt.Errorf("gitlab client: next page: unable to parse next link '%s': %w", next, err)
			return
		}
	}
	page = GitLabAggregate[T](aggr)
	return
}

type GitLabAggregate[T any] []T

func (a GitLabAggregate[T]) Elems() []T {
	return a
}

type GitLabProject struct {
	Description       string `json:"description"`
	PathWithNamespace string `json:"path_with_namespace"`
	RepoURL           string `json:"http_url_to_repo"`
	WebURL            string `json:"web_url"`
	DefaultBranch     string `json:"default_branch"`
}

func FetchGitLabGroupProjects[T *GitLabProject](
	c *GitLabClient,
	group string,
) dp.Pager[T] {
	resStrFmt := "/groups/%s/projects?simple=true"
	escaped := url.PathEscape(group)
	resStr := fmt.Sprintf(resStrFmt, escaped)

	resource, err := url.Parse(resStr)
	if err != nil { // should only occur in case of bugs
		panic(err)
	}
	return dp.NewPager[T](
		newGitLabSubclient[T](c, resource),
		100,
	)
}