Add v2 module, tests; update readme

README.md

@@ -3,9 +3,7 @@
 Trades REST API paging logic and request throttling for a
 channel and a `for` loop.
 
-For the moment, *depager* only supports JSON responses.
-
-*depager* requires structs conforming to the following
+*depager* requires values conforming to the following
 interfaces:
 
 ```go
@@ -13,129 +11,150 @@ interfaces:
 The `Page` interface must wrap server responses. This
 allows pagers to calculate page sizes and iterate over
 response aggregates.
-
-The underlying implementation must be a pointer to a
-struct containing the desired response fields, all tagged
-appropriately. Any fields corresponding to
-platform-specific error responses should also be
-included.
 */
 type Page[T any] interface {
-    // Count must return the total number of items to be paged
-	Count() uint64
-
-    // Elems must return the items from the current page
+	// Elems must return the items from the current page
 	Elems() []T
 }
 
-type Client interface {
-	Get(uri url.URL) ([]byte, error)
-}
-
-type PagedURI interface {
-	PageURI(limit, offset uint64) url.URL
+// Exposes the part of the client that depager understands.
+type Client[T any] interface {
+	// NextPage returns the next page or it returns an error
+	NextPage(
+		offset uint64, // item offset at which to start page
+	) (
+		page Page[T],
+		count uint64, // total count of all items being paged
+		err error,
+	)
 }
-
 ```
 
 And in return, *depager* provides the following:
 
 ```go
 type Pager[T any] interface {
+	// Iter is intended to be used in a for-range loop
 	Iter() <-chan T
+
+	// LastErr must return the first error encountered, if any
 	LastErr() error
 }
+
 ```
 
 ## Example
 
 ```go
+package main
+
 import (
-    dp "idio.link/go/depager"
+    "context"
+    "fmt"
+    "log"
+    "net/http"
+    "net/url"
+    "os"
+
+    dp "idio.link/go/depager/v2"
 )
 
-type pagedURI struct {
-	uri *url.URL
+type MyClient struct {
+	pageSize uint64
+	// more fields
 }
 
-func (u pagedURI) PageURI(limit, offset uint64) url.URL {
-    /*
-    Different APIs use different conventions. Simply map
-    limit and offset to the apposite field names with
-    whatever semantics the server expects.
-    */
-	if limit > MaxPageSize {
-		limit = MaxPageSize
-	}
-	uri := *u.uri
-	q := (&uri).Query()
-	q.Add("first", strconv.FormatUint(offset, 10))
-	q.Add("last", strconv.FormatUint(offset+limit-1, 10))
-	(&uri).RawQuery = q.Encode()
-	return uri
+func (c *MyClient) get(
+	uri *url.URL,
+) (head http.Header, body io.ReadCloser, err error) {
+	// do things
+	return
 }
 
-func NewMyAggregate[T any]() *MyAggregate[T] {
-	return &MyAggregate[T]{Items: make([]T, 0, 64)}
+func (c *MyClient) pagify(
+	pathURI *url.URL,
+	first,
+	last uint64,
+) (uri *url.URL) {
+	// glue path to base URI
+	return
 }
 
-type MyAggregate[T any] struct {
-	Total int32 `json:"total"`
-	Items []T   `json:"items"`
+func (c *MyClient) Things(id int) dp.Pager[*Thing] {
+	// TODO validate; if used elsewhere, take boxed id instead
+	path := "/pile/%d/things"
+	subClient := &MySubclient[*Thing]{
+        MyClient: c,
+        path: url.Parse(fmt.Sprintf(path, id)),
+    }
+
+	return dp.NewPager(subClient, c.pageSize)
 }
 
-func (a *MyAggregate[_]) Count() uint64 {
-	return uint64(a.Total)
+type MySubclient[T any] struct {
+	MyClient
+
+	path *url.URL
 }
 
-func (a *MyAggregate[T]) Elems() []T {
-	return a.Items
+func (c *MySubclient[T]) NextPage(
+	offset uint64,
+) (page dp.Page[T], totalItems uint64, err error) {
+	/*
+	Different APIs use different conventions. Simply map 
+	`offset` to the apposite fields or headers with whatever
+	semantics the server expects.
+
+	Most days, the page size should be the largest that the
+	server is willing to accommodate.
+	*/
+	first := offset
+	last := first + c.pageSize - 1
+
+	uri := c.pagify(c.path, first, last)
+	header, body, err := c.get(uri)
+	page := &MyAggregate[T]{}
+	// parsing, etc.
+
+	/*
+	When returning the total count of all items to be paged,
+	if the server API only provides you the total number of
+	pages, simply calculate total*pageSize and return that.
+	*/
+	return page, totalItems, nil
 }
 
-// The client can use this func for all its paged responses.
-func makeMyPager[T any](
-	client *MyClient,
-	resource *url.URL,
-) dp.Pager[T] {
-	return dp.NewPager(
-		pagedURI{resource},
-		client,
-		MaxPageSize,  // most days, this should be the maximum page size that the server will provide
-		func() dp.Page[T] {
-			return NewMyAggregate[T]()
-		},
-	)
+type MyAggregate[T any] struct {
+	Items []T `json:"items"`
 }
 
-/*
-.
-.
-.
-*/
+func (a *MyAggregate[T]) Elems() []T {
+	return a.Items
+}
 
 type Thing struct {
-	Id   int32  `json:"id"`
+	Id   int32	`json:"id"`
 	Name string `json:"name"`
 }
 
 func main() {
-    id := 82348
-    var thingPager dp.Pager[*Thing] =
-        client.GetThings(id)
-
-    for thing := range thingPager.Iter() {
-        // stay responsive!
-        if ctx.Err() != nil {
-            break
-        }
-        // do stuff with each thing
-    }
+	ctx, _ := context.WithCancel(context.Background())
+	client := NewMyClient(...)
+	id := 82348
+	pager := client.Things(id)
+	for e := range pager.Iter() {
+		// there could be many pages; stay responsive!
+		if ctx.Err() != nil {
+				break
+		}
+		// do stuff with each thing
+	}
 
-    // finally, check for errors
-    if err := thingPager.LastErr(); err != nil {
-        log.Printf("do stuff with things: %v", err)
-        os.Exit(1)
-    }
+	// finally, check for errors
+	if err := pager.LastErr(); err != nil {
+		log.Printf("do stuff with things: %v", err)
+		os.Exit(1)
+	}
 }
 ```
 

v2/depager.go

@@ -0,0 +1,113 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ */
+
+package depager
+
+import (
+	"fmt"
+)
+
+/*
+The `Page` interface must wrap server responses. This
+allows pagers to calculate page sizes and iterate over
+response aggregates.
+
+If the underlying value of this interface is `nil` (e.g. a
+nil pointer to a struct or a nil slice), `Elems()` will
+panic.
+*/
+type Page[T any] interface {
+	// Elems must return the items from the current page
+	Elems() []T
+}
+
+// Exposes the part of the client that depager understands.
+type Client[T any] interface {
+	// NextPage returns the next page or it returns an error
+	NextPage(offset uint64) (
+		page Page[T],
+		count uint64, // total count of all items being paged
+		err error,
+	)
+}
+
+type Pager[T any] interface {
+	// Iter is intended to be used in a for-range loop
+	Iter() <-chan T
+
+	// LastErr must return the first error encountered, if any
+	LastErr() error
+}
+
+func NewPager[T any](
+	c Client[T],
+	pageSize uint64,
+) Pager[T] {
+	return &pager[T]{
+		client: c,
+		n:      pageSize,
+		p:      4,
+	}
+}
+
+/*
+Retrieve n items in the range [m*n, m*n + n - 1], inclusive.
+Keep p pages buffered.
+*/
+type pager[T any] struct {
+	client Client[T]
+	m      uint64
+	n      uint64
+	err    error
+	p      int
+	cnt    uint64
+}
+
+func (p *pager[T]) iteratePages() <-chan Page[T] {
+	ch := make(chan Page[T], p.p)
+	go func() {
+		defer close(ch)
+		for {
+			page, cnt, err :=
+				p.client.NextPage(p.m * p.n)
+			if err != nil {
+				p.err = err
+				return
+			}
+			if p.cnt == 0 {
+				p.cnt = cnt
+			}
+			ch <- page
+
+			if (p.m*p.n + p.n) >= p.cnt {
+				return
+			}
+			p.m++
+		}
+	}()
+	return ch
+}
+
+func (p *pager[T]) Iter() <-chan T {
+	ch := make(chan T, p.n)
+	go func() {
+		defer close(ch)
+		for page := range p.iteratePages() {
+			for _, i := range page.Elems() {
+				ch <- i
+			}
+			if p.err != nil {
+				p.err = fmt.Errorf("pager: iterate items: %s", p.err)
+				return
+			}
+		}
+	}()
+	return ch
+}
+
+func (p *pager[T]) LastErr() error {
+	return p.err
+}

v2/depager_test.go

@@ -0,0 +1,116 @@
+package depager_test
+
+import (
+	"fmt"
+	"testing"
+
+	dp "idio.link/go/depager/v2"
+)
+
+type NoopClient[T any] struct {
+	err   error
+	pages []dp.Page[T]
+	m     int
+	cnt   uint64
+}
+
+func (c *NoopClient[T]) NextPage(
+	_offset uint64,
+) (page dp.Page[T], cnt uint64, err error) {
+	if len(c.pages) == 0 {
+		return
+	}
+	if c.m >= len(c.pages) {
+		err = fmt.Errorf("client: next page: exceeded max pages")
+		return
+	}
+	page = c.pages[c.m]
+	cnt = c.cnt
+	err = c.err
+	c.m++
+	return
+}
+
+func NewNoopClient[T any](
+	cnt int,
+	err error,
+	pages []dp.Page[T],
+) dp.Client[T] {
+	return &NoopClient[T]{
+		cnt:   uint64(cnt),
+		err:   err,
+		pages: pages,
+	}
+}
+
+type Aggr[T any] []T
+
+func (a Aggr[T]) Elems() []T {
+	return []T(a)
+}
+
+func TestUsingNoopClient(t *testing.T) {
+	client := NewNoopClient[any](0, nil,
+		[]dp.Page[any]{
+			Aggr[any]{},
+		},
+	)
+	pager := dp.NewPager(client, 0)
+	for range pager.Iter() {
+	}
+	if err := pager.LastErr(); err != nil {
+		t.Errorf("unexpected error in pager with noop client: %v", err)
+	}
+}
+
+func TestNoopClientReturnsError(t *testing.T) {
+	client := NewNoopClient[any](0, fmt.Errorf("whomp"),
+		[]dp.Page[any]{
+			Aggr[any]{},
+		},
+	)
+	pager := dp.NewPager(client, 0)
+	for range pager.Iter() {
+	}
+	if err := pager.LastErr(); err == nil {
+		t.Errorf("unexpected success: %v", err)
+	}
+}
+
+func TestClientReturnsNonemptyPage(t *testing.T) {
+	pageSize := 2
+	itemCount := 3
+	client := NewNoopClient[any](itemCount, nil,
+		[]dp.Page[any]{
+			Aggr[any]{1, 2},
+			Aggr[any]{3},
+		},
+	)
+	pager := dp.NewPager(client, uint64(pageSize))
+	var elem int
+	for e := range pager.Iter() {
+		elem = e.(int)
+	}
+	if err := pager.LastErr(); err != nil {
+		t.Errorf("unexpected error in pager: %v", err)
+	}
+	if elem != 3 {
+		t.Errorf("unexpected value: '%v'", elem)
+	}
+}
+
+func TestClientReturnsFewerPagesThanExpected(t *testing.T) {
+	pageSize := 1
+	itemCount := pageSize + 1
+	client := NewNoopClient[any](itemCount, nil,
+		[]dp.Page[any]{
+			Aggr[any]{0},
+		},
+	)
+	pager := dp.NewPager(client, uint64(pageSize))
+	for range pager.Iter() {
+	}
+	if err := pager.LastErr(); err == nil {
+		t.Errorf("unexpected success in pager: %v", err)
+	}
+}

v2/go.mod

@@ -0,0 +1,3 @@
+module idio.link/go/depager/v2
+
+go 1.18