Add pagination to schemas (#638)

* Added pagination to schemas.

* Added pagination integration test for schemas.

* tags

doc/swagger.yaml

@@ -1824,6 +1824,10 @@ definitions:
     type: object
   pkg_server_router.ListSchemasResponse:
     properties:
+      nextPageToken:
+        description: Pagination token to retrieve the next page of results. If the
+          value is "", it means no further results for the request.
+        type: string
       schemas:
         description: Schemas is the list of all schemas the service holds
         items:
@@ -4049,6 +4053,17 @@ paths:
       consumes:
       - application/json
       description: List Credential Schemas stored by the service
+      parameters:
+      - description: Hint to the server of the maximum elements to return. More may
+          be returned. When not set, the server will return all elements.
+        in: query
+        name: pageSize
+        type: number
+      - description: Used to indicate to the server to return a specific page of the
+          list results. Must match a previous requests' `nextPageToken`.
+        in: query
+        name: pageToken
+        type: string
       produces:
       - application/json
       responses:

integration/pagination_integration_test.go

@@ -0,0 +1,49 @@
+package integration
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestListSchemaIntegration(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	// In this test we create a couple of schemas, fetch them all using pagination, and validate that the created ones
+	// were returned in any of the results.
+	schema1, err := CreateKYCSchema()
+	assert.NoError(t, err)
+
+	schema1ID, err := getJSONElement(schema1, "$.id")
+	assert.NoError(t, err)
+
+	schema2, err := CreateKYCSchema()
+	assert.NoError(t, err)
+
+	schema2ID, err := getJSONElement(schema2, "$.id")
+	assert.NoError(t, err)
+
+	schemasPage, err := get(endpoint + version + "schemas?pageSize=1")
+	assert.NoError(t, err)
+
+	var allSchemaIDs string
+	allSchemaIDs = schemasPage
+
+	nextPageToken, err := getJSONElement(schemasPage, "$.nextPageToken")
+	assert.NoError(t, err)
+
+	for nextPageToken != "" {
+		schemasPage, err := get(endpoint + version + "schemas?pageSize=1&pageToken=" + nextPageToken)
+		assert.NoError(t, err)
+
+		allSchemaIDs += schemasPage
+
+		nextPageToken, err = getJSONElement(schemasPage, "$.nextPageToken")
+		assert.NoError(t, err)
+	}
+
+	assert.Contains(t, allSchemaIDs, schema1ID)
+	assert.Contains(t, allSchemaIDs, schema2ID)
+}

pkg/server/router/schema.go

@@ -8,9 +8,9 @@ import (
 	"github.com/TBD54566975/ssi-sdk/did"
 	"github.com/gin-gonic/gin"
 	"github.com/pkg/errors"
-
 	"github.com/tbd54566975/ssi-service/internal/keyaccess"
 	"github.com/tbd54566975/ssi-service/pkg/server/framework"
+	"github.com/tbd54566975/ssi-service/pkg/server/pagination"
 	svcframework "github.com/tbd54566975/ssi-service/pkg/service/framework"
 	"github.com/tbd54566975/ssi-service/pkg/service/schema"
 )
@@ -185,6 +185,9 @@ func (sr SchemaRouter) GetSchema(c *gin.Context) {
 type ListSchemasResponse struct {
 	// Schemas is the list of all schemas the service holds
 	Schemas []GetSchemaResponse `json:"schemas,omitempty"`
+
+	// Pagination token to retrieve the next page of results. If the value is "", it means no further results for the request.
+	NextPageToken string `json:"nextPageToken"`
 }
 
 // ListSchemas godoc
@@ -194,11 +197,20 @@ type ListSchemasResponse struct {
 //	@Tags			Schemas
 //	@Accept			json
 //	@Produce		json
-//	@Success		200	{object}	ListSchemasResponse
-//	@Failure		500	{string}	string	"Internal server error"
+//	@Param			pageSize	query		number	false	"Hint to the server of the maximum elements to return. More may be returned. When not set, the server will return all elements."
+//	@Param			pageToken	query		string	false	"Used to indicate to the server to return a specific page of the list results. Must match a previous requests' `nextPageToken`."
+//	@Success		200			{object}	ListSchemasResponse
+//	@Failure		500			{string}	string	"Internal server error"
 //	@Router			/v1/schemas [get]
 func (sr SchemaRouter) ListSchemas(c *gin.Context) {
-	gotSchemas, err := sr.service.ListSchemas(c)
+	var pageRequest pagination.PageRequest
+	if pagination.ParsePaginationQueryValues(c, &pageRequest) {
+		return
+	}
+
+	gotSchemas, err := sr.service.ListSchemas(c, schema.ListSchemasRequest{
+		PageRequest: &pageRequest,
+	})
 	if err != nil {
 		errMsg := "could not list schemas"
 		framework.LoggingRespondErrWithMsg(c, err, errMsg, http.StatusInternalServerError)
@@ -218,6 +230,10 @@ func (sr SchemaRouter) ListSchemas(c *gin.Context) {
 	}
 
 	resp := ListSchemasResponse{Schemas: schemas}
+
+	if pagination.MaybeSetNextPageToken(c, gotSchemas.NextPageToken, &resp.NextPageToken) {
+		return
+	}
 	framework.Respond(c, resp, http.StatusOK)
 }
 

pkg/server/router/schema_test.go

@@ -50,7 +50,7 @@ func TestSchemaRouter(t *testing.T) {
 				assert.Equal(tt, framework.StatusReady, schemaService.Status().Status)
 
 				// get all schemas (none)
-				gotSchemas, err := schemaService.ListSchemas(context.Background())
+				gotSchemas, err := schemaService.ListSchemas(context.Background(), schema.ListSchemasRequest{})
 				assert.NoError(tt, err)
 				assert.Empty(tt, gotSchemas.Schemas)
 
@@ -74,7 +74,7 @@ func TestSchemaRouter(t *testing.T) {
 				assert.EqualValues(tt, createdSchema.Schema, gotSchema.Schema)
 
 				// get all schemas, expect one
-				gotSchemas, err = schemaService.ListSchemas(context.Background())
+				gotSchemas, err = schemaService.ListSchemas(context.Background(), schema.ListSchemasRequest{})
 				assert.NoError(tt, err)
 				assert.NotEmpty(tt, gotSchemas.Schemas)
 				assert.Len(tt, gotSchemas.Schemas, 1)
@@ -88,7 +88,7 @@ func TestSchemaRouter(t *testing.T) {
 				assert.Equal(tt, credschema.JSONSchema2023Type, createdSchema.Type)
 
 				// get all schemas, expect two
-				gotSchemas, err = schemaService.ListSchemas(context.Background())
+				gotSchemas, err = schemaService.ListSchemas(context.Background(), schema.ListSchemasRequest{})
 				assert.NoError(tt, err)
 				assert.NotEmpty(tt, gotSchemas.Schemas)
 				assert.Len(tt, gotSchemas.Schemas, 2)
@@ -101,7 +101,7 @@ func TestSchemaRouter(t *testing.T) {
 				assert.NoError(tt, err)
 
 				// get all schemas, expect one
-				gotSchemas, err = schemaService.ListSchemas(context.Background())
+				gotSchemas, err = schemaService.ListSchemas(context.Background(), schema.ListSchemasRequest{})
 				assert.NoError(tt, err)
 				assert.NotEmpty(tt, gotSchemas.Schemas)
 				assert.Len(tt, gotSchemas.Schemas, 1)

pkg/service/schema/model.go

@@ -3,6 +3,7 @@ package schema
 import (
 	"github.com/TBD54566975/ssi-sdk/credential/schema"
 	"github.com/TBD54566975/ssi-sdk/util"
+	"github.com/tbd54566975/ssi-service/pkg/server/pagination"
 	"github.com/tbd54566975/ssi-service/pkg/service/common"
 
 	"github.com/tbd54566975/ssi-service/internal/keyaccess"
@@ -40,8 +41,13 @@ type CreateSchemaResponse struct {
 	CredentialSchema *keyaccess.JWT          `json:"credentialSchema,omitempty"`
 }
 
+type ListSchemasRequest struct {
+	PageRequest *pagination.PageRequest
+}
+
 type ListSchemasResponse struct {
-	Schemas []GetSchemaResponse `json:"schemas,omitempty"`
+	Schemas       []GetSchemaResponse `json:"schemas,omitempty"`
+	NextPageToken string              `json:"nextPageToken,omitempty"`
 }
 
 type GetSchemaRequest struct {

pkg/service/schema/service.go

@@ -195,15 +195,15 @@ func (s Service) signCredentialSchema(ctx context.Context, cred credential.Verif
 	return credToken, nil
 }
 
-func (s Service) ListSchemas(ctx context.Context) (*ListSchemasResponse, error) {
+func (s Service) ListSchemas(ctx context.Context, request ListSchemasRequest) (*ListSchemasResponse, error) {
 	logrus.Debug("listing all schemas")
 
-	storedSchemas, err := s.storage.ListSchemas(ctx)
+	storedSchemas, err := s.storage.ListSchemas(ctx, *request.PageRequest.ToServicePage())
 	if err != nil {
 		return nil, sdkutil.LoggingErrorMsg(err, "error getting schemas")
 	}
-	schemas := make([]GetSchemaResponse, 0, len(storedSchemas))
-	for _, stored := range storedSchemas {
+	schemas := make([]GetSchemaResponse, 0, len(storedSchemas.Schemas))
+	for _, stored := range storedSchemas.Schemas {
 		schemas = append(schemas, GetSchemaResponse{
 			ID:               stored.ID,
 			Type:             stored.Type,
@@ -212,7 +212,7 @@ func (s Service) ListSchemas(ctx context.Context) (*ListSchemasResponse, error)
 		})
 	}
 
-	return &ListSchemasResponse{Schemas: schemas}, nil
+	return &ListSchemasResponse{Schemas: schemas, NextPageToken: storedSchemas.NextPageToken}, nil
 }
 
 func (s Service) GetSchema(ctx context.Context, request GetSchemaRequest) (*GetSchemaResponse, error) {

pkg/service/schema/storage.go

@@ -3,12 +3,12 @@ package schema
 import (
 	"context"
 
+	"github.com/TBD54566975/ssi-sdk/credential/schema"
 	"github.com/TBD54566975/ssi-sdk/util"
 	"github.com/goccy/go-json"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-
-	"github.com/TBD54566975/ssi-sdk/credential/schema"
+	"github.com/tbd54566975/ssi-service/pkg/service/common"
 
 	"github.com/tbd54566975/ssi-service/internal/keyaccess"
 	"github.com/tbd54566975/ssi-service/pkg/storage"
@@ -18,6 +18,11 @@ const (
 	namespace = "schema"
 )
 
+type StoredSchemas struct {
+	Schemas       []StoredSchema
+	NextPageToken string
+}
+
 type StoredSchema struct {
 	ID               string                  `json:"id"`
 	Type             schema.VCJSONSchemaType `json:"type"`
@@ -64,15 +69,13 @@ func (s *Storage) GetSchema(ctx context.Context, id string) (*StoredSchema, erro
 }
 
 // ListSchemas attempts to get all stored schemas. It will return those it can even if it has trouble with some.
-func (s *Storage) ListSchemas(ctx context.Context) ([]StoredSchema, error) {
-	gotSchemas, err := s.db.ReadAll(ctx, namespace)
+func (s *Storage) ListSchemas(ctx context.Context, page common.Page) (*StoredSchemas, error) {
+	token, size := page.ToStorageArgs()
+	gotSchemas, nextPageToken, err := s.db.ReadPage(ctx, namespace, token, size)
 	if err != nil {
-		return nil, util.LoggingErrorMsg(err, "could not list schemas")
-	}
-	if len(gotSchemas) == 0 {
-		logrus.Info("no schemas to list")
-		return nil, nil
+		return nil, errors.Wrap(err, "reading page of schemas")
 	}
+
 	stored := make([]StoredSchema, 0, len(gotSchemas))
 	for _, schemaBytes := range gotSchemas {
 		var nextSchema StoredSchema
@@ -82,7 +85,10 @@ func (s *Storage) ListSchemas(ctx context.Context) ([]StoredSchema, error) {
 		}
 		stored = append(stored, nextSchema)
 	}
-	return stored, nil
+	return &StoredSchemas{
+		Schemas:       stored,
+		NextPageToken: nextPageToken,
+	}, nil
 }
 
 func (s *Storage) DeleteSchema(ctx context.Context, id string) error {