feat: Cursor based hateoas (#5230)

src/lib/features/feature-search/feature-search-controller.ts

@@ -1,4 +1,4 @@
-import { Response } from 'express';
+import { Response, Request } from 'express';
 import Controller from '../../routes/controller';
 import { FeatureSearchService, OpenApiService } from '../../services';
 import {
@@ -16,6 +16,7 @@ import {
     FeatureSearchQueryParameters,
     featureSearchQueryParameters,
 } from '../../openapi/spec/feature-search-query-parameters';
+import { nextLink } from './next-link';
 
 const PATH = '/features';
 
@@ -79,7 +80,7 @@ export default class FeatureSearchController extends Controller {
                 tag,
                 status,
                 cursor,
-                limit = 50,
+                limit = '50',
             } = req.query;
             const userId = req.user.id;
             const normalizedTag = tag
@@ -92,17 +93,21 @@ export default class FeatureSearchController extends Controller {
                         tag.length === 2 &&
                         ['enabled', 'disabled'].includes(tag[1]),
                 );
-            const normalizedLimit = limit > 0 && limit <= 50 ? limit : 50;
-            const features = await this.featureSearchService.search({
-                query,
-                projectId,
-                type,
-                userId,
-                tag: normalizedTag,
-                status: normalizedStatus,
-                cursor,
-                limit: normalizedLimit,
-            });
+            const normalizedLimit =
+                Number(limit) > 0 && Number(limit) <= 50 ? Number(limit) : 50;
+            const { features, nextCursor } =
+                await this.featureSearchService.search({
+                    query,
+                    projectId,
+                    type,
+                    userId,
+                    tag: normalizedTag,
+                    status: normalizedStatus,
+                    cursor,
+                    limit: normalizedLimit,
+                });
+
+            res.header('Link', nextLink(req, nextCursor));
             res.json({ features });
         } else {
             throw new InvalidOperationError(

src/lib/features/feature-search/feature-search-service.ts

@@ -3,6 +3,7 @@ import {
     IFeatureStrategiesStore,
     IUnleashConfig,
     IUnleashStores,
+    serializeDates,
 } from '../../types';
 import { IFeatureSearchParams } from '../feature-toggle/types/feature-toggle-strategies-store-type';
 
@@ -20,10 +21,24 @@ export class FeatureSearchService {
     }
 
     async search(params: IFeatureSearchParams) {
-        const features = await this.featureStrategiesStore.searchFeatures(
-            params,
-        );
+        // fetch one more item than needed to get a cursor of the next item
+        const features = await this.featureStrategiesStore.searchFeatures({
+            ...params,
+            limit: params.limit + 1,
+        });
 
-        return features;
+        const nextCursor =
+            features.length > params.limit
+                ? features[features.length - 1].createdAt.toJSON()
+                : undefined;
+
+        // do not return the items with the next cursor
+        return {
+            features:
+                features.length > params.limit
+                    ? features.slice(0, -1)
+                    : features,
+            nextCursor,
+        };
     }
 }

src/lib/features/feature-search/feature.search.e2e.test.ts

@@ -48,7 +48,7 @@ const searchFeaturesWithCursor = async (
         query = '',
         projectId = 'default',
         cursor = '',
-        limit = 10,
+        limit = '10',
     }: FeatureSearchQueryParameters,
     expectedCode = 200,
 ) => {
@@ -59,6 +59,10 @@ const searchFeaturesWithCursor = async (
         .expect(expectedCode);
 };
 
+const getPage = async (url: string, expectedCode = 200) => {
+    return app.request.get(url).expect(expectedCode);
+};
+
 const filterFeaturesByType = async (types: string[], expectedCode = 200) => {
     const typeParams = types.map((type) => `type[]=${type}`).join('&');
     return app.request
@@ -107,38 +111,26 @@ test('should paginate with cursor', async () => {
     await app.createFeature('my_feature_c');
     await app.createFeature('my_feature_d');
 
-    const { body: firstPage } = await searchFeaturesWithCursor({
-        query: 'feature',
-        cursor: '',
-        limit: 2,
-    });
-    const nextCursor =
-        firstPage.features[firstPage.features.length - 1].createdAt;
+    const { body: firstPage, headers: firstHeaders } =
+        await searchFeaturesWithCursor({
+            query: 'feature',
+            cursor: '',
+            limit: '2',
+        });
 
     expect(firstPage).toMatchObject({
         features: [{ name: 'my_feature_a' }, { name: 'my_feature_b' }],
     });
 
-    const { body: secondPage } = await searchFeaturesWithCursor({
-        query: 'feature',
-        cursor: nextCursor,
-        limit: 2,
-    });
+    const { body: secondPage, headers: secondHeaders } = await getPage(
+        firstHeaders.link,
+    );
 
     expect(secondPage).toMatchObject({
         features: [{ name: 'my_feature_c' }, { name: 'my_feature_d' }],
     });
-    const lastCursor =
-        secondPage.features[secondPage.features.length - 1].createdAt;
 
-    const { body: lastPage } = await searchFeaturesWithCursor({
-        query: 'feature',
-        cursor: lastCursor,
-        limit: 2,
-    });
-    expect(lastPage).toMatchObject({
-        features: [],
-    });
+    expect(secondHeaders.link).toBe('');
 });
 
 test('should filter features by type', async () => {

src/lib/features/feature-search/next-link.test.ts

@@ -0,0 +1,55 @@
+import { nextLink } from './next-link';
+import { Request } from 'express';
+
+describe('nextLink', () => {
+    it('should generate the correct next link with a cursor', () => {
+        const req = {
+            baseUrl: '/api/events',
+            path: '/',
+            query: { page: '2', limit: '10' },
+        } as Pick<Request, 'baseUrl' | 'path' | 'query'>;
+
+        const cursor = 'abc123';
+        const result = nextLink(req, cursor);
+
+        expect(result).toBe('/api/events/?page=2&limit=10&cursor=abc123');
+    });
+
+    it('should generate the correct next link without a cursor', () => {
+        const req = {
+            baseUrl: '/api/events',
+            path: '/',
+            query: { page: '2', limit: '10' },
+        } as Pick<Request, 'baseUrl' | 'path' | 'query'>;
+
+        const result = nextLink(req);
+
+        expect(result).toBe('');
+    });
+
+    it('should exclude existing cursor from query parameters', () => {
+        const req = {
+            baseUrl: '/api/events',
+            path: '/',
+            query: { page: '2', limit: '10', cursor: 'oldCursor' },
+        } as Pick<Request, 'baseUrl' | 'path' | 'query'>;
+
+        const cursor = 'newCursor';
+        const result = nextLink(req, cursor);
+
+        expect(result).toBe('/api/events/?page=2&limit=10&cursor=newCursor');
+    });
+
+    it('should handle empty query parameters correctly', () => {
+        const req = {
+            baseUrl: '/api/events',
+            path: '/',
+            query: {},
+        } as Pick<Request, 'baseUrl' | 'path' | 'query'>;
+
+        const cursor = 'abc123';
+        const result = nextLink(req, cursor);
+
+        expect(result).toBe('/api/events/?cursor=abc123');
+    });
+});

src/lib/features/feature-search/next-link.ts

@@ -0,0 +1,18 @@
+import { Request } from 'express';
+
+export function nextLink(
+    req: Pick<Request, 'baseUrl' | 'path' | 'query'>,
+    cursor?: string,
+): string {
+    if (!cursor) {
+        return '';
+    }
+
+    const url = `${req.baseUrl}${req.path}?`;
+
+    const params = new URLSearchParams(req.query as Record<string, string>);
+
+    params.set('cursor', cursor);
+
+    return `${url}${params.toString()}`;
+}

src/lib/features/feature-toggle/feature-toggle-strategies-store.ts

@@ -533,7 +533,7 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore {
         cursor,
         limit,
     }: IFeatureSearchParams): Promise<IFeatureOverview[]> {
-        let query = this.db('features');
+        let query = this.db('features').limit(limit);
         if (projectId) {
             query = query.where({ project: projectId });
         }
@@ -582,20 +582,10 @@ class FeatureStrategiesStore implements IFeatureStrategiesStore {
             });
         }
 
-        // workaround for imprecise timestamp that was including the cursor itself
-        const addMillisecond = (cursor: string) =>
-            format(
-                addMilliseconds(parseISO(cursor), 1),
-                "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'",
-            );
         if (cursor) {
-            query = query.where(
-                'features.created_at',
-                '>',
-                addMillisecond(cursor),
-            );
+            query = query.where('features.created_at', '>=', cursor);
         }
-        query = query.orderBy('features.created_at', 'asc').limit(limit);
+        query = query.orderBy('features.created_at', 'asc');
 
         query = query
             .modify(FeatureToggleStore.filterByArchived, false)

src/lib/openapi/spec/feature-search-query-parameters.ts

@@ -61,17 +61,17 @@ export const featureSearchQueryParameters = [
         name: 'cursor',
         schema: {
             type: 'string',
-            example: '1',
+            example: '2023-10-31T09:21:04.056Z',
         },
         description:
-            'The last feature id the client has seen. Used for cursor-based pagination.',
+            'The next feature created at date the client has not seen. Used for cursor-based pagination. Empty if starting from the beginning.',
         in: 'query',
     },
     {
         name: 'limit',
         schema: {
-            type: 'number',
-            example: 10,
+            type: 'string',
+            example: '10',
         },
         description:
             'The number of results to return in a page. By default it is set to 50',