chore: test requirements for openapi (#3511)

## About the changes
This enables us to validate the shape of our OpenAPI schemas by defining
specific json-schema rules that will be evaluated against all our open
API schemas.

Because we know there are things we need to improve, we've added a list
of `knownExceptions`. When fixing some of the known exceptions the tests
will force us to remove the exception from the list, that way
contributing to reducing the number of violations to our own rules.

Co-authored-by: Mateusz Kwasniewski <kwasniewski.mateusz@gmail.com>
Co-authored-by: Thomas Heartman <thomas@getunleash.io>

src/lib/openapi/index.test.ts

@@ -17,15 +17,6 @@ test('all schema files should be added to the schemas object', () => {
     expect(expectedSchemaNames.sort()).toEqual(addedSchemaNames.sort());
 });
 
-test('all schema $id attributes should have the expected format', () => {
-    const schemaIds = Object.values(schemas).map((schema) => schema.$id);
-    const schemaIdRegExp = new RegExp(`^#/components/schemas/[a-z][a-zA-Z]+$`);
-
-    schemaIds.forEach((schemaId) => {
-        expect(schemaId).toMatch(schemaIdRegExp);
-    });
-});
-
 test('removeJsonSchemaProps', () => {
     expect(removeJsonSchemaProps({ a: 'b', $id: 'c', components: {} }))
         .toMatchInlineSnapshot(`

src/lib/openapi/index.ts

@@ -147,8 +147,36 @@ import { clientMetricsEnvSchema } from './spec/client-metrics-env-schema';
 import { updateTagsSchema } from './spec/update-tags-schema';
 import { batchStaleSchema } from './spec/batch-stale-schema';
 
+// Schemas must have an $id property on the form "#/components/schemas/mySchema".
+export type SchemaId = typeof schemas[keyof typeof schemas]['$id'];
+
+// Schemas must list all their $refs in `components`, including nested schemas.
+export type SchemaRef = typeof schemas[keyof typeof schemas]['components'];
+
+// JSON schema properties that should not be included in the OpenAPI spec.
+export interface JsonSchemaProps {
+    $id: string;
+    components: object;
+}
+
+type SchemaWithMandatoryFields = Partial<
+    Omit<
+        OpenAPIV3.ReferenceObject | OpenAPIV3.SchemaObject,
+        '$id' | 'components'
+    >
+> &
+    JsonSchemaProps;
+
+interface UnleashSchemas {
+    [name: string]: SchemaWithMandatoryFields;
+}
+
+interface OpenAPIV3DocumentWithServers extends OpenAPIV3.Document {
+    servers: OpenAPIV3.ServerObject[];
+}
+
 // All schemas in `openapi/spec` should be listed here.
-export const schemas = {
+export const schemas: UnleashSchemas = {
     adminFeaturesQuerySchema,
     addonParameterSchema,
     addonSchema,
@@ -291,18 +319,6 @@ export const schemas = {
     importTogglesValidateItemSchema,
 };
 
-// Schemas must have an $id property on the form "#/components/schemas/mySchema".
-export type SchemaId = typeof schemas[keyof typeof schemas]['$id'];
-
-// Schemas must list all their $refs in `components`, including nested schemas.
-export type SchemaRef = typeof schemas[keyof typeof schemas]['components'];
-
-// JSON schema properties that should not be included in the OpenAPI spec.
-export interface JsonSchemaProps {
-    $id: string;
-    components: object;
-}
-
 // Remove JSONSchema keys that would result in an invalid OpenAPI spec.
 export const removeJsonSchemaProps = <T extends JsonSchemaProps>(
     schema: T,
@@ -328,7 +344,7 @@ export const createOpenApiSchema = ({
     unleashUrl,
     baseUriPath,
 }: Pick<IServerOption, 'unleashUrl' | 'baseUriPath'>): Omit<
-    OpenAPIV3.Document,
+    OpenAPIV3DocumentWithServers,
     'paths'
 > => {
     const url = findRootUrl(unleashUrl, baseUriPath);

src/lib/openapi/meta-schema-rules.test.ts

@@ -0,0 +1,202 @@
+import Ajv, { Schema } from 'ajv';
+import { schemas } from '.';
+
+const ajv = new Ajv();
+
+type SchemaNames = keyof typeof schemas;
+type Rule = {
+    name: string;
+    match?: (
+        schemaName: string,
+        schema: typeof schemas[SchemaNames],
+    ) => boolean;
+    metaSchema: Schema;
+    knownExceptions?: string[];
+};
+
+const metaRules: Rule[] = [
+    {
+        name: 'should have a type',
+        metaSchema: {
+            type: 'object',
+            properties: {
+                type: { type: 'string', enum: ['object', 'array'] },
+            },
+            required: ['type'],
+        },
+        knownExceptions: ['dateSchema'],
+    },
+    {
+        name: 'should have an $id with the expected format',
+        metaSchema: {
+            type: 'object',
+            properties: {
+                $id: {
+                    type: 'string',
+                    pattern: '^#/components/schemas/[a-z][a-zA-Z]+$',
+                },
+            },
+            required: ['$id'],
+        },
+    },
+    {
+        name: 'should have a description',
+        metaSchema: {
+            type: 'object',
+            properties: {
+                description: { type: 'string' },
+            },
+            required: ['description'],
+        },
+        knownExceptions: [
+            'adminFeaturesQuerySchema',
+            'addonParameterSchema',
+            'addonSchema',
+            'addonsSchema',
+            'addonTypeSchema',
+            'apiTokenSchema',
+            'apiTokensSchema',
+            'applicationSchema',
+            'applicationsSchema',
+            'batchFeaturesSchema',
+            'batchStaleSchema',
+            'bulkRegistrationSchema',
+            'bulkMetricsSchema',
+            'changePasswordSchema',
+            'clientApplicationSchema',
+            'clientFeatureSchema',
+            'clientFeaturesQuerySchema',
+            'clientFeaturesSchema',
+            'clientMetricsSchema',
+            'clientMetricsEnvSchema',
+            'cloneFeatureSchema',
+            'contextFieldSchema',
+            'contextFieldsSchema',
+            'createApiTokenSchema',
+            'createFeatureSchema',
+            'createFeatureStrategySchema',
+            'createInvitedUserSchema',
+            'createUserSchema',
+            'dateSchema',
+            'edgeTokenSchema',
+            'emailSchema',
+            'environmentsSchema',
+            'eventSchema',
+            'eventsSchema',
+            'exportResultSchema',
+            'exportQuerySchema',
+            'featureEnvironmentMetricsSchema',
+            'featureEventsSchema',
+            'featureMetricsSchema',
+            'featureSchema',
+            'featuresSchema',
+            'featureStrategySchema',
+            'featureStrategySegmentSchema',
+            'featureTagSchema',
+            'featureTypeSchema',
+            'featureTypesSchema',
+            'featureUsageSchema',
+            'featureVariantsSchema',
+            'feedbackSchema',
+            'groupSchema',
+            'groupsSchema',
+            'groupUserModelSchema',
+            'healthCheckSchema',
+            'healthOverviewSchema',
+            'healthReportSchema',
+            'idSchema',
+            'instanceAdminStatsSchema',
+            'legalValueSchema',
+            'loginSchema',
+            'maintenanceSchema',
+            'toggleMaintenanceSchema',
+            'meSchema',
+            'nameSchema',
+            'overrideSchema',
+            'parametersSchema',
+            'passwordSchema',
+            'patchesSchema',
+            'patchSchema',
+            'patSchema',
+            'patsSchema',
+            'permissionSchema',
+            'playgroundSegmentSchema',
+            'playgroundStrategySchema',
+            'profileSchema',
+            'projectEnvironmentSchema',
+            'proxyClientSchema',
+            'proxyFeatureSchema',
+            'proxyFeaturesSchema',
+            'publicSignupTokenCreateSchema',
+            'publicSignupTokenSchema',
+            'publicSignupTokensSchema',
+            'publicSignupTokenUpdateSchema',
+            'pushVariantsSchema',
+            'resetPasswordSchema',
+            'requestsPerSecondSchema',
+            'requestsPerSecondSegmentedSchema',
+            'roleSchema',
+            'segmentSchema',
+            'setStrategySortOrderSchema',
+            'setUiConfigSchema',
+            'sortOrderSchema',
+            'splashSchema',
+            'stateSchema',
+            'strategiesSchema',
+            'strategySchema',
+            'tagsBulkAddSchema',
+            'tagSchema',
+            'tagsSchema',
+            'tagTypeSchema',
+            'tagTypesSchema',
+            'tagWithVersionSchema',
+            'tokenUserSchema',
+            'uiConfigSchema',
+            'updateApiTokenSchema',
+            'updateFeatureSchema',
+            'updateFeatureStrategySchema',
+            'updateTagTypeSchema',
+            'updateUserSchema',
+            'updateTagsSchema',
+            'upsertContextFieldSchema',
+            'upsertSegmentSchema',
+            'upsertStrategySchema',
+            'userSchema',
+            'usersGroupsBaseSchema',
+            'usersSchema',
+            'usersSearchSchema',
+            'validateEdgeTokensSchema',
+            'validatePasswordSchema',
+            'validateTagTypeSchema',
+            'variantSchema',
+            'variantsSchema',
+            'versionSchema',
+            'importTogglesSchema',
+            'importTogglesValidateSchema',
+            'importTogglesValidateItemSchema',
+        ],
+    },
+];
+
+describe.each(metaRules)('OpenAPI schemas $name', (rule) => {
+    const validateMetaSchema = ajv.compile(rule.metaSchema);
+
+    // test all schemas agaisnt the rule
+    Object.entries(schemas).forEach(([schemaName, schema]) => {
+        if (!rule.match || rule.match(schemaName, schema)) {
+            it(`${schemaName}`, () => {
+                validateMetaSchema(schema);
+
+                // note: whenever you resolve an exception please remove it from the list
+                if (rule.knownExceptions?.includes(schemaName)) {
+                    console.warn(
+                        `${schemaName} is a known exception to rule "${rule.name}" that should be fixed`,
+                    );
+                    expect(validateMetaSchema.errors).not.toBeNull();
+                } else {
+                    expect(validateMetaSchema.errors).toBeNull();
+                }
+            });
+        }
+    });
+});