Stop using rest-api-spec (#5831)

* Remove reAddAvailability function

* Remove jsonSpec as a global variable

This makes it easier to spot functions that use it.

* Stop reading endpoints from jsonSpec

* Stop reading docUrl, docTag, extDocUrl, requestBodyRequired and availability from rest-api-spec

* Stop reading description and urls from rest-api-spec

* Stop validating rest-api-spec

* Stop using jsonSpec in compilation steps

* Stop using rest-api-spec altogether

* Fix compiler tests

(cherry picked from commit 18a4c3f)

# Conflicts:
#	output/schema/validation-errors.json

Author: pquentin

compiler/src/compiler.ts

@@ -21,15 +21,10 @@ import { writeFile, mkdir } from 'fs/promises'
 import { join } from 'path'
 import stringify from 'safe-stable-stringify'
 import { Model } from './model/metamodel'
-import {
-  compileEndpoints,
-  compileSpecification,
-  reAddAvailability
-} from './model/build-model'
-import buildJsonSpec, { JsonSpec } from './model/json-spec'
+import { compileSpecification } from './model/build-model'
 import { ValidationErrors } from './validation-errors'
 
-type StepFunction = (model: Model, restSpec: Map<string, JsonSpec>, errors: ValidationErrors) => Promise<Model>
+type StepFunction = (model: Model, errors: ValidationErrors) => Promise<Model>
 
 /**
  * The main job of the compiler is to generate the Model and write it on disk.
@@ -42,7 +37,6 @@ type StepFunction = (model: Model, restSpec: Map<string, JsonSpec>, errors: Vali
 export default class Compiler {
   queue: StepFunction[]
   model: Model
-  jsonSpec: Map<string, JsonSpec>
   errors: ValidationErrors
   specsFolder: string
   outputFolder: string
@@ -55,16 +49,13 @@ export default class Compiler {
   }
 
   generateModel (): this {
-    this.jsonSpec = buildJsonSpec()
-    const endpoints = compileEndpoints()
-    this.model = compileSpecification(endpoints, this.specsFolder, this.outputFolder)
-    this.model = reAddAvailability(this.model) // resync availability information based on json spec if typescript has none.
+    this.model = compileSpecification(this.specsFolder, this.outputFolder)
     return this
   }
 
   async write (): Promise<void> {
     for (const step of this.queue) {
-      this.model = await step(this.model, this.jsonSpec, this.errors)
+      this.model = await step(this.model, this.errors)
     }
 
     const customStringify = stringify.configure(

compiler/src/index.ts

@@ -21,9 +21,7 @@ import { readFileSync, existsSync, lstatSync } from 'fs'
 import { join, resolve } from 'path'
 import { argv } from 'zx'
 import Compiler from './compiler'
-import validateRestSpec from './steps/validate-rest-spec'
 import addInfo from './steps/add-info'
-import addDescription from './steps/add-description'
 import validateModel from './steps/validate-model'
 import readDefinitionValidation from './steps/read-definition-validation'
 import addDeprecation from './steps/add-deprecation'
@@ -73,8 +71,6 @@ compiler
   .step(addInfo)
   .step(addDeprecation)
   .step(readDefinitionValidation)
-  .step(validateRestSpec)
-  .step(addDescription)
   .step(validateModel)
   .step(addExamples)
   .write()

compiler/src/model/build-model.ts

@@ -32,7 +32,6 @@ import {
   TypeAliasDeclaration
 } from 'ts-morph'
 import * as model from './metamodel'
-import buildJsonSpec from './json-spec'
 import {
   assert,
   customTypes,
@@ -57,62 +56,10 @@ import {
   mediaTypeToStringArray
 } from './utils'
 
-const jsonSpec = buildJsonSpec()
-
-export function reAddAvailability (model: model.Model): model.Model {
-  for (const [api, spec] of jsonSpec.entries()) {
-    for (const endpoint of model.endpoints) {
-      if (endpoint.name === api) {
-        if ((spec.stability != null || spec.visibility != null) && (endpoint.availability.stack === undefined && endpoint.availability.serverless === undefined)) {
-          endpoint.availability = {
-            stack: {
-              stability: spec.stability,
-              visibility: spec.visibility
-            }
-          }
-        }
-      }
-    }
-  }
-  return model
-}
-
-export function compileEndpoints (): Record<string, model.Endpoint> {
-  // Create endpoints and merge them with
-  // the recorded mappings if present.
-  const map = {}
-  for (const [api, spec] of jsonSpec.entries()) {
-    map[api] = {
-      name: api,
-      description: spec.documentation.description,
-      docUrl: spec.documentation.url,
-      docTag: spec.docTag,
-      extDocUrl: spec.externalDocs?.url,
-      // Setting these values by default should be removed
-      // when we no longer use rest-api-spec stubs as the
-      // source of truth for stability/visibility.
-      availability: {},
-      request: null,
-      requestBodyRequired: Boolean(spec.body?.required),
-      response: null,
-      urls: spec.url.paths.map(path => {
-        return {
-          path: path.path,
-          methods: path.methods,
-          ...(path.deprecated != null && { deprecation: path.deprecated })
-        }
-      })
-    }
-    if (typeof spec.feature_flag === 'string') {
-      map[api].availability.stack = { featureFlag: spec.feature_flag }
-    }
-  }
-  return map
-}
-
-export function compileSpecification (endpointMappings: Record<string, model.Endpoint>, specsFolder: string, outputFolder: string): model.Model {
+export function compileSpecification (specsFolder: string, outputFolder: string): model.Model {
   const tsConfigFilePath = join(specsFolder, 'tsconfig.json')
   const project = new Project({ tsConfigFilePath })
+  const endpointMappings: Record<string, model.Endpoint> = {}
 
   verifyUniqueness(project)
 
@@ -151,9 +98,6 @@ export function compileSpecification (endpointMappings: Record<string, model.End
     definedButNeverUsed.join('\n'),
     { encoding: 'utf8', flag: 'w' }
   )
-  for (const api of jsonSpec.keys()) {
-    model.endpoints.push(endpointMappings[api])
-  }
 
   // Visit all class, interface, enum and type alias definitions
   for (const declaration of declarations.classes) {
@@ -175,6 +119,11 @@ export function compileSpecification (endpointMappings: Record<string, model.End
   // Sort the types in alphabetical order
   sortTypeDefinitions(model.types)
 
+  const sortedEndpointKeys = Object.keys(endpointMappings).sort()
+  for (const key of sortedEndpointKeys) {
+    model.endpoints.push(endpointMappings[key])
+  }
+
   return model
 }
 
@@ -225,6 +174,10 @@ function compileClassOrInterfaceDeclaration (declaration: ClassDeclaration | Int
         throw new Error(`Cannot find url template for ${namespace}, very likely the specification folder does not follow the rest-api-spec`)
       }
 
+      if (type.description !== '' && type.description !== null && type.description !== undefined) {
+        mapping.description = type.description
+      }
+
       let pathMember: Node | null = null
       let bodyProperties: model.Property[] = []
       let bodyValue: model.ValueOf | null = null
@@ -288,7 +241,7 @@ function compileClassOrInterfaceDeclaration (declaration: ClassDeclaration | Int
         assert(
           pathMember as Node,
           urlTemplateParams.includes(part.name),
-          `The property '${part.name}' does not exist in the rest-api-spec ${namespace} url template`
+          `The property '${part.name}' does not exist in the url template`
         )
         if (type.query.map(p => p.name).includes(part.name)) {
           const queryType = type.query.find(property => property != null && property.name === part.name) as model.Property
@@ -635,7 +588,7 @@ function visitRequestOrResponseProperty (member: PropertyDeclaration | PropertyS
  * ```
  * urls: [
  *   {
- *     /** @deprecated 1.2.3 Use something else
+ *     /** \@deprecated 1.2.3 Use something else
  *     path: '/some/path',
  *     methods: ["GET", "POST"]
  *   }

compiler/src/model/json-spec.ts

@@ -35,7 +35,6 @@ import {
   Node,
   Project
 } from 'ts-morph'
-import { closest } from 'fastest-levenshtein'
 import semver from 'semver'
 import chalk from 'chalk'
 import * as model from './metamodel'
@@ -629,6 +628,23 @@ function setTags<TType extends model.BaseType | model.Property | model.EnumMembe
   }
 }
 
+export function updateEndpoints (mappings: Record<string, model.Endpoint>, name: string): model.Endpoint {
+  mappings[name] = {
+    name: name,
+    // @ts-expect-error TODO
+    description: null,
+    // @ts-expect-error TODO
+    docUrl: null,
+    request: null,
+    requestBodyRequired: false,
+    response: null,
+    urls: []
+  }
+  mappings[name].availability = {}
+
+  return mappings[name]
+}
+
 /** Lifts jsDoc type annotations to request properties */
 export function hoistRequestAnnotations (
   request: model.Request, jsDocs: JSDoc[], mappings: Record<string, model.Endpoint>, response: model.TypeName | null
@@ -651,9 +667,7 @@ export function hoistRequestAnnotations (
   assert(jsDocs, apiName !== '' && apiName !== null && apiName !== undefined,
     `Request ${request.name.name} does not declare the @rest_spec_name to link back to`)
 
-  const endpoint = mappings[apiName]
-  assert(jsDocs, endpoint != null, `The api '${apiName}' does not exists, did you mean '${closest(apiName, Object.keys(mappings))}'?`)
-
+  const endpoint = updateEndpoints(mappings, apiName)
   endpoint.request = request.name
   endpoint.response = response
 

compiler/src/model/utils.ts

@@ -18,12 +18,11 @@
  */
 
 import * as model from '../model/metamodel'
-import { JsonSpec } from '../model/json-spec'
 
 /**
  * Populates the `deprecation` field for endpoints from the value of the corresponding request definition.
  */
-export default async function addContentType (model: model.Model, jsonSpec: Map<string, JsonSpec>): Promise<model.Model> {
+export default async function addContentType (model: model.Model): Promise<model.Model> {
   for (const endpoint of model.endpoints) {
     if (endpoint.deprecation != null) {
       continue