adding open-api-parser

This commit is contained in:
martin 2020-11-09 07:42:24 +01:00
parent ea4297b8c8
commit b0a4cc349b
24 changed files with 350 additions and 221 deletions

View File

@ -0,0 +1,10 @@
## Table of Content
- [Open-API Analyzer](#open-api-analyzer)
- [Used Libraries:](#used-libraries)
# Open-API Analyzer
A Tool, used to parse an `Open-API`-specification into a nope-specification.
## Used Libraries:
- [**openapi-typescript-codegen**](https://www.npmjs.com/package/openapi-typescript-codegen): A library to parse an `Open-API`-specification into typescript.

View File

@ -10,12 +10,12 @@ import { readFile } from 'fs/promises';
import * as handlebars from 'handlebars';
import { join } from 'path';
import * as TJS from "typescript-json-schema";
import { IJsonSchema } from '../../../resources/types/IJSONSchema';
import { IJsonSchema } from '../../types/IJSONSchema';
import { createFile } from '../../helpers/fileMethods';
import { flattenSchema, schemaGetDefinition } from '../../helpers/jsonSchemaMethods';
import { replaceAll } from '../../helpers/stringMethods';
import { INopeDescriptor } from '../../types/nope/descriptor.interface';
import { IFunctionOptions, IPropertyOptions } from '../../types/nope/nopeModule.interface';
import { INopeDescriptor } from '../../types/nope/nopeDescriptor.interface';
import { IFunctionOptions, IParsableDescription, IPropertyOptions } from '../../types/nope/nopeModule.interface';
import { analyzeNopeModules } from './helpers/analyzeNopeModules';
import { getComment } from './helpers/getCommet';
@ -32,11 +32,7 @@ export async function extractDefinitions(){
const modules = await analyzeNopeModules(Object.assign(options));
const ret: {
name: string
properties: IPropertyOptions[],
methods: IFunctionOptions[]
}[] = [];
const ret: IParsableDescription[] = [];
for (const mod of modules.classes){
@ -116,6 +112,7 @@ export type PARAM = {{{simplifiedType}}};
// Store the flattend schema and extracted Schema of the Element.
schemaMapping.inputs.push({
name: param.name,
optional: param.isOptional,
description: getComment(method.authorDescription, param.name),
schema: schemaGetDefinition(flattenSchema(_schema), '#/definitions/PARAM')
});
@ -249,6 +246,5 @@ export type GETTER = {{{getterType}}};
// Store the Class
ret.push(moduleDefinition);
}
return ret;
}

View File

@ -1,7 +1,14 @@
import { parseToOpenAPI } from '../../../parsers/open-api/OpenApiParser';
import { extractDefinitions } from '../analyzeTypescriptFiles';
const main = async function () {
await extractDefinitions();
const res = await extractDefinitions();
for (const m of res){
await parseToOpenAPI(m, {
outputDir: './open-api/backend'
})
}
}
main().catch(e => console.error(e));

View File

@ -7,7 +7,8 @@
*/
import { connect, Socket } from 'socket.io-client';
import { IAvailableInstanceGeneratorsMsg, IAvailableServicesMsg, IAvailableTopicsMsg, ICommunicationInterface, IExternalEventMsg, IRequestTaskMsg, IResponseTaskMsg, ITaskCancelationMsg } from "../types/nope/communication.interface";
import { getCentralNopeLogger } from "../logger/getLogger";
import { IAvailableInstanceGeneratorsMsg, IAvailableServicesMsg, IAvailableTopicsMsg, ICommunicationInterface, IExternalEventMsg, IRequestTaskMsg, IResponseTaskMsg, ITaskCancelationMsg } from "../types/nope/nopeCommunication.interface";
export class IoSocketClient implements ICommunicationInterface {

View File

@ -8,9 +8,9 @@
import { Server } from "http";
import * as io from 'socket.io';
import { getNopeLogger } from "../logger/getLogger";
import { Logger } from "winston";
import { getLogger } from "../logger/getLogger";
import { IAvailableInstanceGeneratorsMsg, IAvailableServicesMsg, IAvailableTopicsMsg, ICommunicationInterface, IExternalEventMsg, IRequestTaskMsg, IResponseTaskMsg, ITaskCancelationMsg } from "../types/nope/communication.interface";
import { IAvailableInstanceGeneratorsMsg, IAvailableServicesMsg, IAvailableTopicsMsg, ICommunicationInterface, IExternalEventMsg, IRequestTaskMsg, IResponseTaskMsg, ITaskCancelationMsg } from "../types/nope/nopeCommunication.interface";
/**
* Wrapper Interface.
@ -35,7 +35,7 @@ export class IoSocketSeverEventEmitter {
this._socket = (io as any)();
}
this._logger = getLogger('info', 'IO-Socket');
this._logger = getNopeLogger('io-socket');
this._socket.listen(port);

View File

@ -0,0 +1,15 @@
export function getPropertyPath(identifier: string, name: string){
return identifier + '.prop.' + name;
}
export function isPropertyPathCorrect(identifier: string, path: string){
return path.startsWith(identifier + '.prop.');
}
export function getMethodPath(identifier: string, name: string){
return identifier + '.method.' + name;
}
export function isMethodPathCorrect(identifier: string, path: string){
return path.startsWith(identifier + '.method.');
}

View File

@ -7,6 +7,8 @@
*/
import { inject, injectable } from 'inversify';
import { observable } from 'rxjs';
import { getMethodPath, getPropertyPath, isMethodPathCorrect, isPropertyPathCorrect } from '../helpers/dispatcherPathes';
import { replaceAll } from "../helpers/stringMethods";
import { DISPATCHER_INSTANCE } from '../symbols/identifiers';
import { INopeDispatcher } from "../types/nope/nopeDispatcher.interface";
@ -162,11 +164,11 @@ export class NopeBaseModule implements INopeModule {
if (typeof options.topic === 'string'){
options.topic = this.identifier + '.prop.' + name;
} else if (typeof options.topic === 'object'){
if (options.topic.subscribe && !options.topic.subscribe.startsWith(this.identifier + '.prop.') ){
options.topic.subscribe = this.identifier + '.prop.' + options.topic.subscribe;
if (options.topic.subscribe && !isPropertyPathCorrect(this.identifier, options.topic.subscribe)){
options.topic.subscribe = getPropertyPath(this.identifier, options.topic.subscribe);
}
if (options.topic.publish && !options.topic.publish.startsWith(this.identifier + '.prop.')){
options.topic.publish = this.identifier + '.prop.' + options.topic.publish;
if (options.topic.publish && !isPropertyPathCorrect(this.identifier, options.topic.publish)){
options.topic.publish = getPropertyPath(this.identifier, options.topic.publish);
}
}
const _observable = await this._dispatcher.registerObservable(observable, options);
@ -192,7 +194,13 @@ export class NopeBaseModule implements INopeModule {
await this.unregisterFunction(name);
// Adapt the Method ID
options.id = this.identifier + '.method.' + name;
if (options.id){
if (!isMethodPathCorrect(this.identifier, options.id)){
options.id = getMethodPath(this.identifier,options.id);
}
} else {
options.id = getMethodPath(this.identifier,name);
}
const _func = await this._dispatcher.registerFunction(func, options);

View File

@ -1,116 +0,0 @@
import { IJsonSchemaTypes } from "../../types/nope/IJSONSchema";
// Symbols for the Property Registery:
const _registeredOpenAPIMethods_ = Symbol('_registeredOpenAPIMethods_');
const _registeredOpenAPIParams_ = Symbol('_registeredOpenAPIParams_');
// Interfaces for the Class
export interface IExportToOpenAPIParameters {
uri?: string
}
export interface IOpenAPIServiceParameters {
// Additional Convert-Function.
convertfunction?: (req) => any[]
// Custom overwritten
parameters?: {
in: 'path' | 'body' | 'query',
// The element provided in the name MUST BE part of the Parameters.
name: string,
required: boolean,
type: IJsonSchemaTypes
}[],
description?: string,
resultDescription?: string,
tags?: string[],
// The Operation ID. Defaultly will be generated on the Base URI and the Method-URI
operationId?: string
}
export interface IExportMethodToOpenAPIParameters extends IOpenAPIServiceParameters {
// If no Parameter is used => Get, otherwise Post.
// Overwrite Manually.
method?: Array<'POST' | 'GET'> | 'POST' | 'GET',
}
export interface IExportPropertyToOpenAPIParameters {
uri?: string,
readonly?: boolean
}
export interface IExportFunctionToOpenAPIParameters {
uri?: string
}
/**
* Decorator used to export a Class API over openAPI
* @param options
*/
export function exportsElementsToOpenAPI(options: IExportToOpenAPIParameters) {
return function <T extends { new(...args: any[]): {} }>(Base: T) {
return class extends Base {
constructor(...args: any[]) {
super(...args);
const _this = this as any;
// extract the Registered Methods of the Class.
const registeredMethods = Base.prototype[_registeredOpenAPIMethods_] as Map<string, IExportMethodToOpenAPIParameters>;
const registeredParams = Base.prototype[_registeredOpenAPIParams_] as Map<string, IExportPropertyToOpenAPIParameters>;
// Online if they are present, iterate over them
if (registeredMethods) {
_this.__OpenAPIRegisterdMethods = (cb: (methodName: string, callback: (...args) => Promise<any>, options: IExportMethodToOpenAPIParameters) => void) => {
registeredMethods.forEach((options, methodName) => {
// Callback the Method
cb(methodName, async (...args) => _this[methodName](...args), options);
});
}
}
// Online if they are present, iterate over them
if (registeredParams) {
_this.__OpenAPIRegisterdParams = (cb: (methodName: string, callback: (...args) => Promise<any>, options: IExportPropertyToOpenAPIParameters) => void) => {
registeredParams.forEach((options, methodName) => {
// Callback the Method
cb(methodName, async (...args) => _this[methodName](...args), options);
});
}
}
}
};
}
}
/**
* Decorator, used to export the Method.
* @param options
*/
export function exportMethodToOpenAPI(options: IExportMethodToOpenAPIParameters) {
return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
target[_registeredOpenAPIMethods_] = target[_registeredOpenAPIMethods_] || new Map<string, IExportMethodToOpenAPIParameters>();
// Here we just add some information that class decorator will use
target[_registeredOpenAPIMethods_].set(propertyKey, options);
};
}
/**
* Decorator, will create a POST and GET api for the Parameter.
* @param options
*/
export function exportPropertyToOpenAPI(options: IExportPropertyToOpenAPIParameters) {
return function (target: any, propertyKey: string, descriptor: PropertyDescriptor) {
target[_registeredOpenAPIParams_] = target[_registeredOpenAPIParams_] || new Map<string, IExportPropertyToOpenAPIParameters>();
// Here we just add some information that class decorator will use
target[_registeredOpenAPIParams_].set(propertyKey, options);
};
}
/**
* Decorator, that will export the Function to a Dispatcher
* @param func The Function
* @param options The Options.
*/
export function exportFunctionToOpenAPI<T>(func: T, options: IExportFunctionToOpenAPIParameters) {
return func;
}

20
lib/parsers/README.md Normal file
View File

@ -0,0 +1,20 @@
## Table of Content
- [Parsers](#parsers)
- [Parsing Nope-Modules](#parsing-nope-modules)
- [Structur of a Parser](#structur-of-a-parser)
# Parsers
This folders contains parsers. This Parsers can be used to generate accessors for NopeModules in different Languages.
## Parsing Nope-Modules
Before parsing could be applied, a **description** of the modules must be provided. This descritption, can than be used to generate accessors for NopeModules in different Languages. How to write such an **description** is detailed under `/lib/modules`.
To use a Parser a `cli`-tool is provided under `/lib/cli`. Additionally you will find a UI-Kit in `/resources`
## Structur of a Parser
To parse a **description** the provided parsers, utilizes [`handlebar`-template](https://handlebarsjs.com/) to parse the description into accessor classes.

View File

@ -0,0 +1,129 @@
import { IParsableDescription } from "../../types/nope/nopeModule.interface";
import { IJsonSchema } from '../../types/IJSONSchema';
import { Logger } from 'winston';
import * as handlebars from 'handlebars';
import { join, relative } from "path";
import { createFile, createPath } from "../../helpers/fileMethods";
import { INopeDescriptorFunctionParameter } from "../../types/nope/nopeDescriptor.interface";
import { deepClone } from "../../helpers/objectMethods";
import { readFile } from "fs/promises";
/**
* Helper function to merge the Parameters into 1 description.
* @param elements
*/
function _unifySchema(elements: INopeDescriptorFunctionParameter[]){
const ret: IJsonSchema = {
type: 'object',
properties: {},
required: []
}
for (const item of elements){
// Assign the Schema
ret[item.name] = item.schema;
// If the Element is Optional,
if (!item.optional){
ret.required.push(item.name)
}
}
return ret;
}
/**
* Function, to parse a description to an Open-API Element.
* @param description
* @param options
*/
export async function parseToOpenAPI(description: IParsableDescription, options: {
outputDir: string,
logger?: Logger,
}){
const _description = deepClone(description);
// load the Template.
const template = await readFile(join(process.cwd(), 'lib' , 'parsers', 'open-api' ,'templates', 'openApiSchema.handlebars'), {
encoding: 'utf-8'
})
// Renderfuncting
const renderAPI = handlebars.compile(template);
await createPath(join(options.outputDir));
// Now iterate over the Methods of the Module and find parsable Methods.
for (const [idx, method] of _description.methods.entries()){
// Test if the Method contains some functions in the input / Output:
const parsedInputs = JSON.stringify(method.schema.inputs);
const parsedOutput = JSON.stringify(method.schema.outputs);
if (!parsedInputs.includes('"function"') && !parsedOutput.includes('"function"')){
// The Method should be parseable.
// 1. Specify the Mode (No Params = GET, else POST)
(method as any).mode = method.schema.inputs.length > 0 ? 'POST' : 'GET';
// Now adapt the Schema of the Method.
// Iterate over the Inputs, add a Description if not provided
// And determine whehter the Parameters are required or not.
method.schema.inputs = method.schema.inputs.map(param => {
// Provide a Description. (If not provided)
param.description = param.description || 'Not provided';
// Open API uses "required" instead of optional => invert
param.optional = !param.optional;
// Parse the Schema in here.
(param as any).parsedSchema = JSON.stringify(param.schema);
return param;
});
// Make shure the Return type isnt an array
method.schema.outputs = Array.isArray(method.schema.outputs) ?
_unifySchema(method.schema.outputs) :
method.schema.outputs;
// Add an Description to the result.
(method as any).resultDescription = method.schema.outputs.description || 'Not Provided';
// And add a Schema for the Return type.
(method as any).parsedOutput = JSON.stringify(method.schema.outputs);
// Determine the Filename.
const fileDir = join(options.outputDir, _description.name, '{instance}' ,'methods');
const fileName = join(fileDir, method.id+'.ts');
// Determine the Import Pathes.
const imports = [
{
dir: join(process.cwd(), 'lib', 'types', 'nope'),
fileName: 'nopeDispatcher.interface',
name: 'pathOfDispatcher'
},
{
dir: join(process.cwd(), 'lib', 'helpers'),
fileName: 'dispatcherPathes',
name: 'pathOfHelper'
}
]
for (const imp of imports){
const relativDir = relative(fileDir, imp.dir);
(method as any)[imp.name] = join(relativDir, imp.fileName);
}
// Write down the Schema:
await createFile(
// Generate the Path.
fileName,
renderAPI(method)
);
if (options.logger) {
options.logger.info('Generated -> ' + fileName);
}
}
}
}

View File

@ -0,0 +1,26 @@
## Table of Content
- [Description](#description)
- [OpenAPI](#openapi)
- [Limitations](#limitations)
- [Generated Files.](#generated-files)
- [Implementaiton](#implementaiton)
# Description
This Parser translates a description into an `Open-API`-accessor, which can then be used as default `REST`-API of the Nope-Module. This is espacially useful if you consider using modules in other applications.
## OpenAPI
Open-API is a broadly adopted industry standard for describing modern APIs. You can read the full specification here: http://spec.openapis.org/oas/v3.0.3
Taken from openapis.org:
> The goal of the OAI specification is to define a standard, language-agnostic interface to REST APIs which allows both humans and computers to discover and understand the capabilities of the service without access to source code, documentation, or through network traffic inspection. When properly defined, a consumer can understand and interact with the remote service with a minimal amount of implementation logic. Similar to what interfaces have done for lower-level programming
## Limitations
Based on the Structure of the `REST`-protocol, functions and services which uses callbacks as parameters can not be parsed to an Open-API-accessor. They will be skipped.
## Generated Files.
The parser creates the following items:
- an `accessor`-class for Open-API
## Implementaiton
The Nope-`Open-API` Hoster heavily uses `express-openapi`. For Details checkout the following page: https://www.npmjs.com/package/express-openapi

View File

@ -0,0 +1,72 @@
// Automatic Genearted File for NopeModule
// To update run `npm run build:backend`
import { INopeDispatcher } from "{{pathOfDispatcher}}";
import { getMethodPath } from "{{pathOfHelper}}"
import { Operation } from "express-openapi";
export default function (_dispatcher: INopeDispatcher) {
let operations = {
{{mode}},
parameters: [
{
in: 'path',
name: 'instance',
description: 'Instance of the Type which should execute the Task'
required: true,
type: 'string'
}
],
};
// Function to Parse the Inputs
const parseParams = (req) => {
return [{{#each schema.inputs}}req.body.{{name}}{{#unless @last}}, {{/unless}}{{/each}}]
}
// Define the Action.
async function {{mode}}(req, res, next) {
try {
// Transform the Operation to the Callback of the Dispatcher.
const result = await _dispatcher.performCall(getMethodPath(req.params.instance, '{{id}}'), parseParams(req))
// Finish the Task.
res.status(200).json(result)
} catch (e) {
// An error Occourd =>
res.status(500).json(e)
}
}
// Define the apiDoc for this specific Funtion
{{mode}}.apiDoc = {
{{#if methodDescription}}summary: '{{methodDescription}}',{{/if}}
{{#if operationId}}operationId: '{{operationId}}',{{/if}}
parameters: [
{{#each schema.inputs}}
{
name: '{{name}}',
in: "body",
description: '{{description}}',
required: {{optional}},
schema: {{{parsedSchema}}}
}{{#unless @last}},{{/unless}}
{{/each}}
],
responses: {
200: {
{{#if resultDescription}}description: '{{resultDescription}}',{{/if}}
schema: {{{parsedOutput}}}
},
default: {
description: 'An error occurred',
schema: {
additionalProperties: true
}
}
}
} as Operation;
return operations;
}

View File

@ -0,0 +1,10 @@
## Table of Content
- [Description](#description)
- [Generated Files.](#generated-files)
# Description
This Parser translates a description into a `python`-class, which can then be used as wrapper to access a remote Tool. This is espacially useful if you consider using modules of different languages like `typescript` inside of your `python` code.
## Generated Files.
The parser creates the following items:
- an `accessor`-class

View File

@ -0,0 +1,12 @@
## Table of Content
- [Description](#description)
- [Generated Files.](#generated-files)
# Description
This Parser translates a description into a `typescript`-class, which can then be used as wrapper to access a remote Tool. This is espacially useful if you consider using modules of different languages like `python` inside of your `typescript` code.
## Generated Files.
The parser creates the following items:
- `interfaces` for accessing the module via a generic-module
- an `extended` **Generic-Nope-Module** for simpler access
- optionally an **access-class** which contains all accessors and a Nope-Dispatcher

View File

@ -1,68 +0,0 @@
// Automatic Genearted File for Backendclass "{{className}}"
// To update run `npm run build:backend`
import { nopeDispatcher } from "../nopeDispatcher"
import { Operation } from "express-openapi";
export default function (_dispatcher: nopeDispatcher) {
let operations = {
{{method}}
};
// Function to Parse the Inputs
const parseParams = (req) => {
return [{{#each params}}req.body.{{name}}{{#unless @last}}, {{/unless}}{{/each}}]
}
// Define the Action.
async function {{method}}(req, res, next) {
// Transform the Operation to the Callback of the Dispatcher.
const result = await _dispatcher.performCall<{{{returnType.simplifiedSubType}}}>('{{baseUri}}/{{methodUri}}', parseParams(req))
// Finish the Task.
res.status(200).json(result)
}
// Define the apiDoc for this specific Funtion
POST.apiDoc = {
{{#if methodDescription}}summary: '{{methodDescription}}',{{/if}}
{{#if operationId}}operationId: '{{operationId}}'{{/if}},
parameters: [
{{#if useDefaultParameters}}
{
name: "body",
in: "body",
description: "Body of the Message",
required: true,
schema: {{{inputSchema}}}
}
{{/if}}
{{#if useCustomParameters}}
{{useCustomParameters}}
{{/if}}
],
responses: {
{{#if hasReturnType}}
200: {
{{#if resultDescription}}description: '{{resultDescription}}',{{/if}}
{{#unless resultDescription}}description: 'Not Provided', {{/unless}}
schema: {{{outputSchema}}}
},
{{/if}}
{{#unless hasReturnType}}
200: {
{{#if resultDescription}}description: '{{resultDescription}}',{{/if}}
{{#unless resultDescription}}description: 'Not Provided', {{/unless}}
},
{{/unless}}
default: {
description: 'An error occurred',
schema: {
additionalProperties: true
}
}
}
} as Operation;
return operations;
}

View File

@ -171,16 +171,14 @@ export interface INopeDescriptor {
*/
not?: INopeDescriptor;
inputs?: Array<{
name: string,
description?: string,
schema: INopeDescriptor
}>;
inputs?: Array<INopeDescriptorFunctionParameter>;
outputs?: Array<{
name: string,
description?: string,
schema: INopeDescriptor
}> | INopeDescriptor
outputs?: Array<INopeDescriptorFunctionParameter> | INopeDescriptor
}
export interface INopeDescriptorFunctionParameter {
name: string,
description?: string,
optional?: boolean,
schema: INopeDescriptor
}

View File

@ -6,7 +6,7 @@
* @desc [description]
*/
import { ICallOptions, IExternalEventMsg, IInstanceCreationMsg } from "./communication.interface";
import { ICallOptions, IExternalEventMsg, IInstanceCreationMsg } from "./nopeCommunication.interface";
import { IFunctionOptions, INopeModule, INopeModuleDescription, IPropertyOptions } from "./nopeModule.interface";
import { INopeObservable, INopeObserver, INopeSubscriptionOptions, IObservableCallback, IPipe } from "./nopeObservable.interface";
import { INopePromise } from "./nopePromise.interface";

View File

@ -6,8 +6,8 @@
* @desc Defintion of a generic Module.
*/
import { ICallOptions, IExternalEventMsg } from "./communication.interface";
import { INopeDescriptor } from "./descriptor.interface";
import { ICallOptions, IExternalEventMsg } from "./nopeCommunication.interface";
import { INopeDescriptor } from "./nopeDescriptor.interface";
import { INopeObservable, IPipe } from "./nopeObservable.interface";
/**
@ -278,3 +278,12 @@ export interface IFunctionOptions extends Partial<ICallOptions> {
*/
isDynamic?: boolean;
}
/**
* Parsable Description of a Module
*/
export interface IParsableDescription {
name: string
properties: IPropertyOptions[],
methods: IFunctionOptions[]
}

View File

@ -1,6 +1,6 @@
import React from 'react';
import Form from "@rjsf/bootstrap-4";
import { IJsonSchema } from '../../types/IJSONSchema'
import { IJsonSchema } from '../../../lib/types/IJSONSchema'
import { Button } from 'react-bootstrap';
export interface DynamicFormProps<T> {

View File

@ -29,7 +29,7 @@ import { IVisjsOptions } from './interfaces/IVisjsOptions';
import Selection, { SelectionProps } from '../layout/selection';
import DynamicRenderer from '../dynamic/dynamicRenderer';
import TabEntry, { ITabProps } from '../layout/tabs';
import { IJsonSchema } from '../../types/IJSONSchema';
import { IJsonSchema } from '../../../lib/types/IJSONSchema';
import { ITabEntry } from '../layout/interfaces/ITabEntry';
import { parseWithFunctions, stringifyWithFunctions } from '../../../lib/helpers/jsonMethods';
import { readDataFromClipboard, writeToClipboard } from '../helpers/clipboard';

View File

@ -1,4 +1,4 @@
import { IJsonSchema } from '../../../types/IJSONSchema';
import { IJsonSchema } from '../../../../lib/types/IJSONSchema';
export interface IComplexTemplate<D> {
type: 'complex'

View File

@ -1,4 +1,4 @@
import { getLogger } from "../lib/logger/getLogger";
import { getNopeLogger } from "../lib/logger/getLogger";
import { startBackend } from "./startBackend";
startBackend({ port: 3001, logger: getLogger('debug') });
startBackend({ port: 3001, logger: getNopeLogger('open-api-server','debug') });