From 7f658592589f7d15cf5bf135d4ed64d72b2d7b3a Mon Sep 17 00:00:00 2001 From: Martin Karkowski Date: Sat, 23 Jul 2022 07:39:56 +0200 Subject: [PATCH] Updating helper --- py-helpers/prepare_code/js/debug.py | 11 +- py-helpers/prepare_code/js/grammar.js.lark | 27 +++-- py-helpers/prepare_code/js/transformer.py | 128 +++++++++++++++++---- py-helpers/prepare_code/main.py | 73 +++++++----- py-helpers/prepare_code/ts/debug.py | 2 +- py-helpers/prepare_code/ts/transformer.py | 4 +- tsconfig.py.json | 3 +- 7 files changed, 171 insertions(+), 77 deletions(-) diff --git a/py-helpers/prepare_code/js/debug.py b/py-helpers/prepare_code/js/debug.py index 2d78582..f6ebfc2 100644 --- a/py-helpers/prepare_code/js/debug.py +++ b/py-helpers/prepare_code/js/debug.py @@ -4,6 +4,7 @@ from pathlib import Path from parser import get_parser from transformer import transform +from prepare_code import post_process MAX_CPU = mp.cpu_count() @@ -37,23 +38,27 @@ def parse(parser, path_to_file): True ) + code = post_process(code) + print(code) + return + except Exception as err: raise err if __name__ == "__main__": - # "C:\Users\m.karkowski\Documents\00-Repos\nope-backend\dist-py\helpers\runtimeMethods.js" + # "dist-py\dispatcher\baseServices\connectivy.js" input_path = Path(__file__).parent.joinpath( '..', "..", "dist-py", "helpers") path_to_file = os.path.join(input_path, "jsonSchemaMethods.js") input_path = Path(__file__).parent.joinpath( - '..', "..", "..", "dist-py", "helpers") - path_to_file = os.path.join(input_path, "runtimeMethods.js") + '..', "..", "..", "dist-py", "dispatcher", "baseServices") + path_to_file = os.path.join(input_path, "connectivy.js") parse( get_parser(), diff --git a/py-helpers/prepare_code/js/grammar.js.lark b/py-helpers/prepare_code/js/grammar.js.lark index 84bc071..ec96578 100644 --- a/py-helpers/prepare_code/js/grammar.js.lark +++ b/py-helpers/prepare_code/js/grammar.js.lark @@ -22,11 +22,11 @@ STR: /(`.*?`)|(".*?")|(\'.*?\')/ %ignore COMMENT %ignore WS // Ignore Exports -%ignore "export" "default" /w+/ ";" -%ignore "export" "*" "from" STR ";" -%ignore "export" "*" "as" /w+/ "from" STR ";" -%ignore "export" /(?<=export)(.|\n)+?(?=})/ "}" "from" STR ";" -%ignore "export" /(?<=export)(.|\n)+?(?=})/ "}" ";" +// %ignore "export" "default" /w+/ ";" +// %ignore "export" "*" "from" STR ";" +// %ignore "export" "*" "as" /w+/ "from" STR ";" +// %ignore "export" /(?<=export)(.|\n)+?(?=})/ "}" "from" STR ";" +// %ignore "export" /(?<=export)(.|\n)+?(?=})/ "}" ";" // -------------------------------------------------------------------------- @@ -78,7 +78,6 @@ ret_expr_with_terminator: ret_expr terminator return_statement: "return" [ret_expr] statement: ret_expr_with_terminator - | declare | declare_var | declare_var_not_initialized | declare_var_descructed @@ -148,13 +147,13 @@ await_stmt: "await" ret_expr reg_ex: "/" /(?<=\/).+(?=\/\w)/ "/" [/\w/] -?sum: product +sum: product | sum "+" product -> add | sum "-" product -> sub | accessor "+=" product -> assigned_add | accessor "-=" product -> assigned_sub -?product: atom +product: atom | product "*" atom -> mult | product "/" atom -> div | accessor "*=" atom -> assigned_mult @@ -176,9 +175,9 @@ boolean_operator: ">" -> bool_op_gt boolean_input: ret_expr -?atom: ret_expr +atom: ret_expr | id - | "-" atom -> negative_types_repr + | "-" atom | "(" sum ")" // Define Lists. @@ -193,7 +192,7 @@ descruct_list: "[" ((id | (rest_accessor)) [","])* "]" "=" ret_ dict: "{" [dict_items] "}" dict_items: (dict_item [","] )+ dict_item: (id | num | str) ":" ret_expr -> dict_item_default - | id "(" [func_args] ")" func_body -> dict_item_func + | id "(" [func_args] ")" func_body -> dict_item_func | "..." ret_expr -> dict_item_rest | id -> dict_item_short @@ -205,9 +204,9 @@ descruct_dict: "{" ((id | (id ":" id) | (rest_accessor)) [","]) // -------------------------------------------------------------------------- export: "export" -declare: "declare" declare_var_type id terminator -declare_var.10: [export] declare_var_type id "=" ret_expr_with_terminator - | [export] declare_var_type id "=" function +declare_var: [export] declare_var_type id "=" ret_expr_with_terminator + | [export] declare_var_type id "=" function terminator + | [export] declare_var_type id "=" arrow_function terminator declare_var_not_initialized: [export] declare_var_type id terminator declare_var_descructed: declare_var_type descruct_dict terminator diff --git a/py-helpers/prepare_code/js/transformer.py b/py-helpers/prepare_code/js/transformer.py index 63e8234..577a0be 100644 --- a/py-helpers/prepare_code/js/transformer.py +++ b/py-helpers/prepare_code/js/transformer.py @@ -3,9 +3,13 @@ import ast import astor import logging -from prepare_code import get_logger +from ..logger import get_logger from lark import Transformer +def to_snake_case(str): + return ''.join(['_'+i.lower() if i.isupper() + else i for i in str]).lstrip('_') + class CodeTransformeJs(Transformer): @@ -17,6 +21,8 @@ class CodeTransformeJs(Transformer): self._anonymous_func_to_ids = dict() self._ids_to_anonymous_func = dict() + self._to_snake_case = False + # The Key is allways a ast.Node (the Parent Node.) # The Value is a Set, containing sub-nodes, self._anonymous_func_tree = dict() @@ -50,9 +56,6 @@ class CodeTransformeJs(Transformer): Parent = Node Childs = """ - if parent not in self._anonymous_func_tree: - self._anonymous_func_tree[parent] = set() - if type(items) is list: for item in items: @@ -62,10 +65,16 @@ class CodeTransformeJs(Transformer): def _add_to_tree_single(self, parent, item): if item in self._ids_to_anonymous_func: + if parent not in self._anonymous_func_tree: + self._anonymous_func_tree[parent] = set() + self._anonymous_func_tree[parent].add( self._ids_to_anonymous_func[item] ) elif item in self._anonymous_func_tree: + if parent not in self._anonymous_func_tree: + self._anonymous_func_tree[parent] = set() + s = self._anonymous_func_tree[parent] # Store the contained Ids. self._anonymous_func_tree[parent] = s.union( @@ -80,12 +89,29 @@ class CodeTransformeJs(Transformer): self._anonymous_func_tree[item]) self._anonymous_func_tree.pop(item) - return list(defintions_to_add) + body + if len(defintions_to_add): + return self._adapt_body(list(defintions_to_add) + body) + + return body def log(self, name, items): try: self.logger.debug( f"calling -> '{name}' -> type(items) = {type(items)}; len(items) = {len(items)}") + + for idx, item in enumerate(items): + try: + if type(item) is list: + for _idx, _item in enumerate(item): + try: + self.logger.debug(f"\t\titem[{idx}]->{_idx}: {ast.dump(_item)}") + except: + self.logger.debug(f"\t\titem[{idx}]->{_idx}: {_item}") + else: + self.logger.debug(f"\t\titem[{idx}]: {ast.dump(item)}") + except: + self.logger.debug( + f"\t\titem[{idx}]: {item}") except: self.logger.debug( f"calling -> '{name}' -> type(items) = {type(items)}") @@ -124,7 +150,13 @@ class CodeTransformeJs(Transformer): self.log_extracted("return_statement", ret_expr=ret_expr) - return _ast.Return(value=ret_expr) + value = self._adapt_items(ret_expr) + + ret = _ast.Return(value=value) + + self._add_to_tree(ret, value) + + return ret def statement(self, items): return items[0] @@ -137,6 +169,10 @@ class CodeTransformeJs(Transformer): def identifier(self, items): self.log("id", items) + + if self._to_snake_case: + return _ast.Name(id = to_snake_case(items[0].value)) + return _ast.Name(id=items[0].value) def import_stmt_all(self, items): @@ -274,6 +310,15 @@ class CodeTransformeJs(Transformer): ], keywords=[] ) + + def atom(self, items): + return items[0] + + def sum(self, items): + return items[0] + + def product(self, items): + return items[0] def add(self, items): return self._op(items, _ast.Add()) @@ -416,11 +461,10 @@ class CodeTransformeJs(Transformer): keys = [] values = [] - if type(items[0]) == list: - - for item in items[0]: + for item in items: + if item is not None: keys += item["keys"] - values += values["values"] + values += item["values"] ret = _ast.Dict( keys=keys, @@ -456,9 +500,8 @@ class CodeTransformeJs(Transformer): } def dict_item_short(self, items): - print("dict_item_short", items[0]) return { - "keys": [_ast.Constant(value=items[0].value)], + "keys": [_ast.Constant(value=items[0].id)], "values": [items[0]] } @@ -552,11 +595,15 @@ class CodeTransformeJs(Transformer): (export, name, func_args, body) = items - self.logger.debug("_function", name, func_args, body) + func_args = func_args if func_args is not None else _ast.arguments(args=[], defaults=[]) + + self.logger.debug(f"_function: name={name}, func_args={func_args}, body={body}") + + _body = self._adapt_body(body) kwargs = { "name": name.id, - "body": self._adapt_body(body), + "body": _body, "args": func_args, "decorator_list": [] } @@ -568,6 +615,8 @@ class CodeTransformeJs(Transformer): self._ids_to_anonymous_func[name] = ret # self._anonymous_func_tree[ret] = name + self._add_to_tree(ret, _body) + return ret def function(self, items): @@ -582,7 +631,6 @@ class CodeTransformeJs(Transformer): def _arrow_function(self, items, type=None): self.log("_arrow_function", items) - print(items[0]) # TODO: Get name for callback name = _ast.Name(id=self._get_func_name()) @@ -618,6 +666,10 @@ class CodeTransformeJs(Transformer): if len(ret["vararg"]) == 0: ret.pop("vararg") + elif len(ret["vararg"]) > 1: + raise Exception("Using multiple ") + else: + ret["vararg"] = ret["vararg"][0] return _ast.arguments(**ret) @@ -686,17 +738,17 @@ class CodeTransformeJs(Transformer): if type(id) is _ast.Name: id = id + args = self._adapt_items(args) + ret = _ast.Call( - func=id, - args=self._adapt_items(args), + func = id, + args = args, keywords=[] ) - return _ast.Call( - func=id, - args=self._adapt_items(args), - keywords=[] - ) + self._add_to_tree(ret, args) + + return ret def call_args(self, items): self.log("call_args", items) @@ -818,7 +870,7 @@ class CodeTransformeJs(Transformer): def inline_if(self, items): - _items = self._adapt_body(items) + _items = self._adapt_items(items) ret = _ast.IfExp( test=_items[0], @@ -855,10 +907,35 @@ class CodeTransformeJs(Transformer): return _ast.Break() def try_catch(self, items): - return _ast.Break() + + try_body, err_id, catch_body, finally_body = items + + _try_body = self._adapt_body(try_body) + _catch_body = self._adapt_body(catch_body) + _finally_body = self._adapt_body(finally_body if finally_body is not None else []) + + ret = _ast.Try( + body = _try_body, + handlers = [ + _ast.ExceptHandler( + type =_ast.Name(id='Exception'), + name = err_id.id, + body = _catch_body, + ) + + ], + orelse = [], + finalbody = _finally_body + ) + + self._add_to_tree(ret, _try_body + _catch_body + _finally_body) + + + + return ret def try_catch_body(self, items): - return + return items def throw_statement(self, items): return _ast.Raise( @@ -906,6 +983,7 @@ class DebuggedCodeTransformeJs(Transformer): def transform(tree, debug): transformer = CodeTransformeJs( True, logging.DEBUG if debug else logging.INFO) + program = transformer.transform(tree) if debug: diff --git a/py-helpers/prepare_code/main.py b/py-helpers/prepare_code/main.py index f111a16..3d98146 100644 --- a/py-helpers/prepare_code/main.py +++ b/py-helpers/prepare_code/main.py @@ -4,7 +4,7 @@ import os import re import multiprocessing as mp -from . import get_logger, ts, js +from . import get_logger, ts, js, post_process MAX_CPU = mp.cpu_count() @@ -25,30 +25,37 @@ def worker(opt): Args: opt: packed options """ - type = opt[-2] + + type = opt[0] return parse(func[type].get_parser(), *opt) -def parse(parser, logger, input_path, output_path, name, path_to_file, dir_path, type, detailed_logs): - """ Function to parse the corresponding +def parse(parser, type, logger, input_path, output_path, name, path_to_file, dir_path, debug): + """ Function to generate the python-code Args: - parser (_type_): _description_ - logger (_type_): _description_ - input_path (_type_): _description_ - output_path (_type_): _description_ - name (_type_): _description_ - path_to_file (_type_): _description_ - dir_path (_type_): _description_ + parser (_type_): The Parser to use (js/ts) + logger (_type_): The Logger + input_path (_type_): path of the folder to readin + output_path (_type_): main path of the output + name (_type_): name of the file + path_to_file (_type_): path to the file + dir_path (_type_): directory of the file + debug (_type_): Flag to enable debugging Returns: - str: _description_ - """ + ( + ( + str | False, # Directory of the PY-File, where it should be stored. False in the case of an error + str | False, # Filename of the PY-File or False in the case of an error + str | False # Python-Code or False in the case of an error + ), ( + err | False, # The Error or False, if everything was fine + str | False # Filepointer to the error if parsing using lark failed (clickable pointer for the editor) + ) + ) - func = { - "ts": ts, - "js": js - } + """ try: @@ -60,12 +67,19 @@ def parse(parser, logger, input_path, output_path, name, path_to_file, dir_path, pytho_path_to_file = os.path.join(output_path,rel_path,python_name) try: - content = parser.parse( - parser.parse(open(path_to_file, encoding="utf-8").read()) + code = func[type].transform( + parser.parse(open(path_to_file, encoding="utf-8").read()), + debug ) - logger.info(f"converted {path_to_file}") + + logger.debug(f"converted {path_to_file}") + + code = post_process(code) + + logger.info(f"processed-file: '{path_to_file}'") + return ( - (os.path.join(output_path,rel_path), pytho_path_to_file , content), + (os.path.join(output_path,rel_path), pytho_path_to_file , code), (False ,False) ) @@ -88,7 +102,7 @@ def parse(parser, logger, input_path, output_path, name, path_to_file, dir_path, logger.error(f"Failed to convert {ptr_to_err}") - if detailed_logs: + if debug: logger.error(err) else: logger.error(str(err).split("\n")[0]) @@ -123,8 +137,6 @@ def main(): help='Shows debug related output') parser.add_argument('--cores', type=int, default=MAX_CPU, dest='cores', help='The Amount of cores, which must be use') - parser.add_argument('--details', dest='details', action='store_true', - help='Shows all erros.') # Create a Logger: logger = get_logger("nope-py-prepare") @@ -143,8 +155,6 @@ def main(): logger.warn(f"Working wiht '{args.type}' file-ending.") - - # Define the Input Path input_path = os.path.join(os.getcwd(), args.inputFolder) typescript_files = [] @@ -199,14 +209,14 @@ def main(): pool = mp.Pool(cores_to_use) results = pool.map(worker, [ ( + args.type, logger, # The Logger input_path, # The Input Folder output_path, # The Output Path file_name, # Name of the File path_to_file, # Path to the File dir_path, # Path of the Directory. - args.type, - args.details + args.debug ) for file_name, path_to_file, dir_path in typescript_files ] ) @@ -227,12 +237,13 @@ def main(): file.write(content) else: - failed.append(org_file_name) + failed.append((org_file_name, err)) if len(failed): logger.warn(f"The following files failed ({len(failed)}):") - for (idx, file_name) in enumerate(failed): - print("\t", idx, ".\t", file_name) + for (idx, (file_name, err)) in enumerate(failed): + print("\t", idx+1, ".\t", file_name) + print("\t\t\t->",str(err).split("\n")[0]) if (args.debug): print("\n"*2) diff --git a/py-helpers/prepare_code/ts/debug.py b/py-helpers/prepare_code/ts/debug.py index 45400ae..8e96856 100644 --- a/py-helpers/prepare_code/ts/debug.py +++ b/py-helpers/prepare_code/ts/debug.py @@ -33,7 +33,7 @@ def parse(parser, path_to_file): try: code = transform( parser.parse(open(path_to_file, encoding="utf-8").read()), - False + True ) print(code) diff --git a/py-helpers/prepare_code/ts/transformer.py b/py-helpers/prepare_code/ts/transformer.py index 4d4ff91..69a9baf 100644 --- a/py-helpers/prepare_code/ts/transformer.py +++ b/py-helpers/prepare_code/ts/transformer.py @@ -3,7 +3,7 @@ import ast import astor import logging -from prepare_code import get_logger +from ..logger import get_logger from lark import Transformer class CodeTransformeTs(Transformer): @@ -816,7 +816,7 @@ class CodeTransformeTs(Transformer): def inline_if(self, items): - _items = self._adapt_body(items) + _items = self._adapt_items(items) ret = _ast.IfExp( test=_items[0], diff --git a/tsconfig.py.json b/tsconfig.py.json index cc750d9..8298913 100644 --- a/tsconfig.py.json +++ b/tsconfig.py.json @@ -2,7 +2,8 @@ "compilerOptions": { "target": "ES2021", "lib": [ - "ES2021" + "ES2021", + "dom" ], "allowJs": true, "allowUnusedLabels": true,