mkarkowski/nope
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

going to version 1.3.0

Browse Source
This commit is contained in:
Martin Karkowski 2022-07-17 22:24:55 +02:00
parent a919968f9c
commit 7898317d4a
14 changed files with 793 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
py-helpers/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
__pycache__/
*.egg-info
dist/
build/

8
py-helpers/00-install.sh Executable file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env bash
echo "The script you are running has basename $( basename -- "$0"; ), dirname $( dirname -- "$0"; )";
echo "The present working directory is $( pwd; )";
cd "$( dirname -- "$0"; )"
pip install .

9
py-helpers/01-prepare-ts.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
echo "The script you are running has basename $( basename -- "$0"; ), dirname $( dirname -- "$0"; )";
echo "The present working directory is $( pwd; )";
cd "$( dirname -- "$0"; )"
cd ..
nope-py-prepare-code --input lib --output temp --type ts

10
py-helpers/02-prepare-js.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
echo "The script you are running has basename $( basename -- "$0"; ), dirname $( dirname -- "$0"; )";
echo "The present working directory is $( pwd; )";
cd "$( dirname -- "$0"; )"
cd ..
rm -rf temp
nope-py-prepare-code --input dist-nodejs --output temp --type js

21
py-helpers/LICENSE Normal file
View File

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 M.Karkowski
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

4
py-helpers/MANIFEST.in Normal file
View File

@ -0,0 +1,4 @@
include *.py
incluce README.md
include *.lark
recursive-include prepare_code *

3
py-helpers/README.md Normal file
View File

@ -0,0 +1,3 @@
Helper to prepare the code in python.
This tools tries to transpile the ts - code to python code

9
py-helpers/prepare_code/__init__.py Normal file
View File

@ -0,0 +1,9 @@
__title__ = 'prepare_code'
__author__ = 'M.Karkowski'
__license__ = 'MIT'
__copyright__ = 'Copyright 2022 M.Karkowski'
__version__ = '0.1.0'
from .parser import get_parser
from .main import main
from .logger import get_logger

454
py-helpers/prepare_code/grammar.lark Normal file
View File

@ -0,0 +1,454 @@
%import common.WS
%import common.DIGIT
%import common.INT
MULTI_LINE_COMMENT: "/**" /(.|\n)*?/ "*/\n"
COMMENT: "//" /.*/
STR: /(`.*?`)|(".*?")|(\'.*?\')/
%ignore MULTI_LINE_COMMENT
%ignore COMMENT
%ignore WS
// Ignore Exports
%ignore "export" "*" "from" STR ";"
%ignore "export" "*" "as" /w+/ "from" STR ";"
%ignore "export" /(?<=export)(.|\n)+?(?=})/ "}" "from" STR ";"
%ignore "export" /(?<=export)(.|\n)+?(?=})/ "}" ";"
// --------------------------------------------------------------------------
// Summary:
// --------------------------------------------------------------------------
start: statement+
// we have to define, what is a valid return type
ret_expr: id
| str
| str_multi_line
| num
| bool
| null
| undefined
| instanceof
| typeof
| increment
| decrease
| invert
| list
| dict
| descruct_dict
| reassign
| await
| delete
| reg_ex
| throw
| sum
| product
| boolean_operation
| accessor
| function
| arrow_function
| function_call
| inline_if
| new_class
| break
| continue
| ret_expr ("as" type)+
| "(" ret_expr ")"
| continue
| break
| return
// Now we ar able to provide this expressions wiht a terminator.
ret_expr_with_terminator: ret_expr terminator
return: "return" [ret_expr]
statement: ret_expr_with_terminator
| declare
| declare_type
| declare_var
| declare_var_not_initialized
| declare_var_descructed
| import_stmt
| for
| while
| do_while
| if
| switch
| interface
| class
| decorated_class
| function
| try_catch
// --------------------------------------------------------------------------
// --------------------------------------------------------------------------
// Default Terminator:
terminator: ";"
// Default ID:
id: /[a-zA-Z_$][a-zA-Z0-9_$]*/ -> identifier
// We define valid import statements:
import_stmt: "import" str terminator
| "import" id "from" str terminator
| "import" "*" "as" id "from" str terminator
| "import" "{" import_names "}" "from" str terminator
// we may import multiple items:
import_names: import_name ("," import_name)*
// The import name might include multiple lines
import_name: id
| id "as" id -> import_as_name
| /\n/
// Lets define a string;
str: /(`.*?`)|(".*?")|(\'.*?\')/
str_multi_line: /(`(\\`|.|\n)*?`)/
// Lets define a number;
num: INT ["." INT] | "." INT
// Define a boolean;
bool: "false" | "true"
null: "null"
undefined: "undefined"
increment: accessor "++"
| "++" accessor
decrease: accessor "--"
| "--" accessor
invert: "!" ret_expr
instanceof: id "instanceof" id_typed
typeof: "typeof" accessor
delete: "delete" accessor
await: "await" ret_expr
reg_ex: "/" /(?<=\/).+(?=\/\w)/ "/" [/\w/]
?sum: product
| sum "+" product -> add
| sum "-" product -> min
| id "+=" product
| id "-=" product
?product: atom
| product "*" atom -> mul
| product "/" atom -> div
| id "*=" atom
| id "/=" atom
boolean_operation: boolean_input boolean_operator boolean_input
boolean_operator: ">"
| "<"
| "<="
| ">="
| "=="
| "==="
| "!="
| "!=="
| "&&"
| "||"
| "in"
boolean_input: ret_expr
?atom: ret_expr
| id
| "-" atom -> negative_types_repr
| "(" sum ")"
// Define Lists.
list: "[" [list_items] "]"
list_items: (list_item [","])+
list_item: ret_expr
| "..." ret_expr
descruct_list: "[" ((id | | ("..." id)) [","])* "]" "=" ret_expr
// Define Objects
dict: "{" dict_items? "}"
dict_items: (dict_item [","] )+
dict_item: (id | num | str) ":" ret_expr
| id "(" func_args? ")" func_body
| "..." ret_expr
| id
descruct_dict: "{" ((id | (id ":" id) | ("..." id)) [","])* "}" "=" ret_expr
// --------------------------------------------------------------------------
// Lets enable defining variables:
// --------------------------------------------------------------------------
export: "export"
declare: "declare" declare_var_type implicit_or_typed terminator
declare_type: [export] "type" id_typed "=" type terminator
declare_var: [export] declare_var_type implicit_or_typed "=" ret_expr_with_terminator
declare_var_not_initialized: [export] declare_var_type implicit_or_typed terminator
declare_var_descructed: declare_var_type descruct_dict terminator
| declare_var_type descruct_list terminator
// Valid defintions of variables.
declare_var_type: "let"
| "var"
| "const"
// --------------------------------------------------------------------------
// Acess Variables:
// --------------------------------------------------------------------------
// To access some vars
// We may want to convert them to
// specific types.
simple_access: id | str | num | ret_expr
accessor: simple_access
| accessor ["as" type]+
| "(" accessor ["as" type]+ ")"
| accessor ["?"] "." accessor
| accessor ["?"] "[" ret_expr "]"
| function_call
// --------------------------------------------------------------------------
// Reassignment:
// --------------------------------------------------------------------------
// lets define a reassingment:
reassign: accessor "=" ret_expr
// --------------------------------------------------------------------------
// Functions:
// These may contain:
// - Async Functions
// - Sync Functions
// They might be defined as `arrow_function`.
//
// In General, a function is name with an valid id, may be typed,
// may receives a custom amount of arguments and normally provides a
// function body;
// --------------------------------------------------------------------------
function_helper: ["<" type ("," type)* ">"] "(" func_args? ")" [":" type] func_body
function: [export] ["async"] "function" [id] ["<" type ("," type)* ">"] "(" func_args? ")" [":" type] func_body
arrow_function: [export] ["async"] ["<" type ("," type)* ">"] (( "(" func_args? ")" [":" type] ) | func_arg ) "=>" arrow_func_body
arrow_function_type: ["<" type ("," type)* ">"] "(" func_args? ")" [":" type] "=>" type
// Now we have to define the valid arguments:
// The function may receives multiple arguments
// which are typed or implicit (-> any)
func_args: func_arg ("," func_arg)*
func_arg: implicit_or_typed
| "..." implicit_or_typed
| implicit_or_typed "=" ret_expr
| dict | list
implicit_or_typed: id
| id ["?"] ":" type
// Define the Function Body:
// This consists of the brackets and the statements in the function
func_body: "{" [func_statements] "}"
func_statements: func_statement+
func_statement: statement
arrow_func_body: func_body
| ret_expr
// And now we define, which elements are allowed to be included
// in a function. in our case these are more or less all statements
// Additionally, we have ot make shure, that we are able to
// "return" something.
// --------------------------------------------------------------------------
// --------------------------------------------------------------------------
// The Defintion of how a function could be called.
// In here we consider sync as well as async calls.
function_call: ret_expr ["<" type ("," type)* ">"] "(" call_args? ")"
// We define our call args:
call_args: call_arg ("," call_arg)*
call_arg: ret_expr
| "..." ret_expr
type: basic_type
| enhanced_type
| "(" type ")"
basic_type: id
| "any"
| "string" -> string
| "boolean" -> boolean
| "number" -> number
| ["|"] str ("|" str)* // Typess like `const a: "hello" | "world" = "world"
| arrow_function_type // arrow functions
enhanced_type: "|" type // Leading type.
| id "in" type
| id ("<" type ("," type)* ">") // Typess like `Map<string,string>
| "[" type ("," type)* "]" // Typess like `Array<[string,boolean]>
| "{" ( id ["?"] ":" type [";"] )* "}" // Dynamic type like
| "{" "[" id (":" | "in") type "]" ":" type [";"]"}" // Dynamic type like `{ [index: string]: INopeDescriptor }`
| type "extends"? "keyof"? "typeof"? type // Types consisting of keyof , typeof etc
| type ("[" (type | num) "]")+ // Partial type of the Element
| type ("[" "]")+ // Array type
| type ("&" type)+ // Combinded type using &
| type ("|" type)+ // Types which might be
| type "=" type // Types which are assigned with a default type.
| accessor
// Definition for a Nested Type.
id_typed: id | enhanced_type
// --------------------------------------------------------------------------
// Loops
// --------------------------------------------------------------------------
// Define a For - Statement
for: "for" "(" declare_var_type for_iter_var ("of" | "in") ret_expr ")" iter_body
| "for" "(" declare_var_type "[" (for_iter_var [","])+ "]" ("of" | "in") ret_expr ")" iter_body
| "for" "(" declare_var_type id "=" ret_expr ";" ret_expr ";" ret_expr ")" iter_body
for_iter_var: id | dict | list
while: "while" "(" ret_expr ")" iter_body
do_while: "do" iter_body "while" "(" ret_expr ")" terminator
iter_body: "{" iter_statements "}" | iter_statement
iter_statements: iter_statement*
iter_statement: statement
continue: "continue"
break: "break"
// --------------------------------------------------------------------------
// IF-Statements
// --------------------------------------------------------------------------
// Define a If - Statement
// We have to consider "if" "else if" and "else"
if: "if" "(" ret_expr ")" if_body ("else" "if" "(" ret_expr ")" if_body)* ["else" if_body]
if_body: "{" statement* "}"
| ret_expr_with_terminator
inline_if: ret_expr "?" ret_expr ":" ret_expr
// --------------------------------------------------------------------------
// switch-case
// --------------------------------------------------------------------------
switch: "switch" "(" ret_expr ")" switch_body
switch_body: "{" ((switch_case)* [switch_default])* "}"
switch_case: "case" ret_expr ":" [switch_case_body]
switch_default: "default" ":" [switch_case_body]
switch_case_body: ("{" switch_case_statements "}") | switch_case_statements
switch_case_statements: switch_case_statement*
switch_case_statement: statement
| break
// --------------------------------------------------------------------------
// Error Handling
// --------------------------------------------------------------------------
try_catch: "try" try_catch_body "catch" "(" id ")" try_catch_body ["finally" try_catch_body]
try_catch_body: "{" statement* "}"
throw: "throw" ret_expr
// --------------------------------------------------------------------------
// interfaces
// --------------------------------------------------------------------------
interface: [export] "interface" id_typed ["extends" implementing_interfaces] interface_body
extending_interfaces: id_typed ("," id_typed)*
interface_body: "{" interface_declarations* "}"
interface_declarations: interface_declaration
// Interfaces contains implicit or typed attributes or methods
interface_declaration: [readonly] implicit_or_typed terminator
| id ["<" type ("," type)* ">"] "(" func_args? ")" [":" type ] terminator
// --------------------------------------------------------------------------
// classes
// --------------------------------------------------------------------------
class: [export] "class" id_typed ["extends" id_typed] ["implements" implementing_interfaces] class_body
decorated_class: ("@" function_call) class
implementing_interfaces: id_typed ("," id_typed)*
class_body: "{" class_declarations* "}"
class_declarations: constructor
| property_defintion
| property_assigned
| decorated_property_assigned
| decorated_property_defintion
| getter
| setter
| method
| async_method
| decorated_method
| decorated_async_method
constructor: ["public"] "constructor" "(" constructor_args? ")" func_body
constructor_args: constructor_arg ("," constructor_arg)*
constructor_arg: ["@" function_call] [visibility] [readonly] func_arg
visibility: "private"
| "protected"
| "public"
readonly: "readonly"
property_defintion: [visibility] [readonly] implicit_or_typed terminator
property_assigned: [visibility] [readonly] implicit_or_typed "=" ret_expr_with_terminator
decorated_property_defintion: ("@" function_call) property_defintion
decorated_property_assigned: ("@" function_call) property_assigned
getter: [visibility] "get" id "("")" [":" type] func_body
setter: [visibility] "set" id "(" func_arg ")" [":" "void"] func_body
method: [visibility] id ["<" type ("," type)* ">"] "(" func_args? ")" [":" type] func_body
async_method: [visibility] "async" id ["<" type ("," type)* ">"] "(" func_args? ")" [":" type] func_body
decorated_method: ("@" function_call) method
decorated_async_method: ("@" function_call) async_method
// Types for a new Class
new_class: "new" id_typed "(" call_args? ")"

31
py-helpers/prepare_code/logger.py Normal file
View File

@ -0,0 +1,31 @@
import logging
import sys
def get_logger(name: str, level=logging.INFO):
""" Creates a Logger for the Tool.
Args:
name (str): Name of the logger
level: The logging Level. Defaults to logging.INFO.
Returns:
a. logger
"""
_logger = logging.getLogger(name)
# Define a Logging Format
_format = _format = logging.Formatter(
'%(levelname)s - %(message)s')
# Create Console Output
_handler = logging.StreamHandler(sys.stdout)
# Add the Format to the Handler
_handler.setFormatter(_format)
# Set Loglevel to the Desired One.
_handler.setLevel(level)
# Finally add the Handler to the Logger:
_logger.addHandler(_handler)
# Set the Log Level of the Logger.
_logger.setLevel(level)
return _logger

193
py-helpers/prepare_code/main.py Normal file
View File

@ -0,0 +1,193 @@
import argparse
import os
import re
import multiprocessing as mp
from .parser import get_parser
from .logger import get_logger
MAX_CPU = mp.cpu_count()
if MAX_CPU <= 4:
MAX_CPU = 2
def worker(opt):
""" Helper function, which will be called during a multiprocess. Converts the input.
Args:
opt: packed options
"""
return parse(get_parser(), *opt)
def parse(parser, logger, input_path, output_path, name, path_to_file, dir_path):
""" Function to parse the corresponding
Args:
parser (_type_): _description_
logger (_type_): _description_
input_path (_type_): _description_
output_path (_type_): _description_
name (_type_): _description_
path_to_file (_type_): _description_
dir_path (_type_): _description_
Returns:
str: _description_
"""
try:
python_name = name.replace(".ts", ".py")
rel_path = dir_path[len(input_path) + 1:]
logger.debug(f"determined the following rel-path = {rel_path}")
pytho_path_to_file = os.path.join(output_path,rel_path,python_name)
try:
content = parser.parse(open(path_to_file, encoding="utf-8").read())
logger.info(f"converted {path_to_file}")
return (
(pytho_path_to_file , content),
(False ,False)
)
except Exception as err:
ptr_to_err = path_to_file
try:
m = re.search('(?<=line )\d+', str(err))
line = int(m.group(0))
m = re.search('(?<=col )\d+', str(err))
col = int(m.group(0))
ptr_to_err = path_to_file+":"+str(line)+":"+str(col)
except:
pass
logger.error(f"Failed to convert {ptr_to_err}")
logger.error(str(err).split("\n")[0])
return (
(False, False),
(str(err), ptr_to_err)
)
except Exception as err:
# An unknown Error has been found
logger.error(err)
return (
(False,False),
(str(err), False)
)
def main():
""" The main routine.
"""
parser = argparse.ArgumentParser(
description='Tython. A tool to convert the typescript file to the given python files.')
parser.add_argument('--input', type=str, default="./", dest='inputFolder',
help='Defines the Folder with the Files, being search.')
parser.add_argument('--output', type=str, default="./out/", dest='outputFolder',
help='Defines the Files, which should be looked for. Possible Values are "ts" | "js"')
parser.add_argument('--type', type=str, default="ts", dest='type',
help='Defines the Folder where the converted files should be stored.')
parser.add_argument('--debug', dest='debug', action='store_true',
help='Shows debug related output')
parser.add_argument('--cores', type=int, default=MAX_CPU, dest='cores',
help='The Amount of cores, which must be use')
# Create a Logger:
logger = get_logger("nope-py-prepare")
args = parser.parse_args()
files_to_ignore ={
"js": lambda name: name.endswith(".js") and not (name.endswith(".spec.js") or "index" in name),
"ts": lambda name: name.endswith(".ts") and not (name.endswith(".spec.ts") or "index" in name),
}
if not args.type in ("ts","js"):
logger.error("Please use the correct type")
logger.error(f"Determined type: '{args.type}'")
return
logger.warn(f"Working wiht '{args.type}' file-ending.")
# Define the Input Path
input_path = os.path.join(os.getcwd(), args.inputFolder)
typescript_files = []
logger.info(f"Checking dir: '{input_path}'")
# Get all relevant Files.
for dir_path, directories, files in os.walk(input_path):
for file_name in files:
if files_to_ignore[args.type](file_name):
# Generate the Path of files.
path_to_file = os.path.join(dir_path, file_name)
# Show a log message
logger.debug(f"Found file: '{file_name}' at '{dir_path}'")
# Add the file-name, path to the file and the dir path
typescript_files.append((file_name, path_to_file, dir_path))
# Define the Destination
output_path = os.path.join(os.getcwd(), args.outputFolder)
cores_to_use = max(1, min(MAX_CPU, args.cores))
logger.info(f"Founf {len(typescript_files)} files. Starting multiprocess with {cores_to_use} cores.")
# Create Pool.
pool = mp.Pool(cores_to_use)
results = pool.map(worker, [
(
logger, # The Logger
input_path, # The Input Folder
output_path, # The Output Path
file_name, # Name of the File
path_to_file, # Path to the File
dir_path # Path of the Directory.
) for file_name, path_to_file, dir_path in typescript_files
]
)
# Close the Pool
pool.close()
# Wait to finish all.
pool.join()
success = []
failed = []
for (py_file_name,content),(err, org_file_name) in results:
if py_file_name and content:
success.append(py_file_name)
else:
failed.append(org_file_name)
logger.warn(f"The following files failed ({len(failed)}):")
for (idx, file_name) in enumerate(failed):
print("\t", idx, ".\t", file_name)
if (args.debug):
print("\n"*2)
print(f"Created the following files ({len(success)}):")
for file_name in success:
print("\t- ", file_name)
print("\n"*2)
logger.info(f"Parsed {len(success)} of {len(typescript_files)} files ({(len(success)/len(typescript_files))*100:.2f} %).")
if __name__ == "__main__":
main()

14
py-helpers/prepare_code/parser.py Normal file
View File

@ -0,0 +1,14 @@
from lark import Lark
from pathlib import Path
# Define the Grammar File.
grammar_file_path = Path(__file__).parent.joinpath('grammar.lark')
def get_parser():
""" Helper, to generate a parser.
Returns:
A lark parser
"""
return Lark.open(grammar_file_path,debug = True, maybe_placeholders=True)

2
py-helpers/requirements.txt Normal file
View File

@ -0,0 +1,2 @@
astor==0.8.1
lark-parser>=0.8.1

31
py-helpers/setup.py Normal file
View File

@ -0,0 +1,31 @@
from setuptools import setup,find_packages
import re
with open('requirements.txt') as f:
requirements = f.read().splitlines()
with open('prepare_code/__init__.py') as f:
version = re.search("__version__ = '(.+)'", f.read()).group(1)
with open('README.md', encoding='utf-8') as f:
readme = f.read()
setup(
name='prepare_code',
author='M.Karkowski',
python_requires='>=3.6.0',
version=version,
packages=['prepare_code'],
license='MIT',
description='A simple TypeScript to Python transpiler. Helps the user to simplify the code.',
long_description=readme,
long_description_content_type='text/markdown',
include_package_data=True,
install_requires=requirements,
classifiers=[],
entry_points={
'console_scripts': [
'nope-py-prepare-code = prepare_code.main:main',
],
}
)