From ead5c02d0598674835941555de71d0992404f2de Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 2 May 2017 11:46:30 +0200 Subject: [PATCH 01/22] Ignore functions --- pywrap/exporter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pywrap/exporter.py b/pywrap/exporter.py index b50639f..8c35a68 100644 --- a/pywrap/exporter.py +++ b/pywrap/exporter.py @@ -435,6 +435,12 @@ def visit_template_method(self, template_method): self.visit_method(method, cppname=template_method.name) def visit_function(self, function, cppname=None): + if self.config.is_ignored(function.name): + warnings.warn("Function '%s' is on the blacklist and will be " + "ignored." % function.name) + function.ignored = True + return + try: self.functions.append(FunctionDefinition( function.name, function.comment, function.nodes, self.includes, From d0c10b06d7b0bb55e24e07a366050c68062fd6dc Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 2 May 2017 18:28:17 +0200 Subject: [PATCH 02/22] Partially fix namespace issues --- pywrap/parser.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pywrap/parser.py b/pywrap/parser.py index c87ae69..7283e33 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -309,10 +309,11 @@ def convert_ast(self, node, depth): elif node.kind == cindex.CursorKind.CXX_METHOD: if node.access_specifier == cindex.AccessSpecifier.PUBLIC: if node.is_static_method(): - namespace = self.namespace - if namespace != "": - namespace += "::" - namespace += self.last_type.name + if self.namespace != "": + namespace = "%s::%s" % (self.namespace, + self.last_type.name) + else: + namespace = self.last_type.name parse_children = self.add_function( node.spelling, node.result_type.spelling, namespace, convert_to_docstring(node.raw_comment)) @@ -461,7 +462,11 @@ def add_template_function(self, name, tname, comment=""): return True def add_class(self, name, comment=""): - clazz = Clazz(self.include_file, self.namespace, name, comment) + if self.last_type is not None: + namespace = "%s::%s" % (self.namespace, self.last_type.name) + else: + namespace = self.namespace + clazz = Clazz(self.include_file, namespace, name, comment) self.ast.nodes.append(clazz) self.last_type = clazz self.type_info.classes.append(name) From 36b89e25ba58501a4057c79b41838ffc1c3c95b2 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 2 May 2017 23:49:10 +0200 Subject: [PATCH 03/22] Organize namespaces --- pywrap/parser.py | 105 +++++++++++++++++++++---------------- pywrap/test/test_parser.py | 18 +++---- test/classinclass.hpp | 15 ++++++ test/test_features.py | 7 +++ 4 files changed, 91 insertions(+), 54 deletions(-) create mode 100644 test/classinclass.hpp diff --git a/pywrap/parser.py b/pywrap/parser.py index 7283e33..e6a42d1 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -189,7 +189,7 @@ def parse(self): self.init_ast() if self.verbose >= 1: print(make_header("Parsing")) - self.convert_ast(cursor, 0) + self.convert_ast(cursor) if self.verbose >= 2: print(make_header("AST")) print(self.ast) @@ -242,9 +242,8 @@ def init_ast(self): self.last_function = None self.last_template = None self.last_param = None - self.namespace = "" - def convert_ast(self, node, depth): + def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): """Convert AST from Clang to our own representation. Parameters @@ -252,10 +251,15 @@ def convert_ast(self, node, depth): node : clang.cindex.Index Currently visited node of Clang's AST - depth : int + depth : int, optional (default: 0) Current depth in the AST + + namespace : str, optional (default: '') + Current namespace + + additional_namespace : str, optional (default: '') + Additional namespace, e.g. current (inner) class name """ - namespace = self.namespace if self.verbose >= 1: line = " " * depth + "Node: %s" % node.kind if node.spelling: @@ -273,10 +277,10 @@ def convert_ast(self, node, depth): elif node.location.file.name != self.parsable_file: return elif node.kind == cindex.CursorKind.NAMESPACE: - if self.namespace == "": - self.namespace = node.displayname + if namespace == "": + namespace = node.displayname else: - self.namespace = self.namespace + "::" + node.displayname + namespace = namespace + "::" + node.displayname elif node.kind == cindex.CursorKind.PARM_DECL: parse_children = self.add_param( node.displayname, node.type.spelling) @@ -284,16 +288,19 @@ def convert_ast(self, node, depth): elif node.kind == cindex.CursorKind.FUNCTION_DECL: parse_children = self.add_function( node.spelling, node.result_type.spelling, - self.namespace, convert_to_docstring(node.raw_comment)) + namespace, additional_namespace, + convert_to_docstring(node.raw_comment)) elif node.kind == cindex.CursorKind.CLASS_TEMPLATE: name = node.displayname.split("<")[0] self.add_template_class( - name, convert_to_docstring(node.raw_comment)) + name, namespace, additional_namespace, + convert_to_docstring(node.raw_comment)) class_added = True elif node.kind == cindex.CursorKind.FUNCTION_TEMPLATE: if self.last_type is None: self.add_template_function( node.spelling, node.result_type.spelling, + namespace, additional_namespace, convert_to_docstring(node.raw_comment)) else: self.add_template_method( @@ -309,14 +316,11 @@ def convert_ast(self, node, depth): elif node.kind == cindex.CursorKind.CXX_METHOD: if node.access_specifier == cindex.AccessSpecifier.PUBLIC: if node.is_static_method(): - if self.namespace != "": - namespace = "%s::%s" % (self.namespace, - self.last_type.name) - else: - namespace = self.last_type.name + additional_namespace = self.last_type.name parse_children = self.add_function( node.spelling, node.result_type.spelling, - namespace, convert_to_docstring(node.raw_comment)) + namespace, additional_namespace, + convert_to_docstring(node.raw_comment)) else: parse_children = self.add_method( node.spelling, node.result_type.spelling, @@ -331,7 +335,9 @@ def convert_ast(self, node, depth): parse_children = False elif node.kind == cindex.CursorKind.CLASS_DECL: parse_children = self.add_class( - node.displayname, convert_to_docstring(node.raw_comment)) + node.displayname, namespace, additional_namespace, + convert_to_docstring(node.raw_comment)) + additional_namespace = node.displayname class_added = True elif node.kind == cindex.CursorKind.CXX_BASE_SPECIFIER: if self.last_type.base is not None: @@ -343,7 +349,8 @@ def convert_ast(self, node, depth): self.last_type.base = node.type.spelling parse_children = False elif node.kind == cindex.CursorKind.STRUCT_DECL: - parse_children = self.add_struct_decl(node.displayname) + parse_children = self.add_struct_decl( + node.displayname, namespace, additional_namespace) elif node.kind == cindex.CursorKind.FIELD_DECL: if node.access_specifier == cindex.AccessSpecifier.PUBLIC: parse_children = self.add_field( @@ -354,10 +361,12 @@ def convert_ast(self, node, depth): elif node.kind == cindex.CursorKind.TYPEDEF_DECL: tname = node.displayname parse_children = self.add_typedef( - node.underlying_typedef_type.spelling, tname) + node.underlying_typedef_type.spelling, tname, namespace, + additional_namespace) elif node.kind == cindex.CursorKind.ENUM_DECL: parse_children = self.add_enum( - node.displayname, convert_to_docstring(node.raw_comment)) + node.displayname, namespace, additional_namespace, + convert_to_docstring(node.raw_comment)) elif node.kind == cindex.CursorKind.ENUM_CONSTANT_DECL: self.last_enum.constants.append(node.displayname) elif node.kind == cindex.CursorKind.COMPOUND_STMT: @@ -387,15 +396,24 @@ def convert_ast(self, node, depth): if parse_children: for child in node.get_children(): - self.convert_ast(child, depth + 1) + self.convert_ast(child, depth + 1, namespace, + additional_namespace) if class_added: self.last_type = None if param_added: self.last_param = None - self.namespace = namespace + def _full_namespace(self, namespace, additional_namespace): + if namespace == "": + return additional_namespace + else: + if additional_namespace == "": + return namespace + else: + return namespace + "::" + additional_namespace - def add_typedef(self, underlying_tname, tname): + def add_typedef(self, underlying_tname, tname, namespace, + additional_namespace): if underlying_tname == "struct " + tname: if self.unnamed_struct is None: raise LookupError("Struct typedef does not match any " @@ -409,30 +427,25 @@ def add_typedef(self, underlying_tname, tname): else: underlying_tname = cythontype_from_cpptype(underlying_tname) self.includes.add_include_for(underlying_tname) - if self.last_type is None: - namespace = self.namespace - else: - namespace = self.namespace + "::" + self.last_type.name + namespace = self._full_namespace(namespace, additional_namespace) typedef = Typedef(self.include_file, namespace, tname, underlying_tname) self.ast.nodes.append(typedef) self.type_info.typedefs[tname] = underlying_tname return True - def add_struct_decl(self, name): + def add_struct_decl(self, name, namespace, additional_namespace): if name == "" and self.unnamed_struct is None: + namespace = self._full_namespace(namespace, additional_namespace) self.unnamed_struct = Clazz( - self.include_file, self.namespace, name, "") + self.include_file, namespace, name, "") self.last_type = self.unnamed_struct else: - self.add_class(name) + self.add_class(name, namespace, additional_namespace) return True - def add_enum(self, name, comment=""): - if self.last_type is not None: - namespace = "%s::%s" % (self.namespace, self.last_type.name) - else: - namespace = self.namespace + def add_enum(self, name, namespace, additional_namespace, comment=""): + namespace = self._full_namespace(namespace, additional_namespace) enum = Enum(self.include_file, namespace, name, comment) self.type_info.enums.append(name) self.last_enum = enum @@ -442,7 +455,9 @@ def add_enum(self, name, comment=""): def add_template_type(self, template_type): self.last_template.template_types.append(template_type) - def add_function(self, name, tname, namespace, comment=""): + def add_function(self, name, tname, namespace, additional_namespace, + comment=""): + namespace = self._full_namespace(namespace, additional_namespace) tname = cythontype_from_cpptype(tname) self.includes.add_include_for(tname) function = Function( @@ -451,29 +466,29 @@ def add_function(self, name, tname, namespace, comment=""): self.last_function = function return True - def add_template_function(self, name, tname, comment=""): + def add_template_function(self, name, tname, namespace, + additional_namespace, comment=""): tname = cythontype_from_cpptype(tname) self.includes.add_include_for(tname) - function = TemplateFunction(self.include_file, self.namespace, name, + function = TemplateFunction(self.include_file, namespace, name, tname, comment) self.ast.nodes.append(function) self.last_function = function self.last_template = function return True - def add_class(self, name, comment=""): - if self.last_type is not None: - namespace = "%s::%s" % (self.namespace, self.last_type.name) - else: - namespace = self.namespace + def add_class(self, name, namespace, additional_namespace, comment=""): + namespace = self._full_namespace(namespace, additional_namespace) clazz = Clazz(self.include_file, namespace, name, comment) self.ast.nodes.append(clazz) self.last_type = clazz self.type_info.classes.append(name) return True - def add_template_class(self, name, comment=""): - clazz = TemplateClass(self.include_file, self.namespace, name, comment) + def add_template_class(self, name, namespace, additional_namespace, + comment=""): + namespace = self._full_namespace(namespace, additional_namespace) + clazz = TemplateClass(self.include_file, namespace, name, comment) self.ast.nodes.append(clazz) self.last_type = clazz self.last_template = clazz diff --git a/pywrap/test/test_parser.py b/pywrap/test/test_parser.py index c68d81a..9701b05 100644 --- a/pywrap/test/test_parser.py +++ b/pywrap/test/test_parser.py @@ -22,16 +22,16 @@ def test_include_map(): def test_add_typedef(): parser = Parser("test.hpp") parser.init_ast() - parser.add_typedef("double", "tdef") + parser.add_typedef("double", "tdef", "", "") assert_equal(len(parser.ast.nodes), 1) def test_distributed_struct(): parser = Parser("test.hpp") parser.init_ast() - parser.add_struct_decl("") + parser.add_struct_decl("", "", "") assert_is_not_none(parser.unnamed_struct) - parser.add_typedef("struct mystruct", "mystruct") + parser.add_typedef("struct mystruct", "mystruct", "", "") assert_is_none(parser.unnamed_struct) assert_equal(len(parser.ast.nodes), 1) @@ -39,21 +39,21 @@ def test_distributed_struct(): def test_struct(): parser = Parser("test.hpp") parser.init_ast() - parser.add_struct_decl("MyStruct") + parser.add_struct_decl("MyStruct", "", "") assert_equal(len(parser.ast.nodes), 1) def test_add_function(): parser = Parser("test.hpp") parser.init_ast() - parser.add_function("myFun", "void", "") + parser.add_function("myFun", "void", "", "") assert_equal(len(parser.ast.nodes), 1) def test_add_class_with_field_ctor_and_method(): parser = Parser("test.hpp") parser.init_ast() - parser.add_class("MyClass") + parser.add_class("MyClass", "", "") assert_equal(len(parser.ast.nodes), 1) parser.add_ctor() assert_equal(len(parser.ast.nodes[0].nodes), 1) @@ -75,7 +75,7 @@ def test_add_argument_without_method(): def test_add_template_function(): parser = Parser("test.hpp") parser.init_ast() - parser.add_template_function("myFun", "void") + parser.add_template_function("myFun", "void", "", "") assert_equal(len(parser.ast.nodes), 1) parser.add_template_type("T") assert_equal(len(parser.ast.nodes[0].template_types), 1) @@ -84,7 +84,7 @@ def test_add_template_function(): def test_add_template_class(): parser = Parser("test.hpp") parser.init_ast() - parser.add_template_class("MyTemplateClass") + parser.add_template_class("MyTemplateClass", "", "") assert_equal(len(parser.ast.nodes), 1) parser.add_template_type("T") assert_equal(len(parser.ast.nodes[0].template_types), 1) @@ -101,7 +101,7 @@ def test_add_template_class(): def test_add_template_method(): parser = Parser("test.hpp") parser.init_ast() - parser.add_class("MyClass") + parser.add_class("MyClass", "", "") assert_equal(len(parser.ast.nodes), 1) parser.add_ctor() assert_equal(len(parser.ast.nodes[0].nodes), 1) diff --git a/test/classinclass.hpp b/test/classinclass.hpp new file mode 100644 index 0000000..c76b381 --- /dev/null +++ b/test/classinclass.hpp @@ -0,0 +1,15 @@ +#include + + +class ClassA +{ +public: + class ClassB + { + public: + int myfun() + { + return 123; + } + }; +}; diff --git a/test/test_features.py b/test/test_features.py index 409b79f..dc5124c 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -115,6 +115,13 @@ def test_enum_in_class(): assert_equal(enum_to_string(MyEnum.THIRDOPTION), "third") +def test_class_in_class(): + with cython_extension_from("classinclass.hpp"): + from classinclass import ClassB + b = ClassB() + assert_equal(b.myfun(), 123) + + def test_static_method(): with cython_extension_from("staticmethod.hpp"): from staticmethod import plus1, plus2 From ff04a37b331be0ec1df62ecd1ab30a5d8ad123a6 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 2 May 2017 23:57:11 +0200 Subject: [PATCH 04/22] Handle nested additional namespaces --- pywrap/parser.py | 12 ++++++++++-- test/classinclass.hpp | 5 +++++ test/test_features.py | 3 ++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pywrap/parser.py b/pywrap/parser.py index e6a42d1..995629f 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -316,7 +316,11 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): elif node.kind == cindex.CursorKind.CXX_METHOD: if node.access_specifier == cindex.AccessSpecifier.PUBLIC: if node.is_static_method(): - additional_namespace = self.last_type.name + if additional_namespace == "": + additional_namespace = self.last_type.name + else: + additional_namespace = (additional_namespace + + "::" + self.last_type.name) parse_children = self.add_function( node.spelling, node.result_type.spelling, namespace, additional_namespace, @@ -337,7 +341,11 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): parse_children = self.add_class( node.displayname, namespace, additional_namespace, convert_to_docstring(node.raw_comment)) - additional_namespace = node.displayname + if additional_namespace == "": + additional_namespace = node.displayname + else: + additional_namespace = (additional_namespace + "::" + + node.displayname) class_added = True elif node.kind == cindex.CursorKind.CXX_BASE_SPECIFIER: if self.last_type.base is not None: diff --git a/test/classinclass.hpp b/test/classinclass.hpp index c76b381..0cc0479 100644 --- a/test/classinclass.hpp +++ b/test/classinclass.hpp @@ -11,5 +11,10 @@ class ClassA { return 123; } + + static int mystatfun() + { + return 124; + } }; }; diff --git a/test/test_features.py b/test/test_features.py index dc5124c..364ad87 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -117,9 +117,10 @@ def test_enum_in_class(): def test_class_in_class(): with cython_extension_from("classinclass.hpp"): - from classinclass import ClassB + from classinclass import ClassB, mystatfun b = ClassB() assert_equal(b.myfun(), 123) + assert_equal(mystatfun(), 124) def test_static_method(): From 6e4dc7c7eb358e8ec6ef58eafc45eb5e6d46bc08 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Wed, 3 May 2017 13:41:44 +0200 Subject: [PATCH 05/22] Handle nested classes correctly --- pywrap/parser.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/pywrap/parser.py b/pywrap/parser.py index 995629f..920e83a 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -271,6 +271,7 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): parse_children = True class_added = False param_added = False + last_type = self.last_type try: if node.location.file is None: pass @@ -297,7 +298,7 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): convert_to_docstring(node.raw_comment)) class_added = True elif node.kind == cindex.CursorKind.FUNCTION_TEMPLATE: - if self.last_type is None: + if last_type is None: self.add_template_function( node.spelling, node.result_type.spelling, namespace, additional_namespace, @@ -317,10 +318,10 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): if node.access_specifier == cindex.AccessSpecifier.PUBLIC: if node.is_static_method(): if additional_namespace == "": - additional_namespace = self.last_type.name + additional_namespace = last_type.name else: additional_namespace = (additional_namespace + - "::" + self.last_type.name) + "::" + last_type.name) parse_children = self.add_function( node.spelling, node.result_type.spelling, namespace, additional_namespace, @@ -348,16 +349,16 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): node.displayname) class_added = True elif node.kind == cindex.CursorKind.CXX_BASE_SPECIFIER: - if self.last_type.base is not None: + if last_type.base is not None: warnings.warn("Class '%s' already has a base class: '%s', " "ignoring '%s'." - % (self.last_type.name, self.last_type.base, + % (last_type.name, last_type.base, node.type.spelling)) else: - self.last_type.base = node.type.spelling + last_type.base = node.type.spelling parse_children = False elif node.kind == cindex.CursorKind.STRUCT_DECL: - parse_children = self.add_struct_decl( + parse_children, additional_namespace = self.add_struct_decl( node.displayname, namespace, additional_namespace) elif node.kind == cindex.CursorKind.FIELD_DECL: if node.access_specifier == cindex.AccessSpecifier.PUBLIC: @@ -406,6 +407,8 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): for child in node.get_children(): self.convert_ast(child, depth + 1, namespace, additional_namespace) + self.last_type = last_type + if class_added: self.last_type = None if param_added: @@ -450,7 +453,11 @@ def add_struct_decl(self, name, namespace, additional_namespace): self.last_type = self.unnamed_struct else: self.add_class(name, namespace, additional_namespace) - return True + if additional_namespace == "": + additional_namespace = name + else: + additional_namespace = additional_namespace + "::" + name + return True, additional_namespace def add_enum(self, name, namespace, additional_namespace, comment=""): namespace = self._full_namespace(namespace, additional_namespace) From 986d4197e02f95c525bf2a18e4d1aa937b52a4d1 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Wed, 3 May 2017 14:09:53 +0200 Subject: [PATCH 06/22] Add size_t to standard types --- pywrap/type_conversion.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pywrap/type_conversion.py b/pywrap/type_conversion.py index 16a698c..ad27838 100644 --- a/pywrap/type_conversion.py +++ b/pywrap/type_conversion.py @@ -8,8 +8,8 @@ def is_basic_type_with_automatic_conversion(typename): # source: http://docs.cython.org/src/userguide/wrapping_CPlusPlus.html#standard-library return typename in ["bool", "string", "char *", - "int", "unsigned int", "long", "unsigned long", - "float", "double"] + "int", "unsigned int", "size_t", "long", + "unsigned long", "float", "double"] def is_stl_type_with_automatic_conversion(typename): From 87ad4fe078ae420da1bfcfa4dec4ccce006e2f81 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Wed, 3 May 2017 14:39:19 +0200 Subject: [PATCH 07/22] Update readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index cfbe779..560520b 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ There are more examples in the subdirectory `examples`. Many things are not implemented yet, e.g. * default values -* linking to other libraries (you can modify the `setup.py` though) * integrating other Cython extensions (you can modify the `setup.py` though) Feel free to work on any of these features. :) From 2551b236544687225f1d51c9afe9d9ff77e16024 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Wed, 3 May 2017 22:53:15 +0200 Subject: [PATCH 08/22] Rename static methods --- NEWS.md | 5 +++++ pywrap/ast.py | 4 +++- pywrap/exporter.py | 7 +++++-- pywrap/parser.py | 7 ++++--- test/test_features.py | 18 +++++++++--------- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/NEWS.md b/NEWS.md index 8c1c126..0fc4231 100644 --- a/NEWS.md +++ b/NEWS.md @@ -18,6 +18,11 @@ Not released yet. * Enums can be defined in classes. * Linking of external libraries. +### Breaking Changes + +* Static methods of a class 'CLS' will now have the prefix 'CLS_', e.g. + 'CLS_static_method()'. + ## Version 0.1 2017/04/03 diff --git a/pywrap/ast.py b/pywrap/ast.py index 9fd1208..27c676d 100644 --- a/pywrap/ast.py +++ b/pywrap/ast.py @@ -120,11 +120,13 @@ def __str__(self): class Function(FunctionBase): - def __init__(self, filename, namespace, name, result_type, comment=""): + def __init__(self, filename, namespace, name, result_type, comment="", + clazz=None): super(Function, self).__init__(name, comment) self.filename = filename self.namespace = namespace self.result_type = result_type + self.clazz = clazz def __str__(self): result = super(Function, self).__str__() diff --git a/pywrap/exporter.py b/pywrap/exporter.py index 8c35a68..7d19d66 100644 --- a/pywrap/exporter.py +++ b/pywrap/exporter.py @@ -445,7 +445,7 @@ def visit_function(self, function, cppname=None): self.functions.append(FunctionDefinition( function.name, function.comment, function.nodes, self.includes, function.result_type, self.type_info, - self.config, cppname=cppname).make()) + self.config, cppname=cppname, clazz=function.clazz).make()) except NotImplementedError as e: warnings.warn(e.message + " Ignoring function '%s'" % function.name) function.ignored = True @@ -461,7 +461,7 @@ def visit_param(self, param): class FunctionDefinition(object): def __init__(self, name, comment, arguments, includes, result_type, - type_info, config, cppname=None): + type_info, config, cppname=None, clazz=None): self.name = name self.comment = comment self.arguments = arguments @@ -474,6 +474,7 @@ def __init__(self, name, comment, arguments, includes, result_type, self.cppname = self.name else: self.cppname = cppname + self.clazz = clazz self.output_is_copy = True self._create_type_converters() @@ -506,6 +507,8 @@ def make(self): def _signature(self): function_name = from_camel_case( self.config.cpp_to_py_operator(self.name)) + if self.clazz is not None: + function_name = self.clazz + "_" + function_name return {"def_prefix": self._def_prefix(function_name), "args": ", ".join(self._cython_signature_args()), "name": function_name} diff --git a/pywrap/parser.py b/pywrap/parser.py index 920e83a..3f36608 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -289,7 +289,7 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): elif node.kind == cindex.CursorKind.FUNCTION_DECL: parse_children = self.add_function( node.spelling, node.result_type.spelling, - namespace, additional_namespace, + namespace, additional_namespace, None, convert_to_docstring(node.raw_comment)) elif node.kind == cindex.CursorKind.CLASS_TEMPLATE: name = node.displayname.split("<")[0] @@ -325,6 +325,7 @@ def convert_ast(self, node, depth=0, namespace="", additional_namespace=""): parse_children = self.add_function( node.spelling, node.result_type.spelling, namespace, additional_namespace, + last_type.name, convert_to_docstring(node.raw_comment)) else: parse_children = self.add_method( @@ -471,12 +472,12 @@ def add_template_type(self, template_type): self.last_template.template_types.append(template_type) def add_function(self, name, tname, namespace, additional_namespace, - comment=""): + clazz=None, comment=""): namespace = self._full_namespace(namespace, additional_namespace) tname = cythontype_from_cpptype(tname) self.includes.add_include_for(tname) function = Function( - self.include_file, namespace, name, tname, comment) + self.include_file, namespace, name, tname, comment, clazz) self.ast.nodes.append(function) self.last_function = function return True diff --git a/test/test_features.py b/test/test_features.py index 364ad87..f226040 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -107,24 +107,24 @@ def test_enum(): def test_enum_in_class(): with cython_extension_from("enuminclass.hpp"): - from enuminclass import MyEnum, enum_to_string + from enuminclass import MyEnum, MyEnumClass_enum_to_string assert_not_equal(MyEnum.FIRSTOPTION, MyEnum.SECONDOPTION) assert_not_equal(MyEnum.SECONDOPTION, MyEnum.THIRDOPTION) - assert_equal(enum_to_string(MyEnum.FIRSTOPTION), "first") - assert_equal(enum_to_string(MyEnum.SECONDOPTION), "second") - assert_equal(enum_to_string(MyEnum.THIRDOPTION), "third") + assert_equal(MyEnumClass_enum_to_string(MyEnum.FIRSTOPTION), "first") + assert_equal(MyEnumClass_enum_to_string(MyEnum.SECONDOPTION), "second") + assert_equal(MyEnumClass_enum_to_string(MyEnum.THIRDOPTION), "third") def test_class_in_class(): with cython_extension_from("classinclass.hpp"): - from classinclass import ClassB, mystatfun + from classinclass import ClassB, ClassB_mystatfun b = ClassB() assert_equal(b.myfun(), 123) - assert_equal(mystatfun(), 124) + assert_equal(ClassB_mystatfun(), 124) def test_static_method(): with cython_extension_from("staticmethod.hpp"): - from staticmethod import plus1, plus2 - assert_equal(plus1(1), 2) - assert_equal(plus2(1), 3) + from staticmethod import A_plus1, B_plus2 + assert_equal(A_plus1(1), 2) + assert_equal(B_plus2(1), 3) From 6deb75192237d427cfedc1acaa93935f45e771f8 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Thu, 4 May 2017 10:49:58 +0200 Subject: [PATCH 09/22] Fix typo: rename decleration to declaration --- NEWS.md | 3 +++ README.md | 10 +++++++++- doc/source/_static/architecture.svg | 16 ++++++++-------- doc/source/memory_management.rst | 2 +- doc/source/template_class.rst | 8 +++++--- pywrap/cython.py | 2 +- pywrap/defaultconfig.py | 6 +++--- pywrap/test/test_cython.py | 4 ++-- pywrap/type_conversion.py | 6 +++--- test/test_custom_conversions.py | 4 ++-- 10 files changed, 37 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index 0fc4231..8819fb5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -22,6 +22,9 @@ Not released yet. * Static methods of a class 'CLS' will now have the prefix 'CLS_', e.g. 'CLS_static_method()'. +* Fixed typo in interface of Config: renamed all occurences of 'decleration' + to 'declaration'. In particular, the method Config.add_decleration() + has been renamed to Config.add_declaration(). ## Version 0.1 diff --git a/README.md b/README.md index 560520b..ac6d9de 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,15 @@ Feel free to work on any of these features. :) ## Why? -I think Cython is a great tool to wrap C++ code in Python. However, it requires a lot of manual work to wrap C++ code: you have to write a file that lists all the classes and functions that you want to use and you must convert custom types manually. A lot of this work can be done automatically. A pitfall is the naming of classes and functions. It is hard to have consistent names while you write a wrapper manually. With this tool you do not rely on every developer on the project to follow the same naming scheme because the names of classes and functions in the wrapper are determined automatically from their C++ decleration. +I think Cython is a great tool to wrap C++ code in Python. However, it requires +a lot of manual work to wrap C++ code: you have to write a file that lists all +the classes and functions that you want to use and you must convert custom +types manually. A lot of this work can be done automatically. A pitfall is +the naming of classes and functions. It is hard to have consistent names while +you write a wrapper manually. With this tool you do not rely on every developer +on the project to follow the same naming scheme because the names of classes +and functions in the wrapper are determined automatically from their C++ +declaration. ## Install diff --git a/doc/source/_static/architecture.svg b/doc/source/_static/architecture.svg index 33e1ab6..62ff883 100644 --- a/doc/source/_static/architecture.svg +++ b/doc/source/_static/architecture.svg @@ -13,7 +13,7 @@ height="800" id="svg2" version="1.1" - inkscape:version="0.48.4 r9939" + inkscape:version="0.91 r13725" sodipodi:docname="architecture.svg" inkscape:export-filename="/home/afabisch/Projekte/pywrap/doc/source/_static/architecture.png" inkscape:export-xdpi="90" @@ -33,10 +33,10 @@ inkscape:document-units="px" inkscape:current-layer="layer1" showgrid="false" - inkscape:window-width="1366" - inkscape:window-height="748" - inkscape:window-x="1277" - inkscape:window-y="-3" + inkscape:window-width="1223" + inkscape:window-height="1000" + inkscape:window-x="57" + inkscape:window-y="24" inkscape:window-maximized="1" /> @@ -46,7 +46,7 @@ image/svg+xml - + @@ -165,7 +165,7 @@ y="735.87366" /> decleration + style="font-size:24px">declaration Date: Thu, 4 May 2017 11:37:58 +0200 Subject: [PATCH 10/22] Multiple static methods can have the same name --- pywrap/ast.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pywrap/ast.py b/pywrap/ast.py index 27c676d..cc1a83b 100644 --- a/pywrap/ast.py +++ b/pywrap/ast.py @@ -306,12 +306,15 @@ def _remove_overloaded_functions(asts): function_names = [] removed_functions = [] for f in functions: - if f.name in function_names: + function_name = f.name + if f.clazz is not None: + function_name = function_name + "_" + f.clazz + if function_name in function_names: warnings.warn( "Function '%s' is already defined. Only one method " - "will be exposed." % f.name) + "will be exposed." % function_name) removed_functions.append(f) else: - function_names.append(f.name) + function_names.append(function_name) for ast in asts: ast.nodes = [n for n in ast.nodes if n not in removed_functions] From c06e793f33db54014484f4ac48cd0e661e341975 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Thu, 4 May 2017 11:43:40 +0200 Subject: [PATCH 11/22] Extract method --- pywrap/ast.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pywrap/ast.py b/pywrap/ast.py index cc1a83b..1ecdd83 100644 --- a/pywrap/ast.py +++ b/pywrap/ast.py @@ -135,6 +135,12 @@ def __str__(self): "Returns (%s)" % self.result_type, 1) return result + def exported_name(self): + if self.clazz is None: + return self.name + else: + return self.clazz + "_" + self.name + class Constructor(FunctionBase): def __init__(self, class_name, comment=""): @@ -306,9 +312,7 @@ def _remove_overloaded_functions(asts): function_names = [] removed_functions = [] for f in functions: - function_name = f.name - if f.clazz is not None: - function_name = function_name + "_" + f.clazz + function_name = f.exported_name() if function_name in function_names: warnings.warn( "Function '%s' is already defined. Only one method " From 9bad1229e158b511cc33724742f44aefda2231e5 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Thu, 11 May 2017 21:25:29 +0200 Subject: [PATCH 12/22] Store file of type definition --- pywrap/parser.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pywrap/parser.py b/pywrap/parser.py index 3f36608..3a2b723 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -77,11 +77,11 @@ def implementations_import(self): class TypeInfo: def __init__(self, config=Config(), typedefs=None): self.config = config - self.classes = [] + self.classes = {} self.typedefs = {} if typedefs is not None: self.typedefs.update(typedefs) - self.enums = [] + self.enums = {} self.spec = {} def attach_specialization(self, spec): @@ -432,7 +432,7 @@ def add_typedef(self, underlying_tname, tname, namespace, "unnamed struct") self.unnamed_struct.name = tname self.ast.nodes.append(self.unnamed_struct) - self.type_info.classes.append(tname) + self.type_info.classes[tname] = self.include_file self.unnamed_struct = None self.last_type = None return False @@ -463,7 +463,7 @@ def add_struct_decl(self, name, namespace, additional_namespace): def add_enum(self, name, namespace, additional_namespace, comment=""): namespace = self._full_namespace(namespace, additional_namespace) enum = Enum(self.include_file, namespace, name, comment) - self.type_info.enums.append(name) + self.type_info.enums[name] = self.include_file self.last_enum = enum self.ast.nodes.append(enum) return True @@ -498,7 +498,7 @@ def add_class(self, name, namespace, additional_namespace, comment=""): clazz = Clazz(self.include_file, namespace, name, comment) self.ast.nodes.append(clazz) self.last_type = clazz - self.type_info.classes.append(name) + self.type_info.classes[name] = self.include_file return True def add_template_class(self, name, namespace, additional_namespace, @@ -513,7 +513,7 @@ def add_template_class(self, name, namespace, additional_namespace, for key in registered_specs: if name == key: for spec_name, _ in registered_specs[key]: - self.type_info.classes.append(spec_name) + self.type_info.classes[spec_name] = self.include_file break return True From 4c0e0c086e73dc09b9b76bc89aa6398ad3756aea Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Sat, 13 May 2017 19:18:38 +0200 Subject: [PATCH 13/22] Generate separate modules --- pywrap/cython.py | 87 +++++++++++--------- pywrap/exporter.py | 67 ++++++++------- pywrap/parser.py | 24 +++++- pywrap/template_data/class.template | 2 +- pywrap/template_data/convert_vector.template | 4 +- pywrap/template_data/enum.template | 2 +- pywrap/template_data/setup.template | 4 +- pywrap/templates.py | 6 +- pywrap/type_conversion.py | 79 +++++++++++++----- pywrap/utils.py | 5 ++ 10 files changed, 182 insertions(+), 98 deletions(-) diff --git a/pywrap/cython.py b/pywrap/cython.py index 5b0dfd8..8d1808d 100644 --- a/pywrap/cython.py +++ b/pywrap/cython.py @@ -5,7 +5,8 @@ from .parser import Parser, Includes, TypeInfo from .ast import postprocess_asts from .templates import render -from .utils import make_header, file_ending, hidden_stdout, hidden_stderr +from .utils import (make_header, file_ending, hidden_stdout, hidden_stderr, + derive_module_name_from) def load_config(custom_config): @@ -26,7 +27,7 @@ def load_config(custom_config): parts = custom_config.split(os.sep) path = os.sep.join(parts[:-1]) filename = parts[-1] - module = _derive_module_name_from(filename) + module = derive_module_name_from(filename) sys.path.insert(0, path) imported_module = __import__(module) @@ -74,7 +75,7 @@ def make_cython_wrapper(filenames, sources, modulename=None, target=".", if isinstance(filenames, str): filenames = [filenames] if len(filenames) == 1 and modulename is None: - modulename = _derive_module_name_from(filenames[0]) + modulename = derive_module_name_from(filenames[0]) if modulename is None: raise ValueError("Please give a module name when there are multiple " "C++ files that you want to wrap.") @@ -93,14 +94,14 @@ def make_cython_wrapper(filenames, sources, modulename=None, target=".", includes, type_info, asts = _parse_files( filenames, config, incdirs, verbose) - postprocess_asts(asts) + # TODO check if duplicates from separate modules are removed + postprocess_asts(asts.values()) - results = dict( - [_make_extension(modulename, asts, includes, type_info, config), - _make_declarations(asts, includes, config), - _make_setup(sources, modulename, target, incdirs, compiler_flags, - config)] - ) + extensions = _make_extension(asts, includes, type_info, config) + declarations = _make_declarations(asts, includes, config) + setup = _make_setup(sources, modulename, asts.keys(), target, incdirs, + compiler_flags, config) + results = dict(extensions + declarations + [setup]) if verbose >= 2: for filename in sorted(results.keys()): @@ -110,52 +111,62 @@ def make_cython_wrapper(filenames, sources, modulename=None, target=".", return results -def _derive_module_name_from(filename): - filename = filename.split(os.sep)[-1] - return filename.split(".")[0] - - def _parse_files(filenames, config, incdirs, verbose): includes = Includes() type_info = TypeInfo(config) - asts = [] + asts = {} for filename in filenames: + modulename = derive_module_name_from(filename) parser = Parser(filename, includes, type_info, incdirs, verbose) - asts.append(parser.parse()) + asts[modulename] = parser.parse() return includes, type_info, asts -def _make_extension(modulename, asts, includes, type_info, config): - cie = CythonImplementationExporter(includes, type_info, config) - for ast in asts: +def _make_extension(asts, includes, type_info, config): + cies = {} + for modulename, ast in asts.iteritems(): + cie = CythonImplementationExporter( + modulename, includes, type_info, config) ast.accept(cie) - pyx_filename = modulename + "." + config.pyx_file_ending - body = cie.export() - extension = includes.implementations_import() + body - return pyx_filename, extension + includes.add_custom_module(modulename) + cies[modulename] = cie + results = [] + for modulename, cie in cies.iteritems(): + pyx_filename = modulename + "." + config.pyx_file_ending + body = cie.export() + extension = includes.implementations_import() + body + results.append((pyx_filename, extension)) + return results def _make_declarations(asts, includes, config): - cde = CythonDeclarationExporter(includes, config) - for ast in asts: + cdes = {} + for modulename, ast in asts.iteritems(): + cde = CythonDeclarationExporter(includes, config) ast.accept(cde) - body = cde.export() - declarations = includes.declarations_import() + body - for decl in config.additional_declarations: - declarations += decl - pxd_filename = "_declarations." + config.pxd_file_ending - return pxd_filename, declarations + cdes[modulename] = cde + results = [] + for modulename, cde in cdes.iteritems(): + pxd_filename = "_%s.%s" % (modulename, config.pxd_file_ending) + body = cde.export() + declarations = includes.declarations_import() + body + for decl in config.additional_declarations: + declarations += decl + results.append((pxd_filename, declarations)) + return results -def _make_setup(sources, modulename, target, incdirs, compiler_flags, config): +def _make_setup(sources, modulename, modules, target, incdirs, compiler_flags, + config): sourcedir = os.path.relpath(".", start=target) source_relpaths = [os.path.relpath(filename, start=target) for filename in sources] - return "setup.py", render("setup", filenames=source_relpaths, - module=modulename, sourcedir=sourcedir, - incdirs=incdirs, compiler_flags=compiler_flags, - library_dirs=config.library_dirs, - libraries=config.libraries) + setup_py = render( + "setup", filenames=source_relpaths, modulename=modulename, + modules=modules, sourcedir=sourcedir, incdirs=incdirs, + compiler_flags=compiler_flags, library_dirs=config.library_dirs, + libraries=config.libraries) + return "setup.py", setup_py def write_files(files, target="."): diff --git a/pywrap/exporter.py b/pywrap/exporter.py index 7d19d66..80eb148 100644 --- a/pywrap/exporter.py +++ b/pywrap/exporter.py @@ -307,9 +307,10 @@ class CythonImplementationExporter(AstExporter): config : Config, optional Configuration that controls e.g. template specializations """ - def __init__(self, includes=Includes(), type_info=TypeInfo(), + def __init__(self, modulename, includes=Includes(), type_info=TypeInfo(), config=Config()): super(CythonImplementationExporter, self).__init__() + self.modulename = modulename self.includes = includes self.type_info = type_info self.config = config @@ -319,7 +320,7 @@ def visit_ast(self, ast): functions=self.functions, classes=self.classes) def visit_enum(self, enum): - self.enums.append(render("enum", enum=enum)) + self.enums.append(render("enum", enum=enum, modulename=self.modulename)) def visit_typedef(self, typedef): pass @@ -346,6 +347,7 @@ def visit_clazz(self, clazz, cppname=None): self.type_info.attach_specialization(clazz.get_attached_typeinfo()) class_def = {} class_def.update(clazz.__dict__) + class_def["modulename"] = self.modulename class_def["cppname"] = cppname class_def["comment"] = clazz.comment class_def["fields"] = map(partial( @@ -372,10 +374,10 @@ def visit_field(self, field): def _process_field(self, field, selftype): try: setter_def = SetterDefinition( - selftype, field, self.includes, self.type_info, + self.modulename, selftype, field, self.includes, self.type_info, self.config).make() getter_def = GetterDefinition( - selftype, field, self.includes, self.type_info, + self.modulename, selftype, field, self.includes, self.type_info, self.config).make() return { "name": from_camel_case(field.name), @@ -399,8 +401,8 @@ def _process_constructor(self, ctor, selftype, cpptype): try: constructor_def = ConstructorDefinition( - selftype, ctor.comment, ctor.nodes, self.includes, - self.type_info, self.config, cpptype) + self.modulename, selftype, ctor.comment, ctor.nodes, + self.includes, self.type_info, self.config, cpptype) return constructor_def.make() except NotImplementedError as e: warnings.warn(e.message + " Ignoring method '%s'" % ctor.name) @@ -420,9 +422,9 @@ def _process_method(self, arg, selftype): method, cppname = arg try: method_def = MethodDefinition( - selftype, method.comment, method.name, method.nodes, - self.includes, method.result_type, self.type_info, self.config, - cppname=cppname) + self.modulename, selftype, method.comment, method.name, + method.nodes, self.includes, method.result_type, self.type_info, + self.config, cppname=cppname) return method_def.make() except NotImplementedError as e: warnings.warn(e.message + " Ignoring method '%s'" % method.name) @@ -443,9 +445,10 @@ def visit_function(self, function, cppname=None): try: self.functions.append(FunctionDefinition( - function.name, function.comment, function.nodes, self.includes, - function.result_type, self.type_info, - self.config, cppname=cppname, clazz=function.clazz).make()) + self.modulename, function.name, function.comment, + function.nodes, self.includes, function.result_type, + self.type_info, self.config, cppname=cppname, + clazz=function.clazz).make()) except NotImplementedError as e: warnings.warn(e.message + " Ignoring function '%s'" % function.name) function.ignored = True @@ -460,8 +463,9 @@ def visit_param(self, param): class FunctionDefinition(object): - def __init__(self, name, comment, arguments, includes, result_type, - type_info, config, cppname=None, clazz=None): + def __init__(self, modulename, name, comment, arguments, includes, + result_type, type_info, config, cppname=None, clazz=None): + self.modulename = modulename self.name = name self.comment = comment self.arguments = arguments @@ -533,30 +537,33 @@ def _call_args(self): tc.cpp_call_args() for tc in self.type_converters)) def _call_cpp_function(self, call_args): - call = templates.fun_call % {"name": self.cppname, + call = templates.fun_call % {"modulename": self.modulename, + "name": self.cppname, "call_args": ", ".join(call_args)} return catch_result(self.output_type_converter.cpp_type_decl(), call) class ConstructorDefinition(FunctionDefinition): - def __init__(self, class_name, comment, arguments, includes, type_info, - config, cpp_classname): + def __init__(self, modulename, class_name, comment, arguments, includes, + type_info, config, cpp_classname): super(ConstructorDefinition, self).__init__( - "__init__", comment, arguments, includes, result_type=None, - type_info=type_info, config=config) + modulename, "__init__", comment, arguments, includes, + result_type=None, type_info=type_info, config=config) self.initial_args = ["%s self" % class_name] self.cpp_classname = cpp_classname def _call_cpp_function(self, call_args): - return templates.ctor_call % {"class_name": self.cpp_classname, + return templates.ctor_call % {"modulename": self.modulename, + "class_name": self.cpp_classname, "call_args": ", ".join(call_args)} class MethodDefinition(FunctionDefinition): - def __init__(self, class_name, comment, name, arguments, includes, - result_type, type_info, config, cppname=None): + def __init__(self, modulename, class_name, comment, name, arguments, + includes, result_type, type_info, config, cppname=None): super(MethodDefinition, self).__init__( - name, comment, arguments, includes, result_type, type_info, config, cppname) + modulename, name, comment, arguments, includes, result_type, + type_info, config, cppname) self.initial_args = ["%s self" % class_name] def _call_cpp_function(self, call_args): @@ -567,11 +574,12 @@ def _call_cpp_function(self, call_args): class SetterDefinition(MethodDefinition): - def __init__(self, python_classname, field, includes, type_info, config): + def __init__(self, modulename, python_classname, field, includes, type_info, + config): name = "__set_%s" % field.name super(SetterDefinition, self).__init__( - python_classname, "", name, [field], includes, "void", type_info, - config) + modulename, python_classname, "", name, [field], includes, "void", + type_info, config) self.field_name = field.name def _call_cpp_function(self, call_args): @@ -581,11 +589,12 @@ def _call_cpp_function(self, call_args): class GetterDefinition(MethodDefinition): - def __init__(self, python_classname, field, includes, type_info, config): + def __init__(self, modulename, python_classname, field, includes, type_info, + config): name = "__get_%s" % field.name super(GetterDefinition, self).__init__( - python_classname, "", name, [], includes, field.tipe, type_info, - config) + modulename, python_classname, "", name, [], includes, field.tipe, + type_info, config) self.output_is_copy = False self.field_name = field.name diff --git a/pywrap/parser.py b/pywrap/parser.py index 3a2b723..15c9a6b 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -6,7 +6,7 @@ from .ast import (Ast, Enum, Typedef, Clazz, Function, TemplateClass, TemplateFunction, Constructor, Method, TemplateMethod, Param, Field) -from .utils import make_header, convert_to_docstring +from .utils import make_header, convert_to_docstring, derive_module_name_from class ClangError(Exception): @@ -31,6 +31,7 @@ def __init__(self): "set": False, "stack": False} self.deref = False + self.custom_modules = set() def add_include_for(self, tname): for t in self.stl.keys(): @@ -43,6 +44,10 @@ def add_include_for_deref(self): def add_include_for_numpy(self): self.numpy = True + def add_custom_module(self, modulename): + if modulename is not None: + self.custom_modules.add(modulename) + def _part_of_tname(self, tname, subtname): return (tname == subtname or tname.startswith(subtname) or ("<" + subtname + ">") in tname or @@ -60,6 +65,9 @@ def declarations_import(self): includes += ("from libcpp.%(type)s cimport %(type)s" % {"type": t}) + os.linesep + for modulename in self.custom_modules: + includes += "cimport _%s" % modulename + os.linesep + return includes def implementations_import(self): @@ -70,7 +78,9 @@ def implementations_import(self): if self.deref: includes += ("from cython.operator cimport dereference as deref" + os.linesep) - includes += "cimport _declarations as cpp" + os.linesep + for modulename in self.custom_modules: + includes += "cimport _%s" % modulename + os.linesep + includes += "import %s" % modulename + os.linesep return includes @@ -101,6 +111,15 @@ def underlying_type(self, tname): def get_specialization(self, tname): return self.spec.get(tname, tname) + def get_modulename(self, tname): + if tname in self.classes: + filename = self.classes[tname] + elif tname in self.enums: + filename = self.enums[tname] + else: + raise KeyError("No module for %s" % tname) + return derive_module_name_from(filename) + IGNORED_NODES = [ cindex.CursorKind.CALL_EXPR, @@ -444,6 +463,7 @@ def add_typedef(self, underlying_tname, tname, namespace, underlying_tname) self.ast.nodes.append(typedef) self.type_info.typedefs[tname] = underlying_tname + self.type_info.classes[tname] = self.include_file return True def add_struct_decl(self, name, namespace, additional_namespace): diff --git a/pywrap/template_data/class.template b/pywrap/template_data/class.template index 1fe339a..cfe6236 100644 --- a/pywrap/template_data/class.template +++ b/pywrap/template_data/class.template @@ -3,7 +3,7 @@ cdef class {{ name }}: """{{ comment|indent(4) }} """ {%- endif %} - cdef cpp.{{ cppname }} * thisptr + cdef _{{ modulename }}.{{ cppname }} * thisptr cdef bool delete_thisptr def __cinit__(self): diff --git a/pywrap/template_data/convert_vector.template b/pywrap/template_data/convert_vector.template index cd412cc..c666061 100644 --- a/pywrap/template_data/convert_vector.template +++ b/pywrap/template_data/convert_vector.template @@ -1,8 +1,8 @@ # this seems to be possible only when we use C++11 (-std=c++11) # maybe it will be fixed in a future version of Cython {{ cpp_type_decl }} {{ cython_argname }} -cdef cpp.{{ cpp_tname }} * {{ python_argname }}_ptr = NULL +cdef {{ type_prefix }}{{ cpp_tname }} * {{ python_argname }}_ptr = NULL cdef {{ cpp_tname }} {{ python_argname }}_element for {{ python_argname }}_element in {{ python_argname }}: - {{ python_argname }}_ptr = {{ python_argname }}_element.thisptr + {{ python_argname }}_ptr = <{{ type_prefix }}{{ cpp_tname }}*> {{ python_argname }}_element.thisptr cpp_{{ python_argname }}.push_back(deref({{ python_argname }}_ptr)) \ No newline at end of file diff --git a/pywrap/template_data/enum.template b/pywrap/template_data/enum.template index 28d30b7..a09b144 100644 --- a/pywrap/template_data/enum.template +++ b/pywrap/template_data/enum.template @@ -4,5 +4,5 @@ cdef class {{ enum.tipe }}: """ {%- endif %} {%- for constant in enum.constants %} - {{ constant }} = cpp.{{ constant }} + {{ constant }} = _{{ modulename }}.{{ constant }} {%- endfor %} diff --git a/pywrap/template_data/setup.template b/pywrap/template_data/setup.template index f6fd309..5e73bad 100644 --- a/pywrap/template_data/setup.template +++ b/pywrap/template_data/setup.template @@ -18,9 +18,11 @@ if __name__ == '__main__': extensions = [ Extension( - "{{ module }}", + "{{ modulename }}", [ +{%- for module in modules %} "{{ module }}.pyx", +{%- endfor %} {%- for filename in filenames %} "{{ filename }}", {%- endfor %} diff --git a/pywrap/templates.py b/pywrap/templates.py index 40fc801..3317bc2 100644 --- a/pywrap/templates.py +++ b/pywrap/templates.py @@ -43,9 +43,9 @@ def render(template, **kwargs): # member definitions ctor_default_def = """ def __init__(cpp.%(name)s self): - self.thisptr = new cpp.%(name)s()""" -fun_call = "cpp.%(name)s(%(call_args)s)" -ctor_call = "self.thisptr = new cpp.%(class_name)s(%(call_args)s)" + self.thisptr = new _%(modulename)s.%(name)s()""" +fun_call = "_%(modulename)s.%(name)s(%(call_args)s)" +ctor_call = "self.thisptr = new _%(modulename)s.%(class_name)s(%(call_args)s)" method_call = "self.thisptr.%(name)s(%(call_args)s)" setter_call = "self.thisptr.%(name)s = %(call_args)s" getter_call = "self.thisptr.%(name)s" diff --git a/pywrap/type_conversion.py b/pywrap/type_conversion.py index 9ce2b45..0c5ccdc 100644 --- a/pywrap/type_conversion.py +++ b/pywrap/type_conversion.py @@ -71,9 +71,10 @@ def _type_without_pointer(tname): return tname.split()[0] -def typedef_prefix(tname, typedefs): +def typedef_prefix(tname, typedefs, type_info): if tname in typedefs: - return "cpp." + tname + type_prefix = lookup_type_prefix(type_info, tname) + return "%s%s" % (type_prefix, tname) else: return tname @@ -93,6 +94,23 @@ def find_all_subtypes(tname): return list(result) +def lookup_module(type_info, tname): + tname = _remove_pointer(tname) + return type_info.get_modulename(tname) + + +def _remove_pointer(tname): + return tname.replace(" *", "") + + +def lookup_type_prefix(type_info, tname): + modulename = lookup_module(type_info, tname) + if modulename is None: + return "" + else: + return "_" + modulename + "." + + def create_type_converter(tname, python_argname, type_info, config, context=None): converters = [] @@ -123,7 +141,7 @@ class AbstractTypeConverter(object): ... - cpdef my_function(cpp.MyType self, object a, double b): + cpdef my_function(_module.MyType self, object a, double b): cdef vector[double] cpp_a = a cdef double cpp_b = b cdef int result = self.thisptr.myFunction(cpp_a, cpp_b) @@ -231,12 +249,14 @@ def python_to_cpp(self): def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) - return "%s %s" % (typedef_prefix(spec, self.type_info.typedefs), - self.python_argname) + return "%s %s" % ( + typedef_prefix(spec, self.type_info.typedefs, self.type_info), + self.python_argname) def cpp_type_decl(self): spec = self.type_info.get_specialization(self.tname) - return "cdef " + typedef_prefix(spec, self.type_info.typedefs) + return "cdef " + typedef_prefix(spec, self.type_info.typedefs, + self.type_info) class AutomaticPointerTypeConverter(AbstractTypeConverter): @@ -260,7 +280,8 @@ def python_to_cpp(self): def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) python_tname = _type_without_pointer(typedef_prefix( - _type_without_pointer(spec), self.type_info.typedefs)) + _type_without_pointer(spec), self.type_info.typedefs, + self.type_info)) return "%s %s" % (python_tname, self.python_argname) def cpp_type_decl(self): @@ -374,7 +395,7 @@ def n_cpp_args(self): return 1 def add_includes(self, includes): - pass + includes.add_custom_module(lookup_module(self.type_info, self.tname)) def python_to_cpp(self): return "" @@ -387,7 +408,8 @@ def return_output(self, copy=True): def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) - return "cpp.%s %s" % (spec, self.python_argname) + type_prefix = lookup_type_prefix(self.type_info, self.tname) + return "%s%s %s" % (type_prefix, spec, self.python_argname) def cpp_type_decl(self): raise NotImplementedError("Cannot declare new enum instance") @@ -403,6 +425,7 @@ def n_cpp_args(self): def add_includes(self, includes): includes.add_include_for_deref() + includes.add_custom_module(lookup_module(self.type_info, self.tname)) def python_to_cpp(self): cython_argname = "cpp_" + self.python_argname @@ -420,11 +443,13 @@ def return_output(self, copy=True): def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) - return "%s %s" % (typedef_prefix(spec, self.type_info.typedefs), - self.python_argname) + return "%s %s" % ( + typedef_prefix(spec, self.type_info.typedefs, self.type_info), + self.python_argname) def cpp_type_decl(self): - return "cdef cpp.%s" % self.tname + type_prefix = lookup_type_prefix(self.type_info, self.tname) + return "cdef %s%s" % (type_prefix, self.tname) class CppPointerTypeConverter(AbstractTypeConverter): @@ -445,6 +470,9 @@ def matches(self): def n_cpp_args(self): return 1 + def add_includes(self, includes): + includes.add_custom_module(lookup_module(self.type_info, self.tname)) + def python_to_cpp(self): cython_argname = "cpp_" + self.python_argname return ("%s %s = %s.thisptr" @@ -463,12 +491,14 @@ def return_output(self, copy=True): return lines(*l) def python_type_decl(self): - return "%s %s" % (typedef_prefix(self.tname_wo_ptr, - self.type_info.typedefs), - self.python_argname) + return "%s %s" % ( + typedef_prefix(self.tname_wo_ptr, self.type_info.typedefs, + self.type_info), + self.python_argname) def cpp_type_decl(self): - return "cdef cpp.%s" % self.tname + type_prefix = lookup_type_prefix(self.type_info, self.tname) + return "cdef %s%s" % (type_prefix, self.tname) class StlTypeConverter(AbstractTypeConverter): @@ -481,6 +511,8 @@ def n_cpp_args(self): def add_includes(self, includes): includes.add_include_for_deref() + # TODO + #includes.add_custom_module(lookup_module(self.type_info, self.tname)) def cpp_call_args(self): return ["cpp_" + self.python_argname] @@ -499,7 +531,9 @@ def python_to_cpp(self): if (tname.startswith("vector") and subtypes[1] in self.type_info.classes): conversion = render( - "convert_vector", python_argname=self.python_argname, + "convert_vector", type_prefix=lookup_type_prefix( + self.type_info, subtypes[1]), + python_argname=self.python_argname, cpp_tname=self.type_info.underlying_type(subtypes[1]), cpp_type_decl=self.cpp_type_decl(), cython_argname=cython_argname) @@ -513,13 +547,16 @@ def cpp_type_decl(self): subtypes = find_all_subtypes(tname) for subtype in subtypes: spec_subtype = self.type_info.get_specialization(subtype) - prefixed_subtype = typedef_prefix(spec_subtype, - self.type_info.typedefs) + prefixed_subtype = typedef_prefix( + spec_subtype, self.type_info.typedefs, self.type_info) if (prefixed_subtype in self.type_info.enums or prefixed_subtype in self.type_info.classes): - prefixed_subtype = "cpp." + prefixed_subtype + type_prefix = lookup_type_prefix(self.type_info, + prefixed_subtype) + prefixed_subtype = "%s%s" % (type_prefix, prefixed_subtype) tname = tname.replace(subtype, prefixed_subtype) - return "cdef " + typedef_prefix(tname, self.type_info.typedefs) + return "cdef " + typedef_prefix( + tname, self.type_info.typedefs, self.type_info) default_converters = [ diff --git a/pywrap/utils.py b/pywrap/utils.py index 89e68d5..ce304e4 100644 --- a/pywrap/utils.py +++ b/pywrap/utils.py @@ -144,3 +144,8 @@ def replace_keyword_argnames(argname): if keyword.iskeyword(argname): argname = "_" + argname return argname + + +def derive_module_name_from(filename): + filename = filename.split(os.sep)[-1] + return filename.split(".")[0] From 4ec7f3d6e49753f7dc89f44a3a9eef61a066f684 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Sat, 13 May 2017 20:25:44 +0200 Subject: [PATCH 14/22] Rename header to fix test --- test/{map.hpp => maparg.hpp} | 0 test/test_type_conversions.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename test/{map.hpp => maparg.hpp} (100%) diff --git a/test/map.hpp b/test/maparg.hpp similarity index 100% rename from test/map.hpp rename to test/maparg.hpp diff --git a/test/test_type_conversions.py b/test/test_type_conversions.py index 1c24754..717305b 100644 --- a/test/test_type_conversions.py +++ b/test/test_type_conversions.py @@ -28,8 +28,8 @@ def test_complex_arg(): def test_map(): - with cython_extension_from("map.hpp"): - from map import lookup + with cython_extension_from("maparg.hpp"): + from maparg import lookup m = {"test": 0} assert_equal(lookup(m), 0) From 5ec41581a6754d8cdb851a29d9ce5d06263e288a Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Sat, 13 May 2017 20:56:18 +0200 Subject: [PATCH 15/22] Cython module prefix for type conversions --- pywrap/cython.py | 2 ++ pywrap/parser.py | 7 +++++++ pywrap/type_conversion.py | 41 ++++++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/pywrap/cython.py b/pywrap/cython.py index 8d1808d..c2cf44a 100644 --- a/pywrap/cython.py +++ b/pywrap/cython.py @@ -127,7 +127,9 @@ def _make_extension(asts, includes, type_info, config): for modulename, ast in asts.iteritems(): cie = CythonImplementationExporter( modulename, includes, type_info, config) + type_info.enter_module(modulename) ast.accept(cie) + type_info.exit_module() includes.add_custom_module(modulename) cies[modulename] = cie results = [] diff --git a/pywrap/parser.py b/pywrap/parser.py index 15c9a6b..056d17f 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -93,6 +93,13 @@ def __init__(self, config=Config(), typedefs=None): self.typedefs.update(typedefs) self.enums = {} self.spec = {} + self.current_module = None + + def enter_module(self, modulename): + self.current_module = modulename + + def exit_module(self): + self.current_module = None def attach_specialization(self, spec): self.spec = spec diff --git a/pywrap/type_conversion.py b/pywrap/type_conversion.py index 0c5ccdc..33c6a15 100644 --- a/pywrap/type_conversion.py +++ b/pywrap/type_conversion.py @@ -73,7 +73,7 @@ def _type_without_pointer(tname): def typedef_prefix(tname, typedefs, type_info): if tname in typedefs: - type_prefix = lookup_type_prefix(type_info, tname) + type_prefix = cpp_type_prefix(type_info, tname) return "%s%s" % (type_prefix, tname) else: return tname @@ -103,7 +103,15 @@ def _remove_pointer(tname): return tname.replace(" *", "") -def lookup_type_prefix(type_info, tname): +def cython_type_prefix(type_info, tname): + modulename = lookup_module(type_info, tname) + if modulename is None or modulename == type_info.current_module: + return "" + else: + return modulename + "." + + +def cpp_type_prefix(type_info, tname): modulename = lookup_module(type_info, tname) if modulename is None: return "" @@ -408,7 +416,7 @@ def return_output(self, copy=True): def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) - type_prefix = lookup_type_prefix(self.type_info, self.tname) + type_prefix = cpp_type_prefix(self.type_info, self.tname) return "%s%s %s" % (type_prefix, spec, self.python_argname) def cpp_type_decl(self): @@ -437,18 +445,21 @@ def cpp_call_args(self): def return_output(self, copy=True): # TODO only works with default and assignment operator - return lines("ret = %s()" % self.tname, - "ret.thisptr[0] = result", - "return ret") + return lines( + "ret = %s%s()" % (cython_type_prefix(self.type_info, self.tname), + self.tname), + "ret.thisptr[0] = result", + "return ret") def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) - return "%s %s" % ( + return "%s%s %s" % ( + cython_type_prefix(self.type_info, self.tname), typedef_prefix(spec, self.type_info.typedefs, self.type_info), self.python_argname) def cpp_type_decl(self): - type_prefix = lookup_type_prefix(self.type_info, self.tname) + type_prefix = cpp_type_prefix(self.type_info, self.tname) return "cdef %s%s" % (type_prefix, self.tname) @@ -483,7 +494,8 @@ def cpp_call_args(self): def return_output(self, copy=True): # TODO only works with default constructor - l = ["ret = %s()" % self.tname_wo_ptr, + l = ["ret = %s%s()" % (cython_type_prefix(self.type_info, self.tname), + self.tname_wo_ptr), "ret.thisptr = result"] if not copy: l.append("ret.delete_thisptr = False") @@ -491,13 +503,14 @@ def return_output(self, copy=True): return lines(*l) def python_type_decl(self): - return "%s %s" % ( + return "%s%s %s" % ( + cython_type_prefix(self.type_info, self.tname), typedef_prefix(self.tname_wo_ptr, self.type_info.typedefs, self.type_info), self.python_argname) def cpp_type_decl(self): - type_prefix = lookup_type_prefix(self.type_info, self.tname) + type_prefix = cpp_type_prefix(self.type_info, self.tname) return "cdef %s%s" % (type_prefix, self.tname) @@ -531,7 +544,7 @@ def python_to_cpp(self): if (tname.startswith("vector") and subtypes[1] in self.type_info.classes): conversion = render( - "convert_vector", type_prefix=lookup_type_prefix( + "convert_vector", type_prefix=cpp_type_prefix( self.type_info, subtypes[1]), python_argname=self.python_argname, cpp_tname=self.type_info.underlying_type(subtypes[1]), @@ -551,8 +564,8 @@ def cpp_type_decl(self): spec_subtype, self.type_info.typedefs, self.type_info) if (prefixed_subtype in self.type_info.enums or prefixed_subtype in self.type_info.classes): - type_prefix = lookup_type_prefix(self.type_info, - prefixed_subtype) + type_prefix = cpp_type_prefix(self.type_info, + prefixed_subtype) prefixed_subtype = "%s%s" % (type_prefix, prefixed_subtype) tname = tname.replace(subtype, prefixed_subtype) return "cdef " + typedef_prefix( From af2630fb0a8252640cdce437e0d50685493b99e9 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Sat, 13 May 2017 21:40:17 +0200 Subject: [PATCH 16/22] Put everything in one module --- pywrap/cython.py | 34 ++++++++++++----------------- pywrap/parser.py | 8 ------- pywrap/template_data/setup.template | 4 +--- pywrap/type_conversion.py | 31 +++++++------------------- 4 files changed, 23 insertions(+), 54 deletions(-) diff --git a/pywrap/cython.py b/pywrap/cython.py index c2cf44a..d72df27 100644 --- a/pywrap/cython.py +++ b/pywrap/cython.py @@ -97,11 +97,12 @@ def make_cython_wrapper(filenames, sources, modulename=None, target=".", # TODO check if duplicates from separate modules are removed postprocess_asts(asts.values()) - extensions = _make_extension(asts, includes, type_info, config) + extensions = _make_extension( + modulename, asts, includes, type_info, config) declarations = _make_declarations(asts, includes, config) setup = _make_setup(sources, modulename, asts.keys(), target, incdirs, compiler_flags, config) - results = dict(extensions + declarations + [setup]) + results = dict(declarations + [extensions, setup]) if verbose >= 2: for filename in sorted(results.keys()): @@ -122,23 +123,17 @@ def _parse_files(filenames, config, incdirs, verbose): return includes, type_info, asts -def _make_extension(asts, includes, type_info, config): - cies = {} - for modulename, ast in asts.iteritems(): +def _make_extension(modulename, asts, includes, type_info, config): + body = "" + for module, ast in asts.iteritems(): cie = CythonImplementationExporter( - modulename, includes, type_info, config) - type_info.enter_module(modulename) + module, includes, type_info, config) ast.accept(cie) - type_info.exit_module() - includes.add_custom_module(modulename) - cies[modulename] = cie - results = [] - for modulename, cie in cies.iteritems(): - pyx_filename = modulename + "." + config.pyx_file_ending - body = cie.export() - extension = includes.implementations_import() + body - results.append((pyx_filename, extension)) - return results + includes.add_custom_module(module) + body += cie.export() + extension = includes.implementations_import() + body + pyx_filename = modulename + "." + config.pyx_file_ending + return pyx_filename, extension def _make_declarations(asts, includes, config): @@ -165,9 +160,8 @@ def _make_setup(sources, modulename, modules, target, incdirs, compiler_flags, for filename in sources] setup_py = render( "setup", filenames=source_relpaths, modulename=modulename, - modules=modules, sourcedir=sourcedir, incdirs=incdirs, - compiler_flags=compiler_flags, library_dirs=config.library_dirs, - libraries=config.libraries) + sourcedir=sourcedir, incdirs=incdirs, compiler_flags=compiler_flags, + library_dirs=config.library_dirs, libraries=config.libraries) return "setup.py", setup_py diff --git a/pywrap/parser.py b/pywrap/parser.py index 056d17f..b671353 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -80,7 +80,6 @@ def implementations_import(self): os.linesep) for modulename in self.custom_modules: includes += "cimport _%s" % modulename + os.linesep - includes += "import %s" % modulename + os.linesep return includes @@ -93,13 +92,6 @@ def __init__(self, config=Config(), typedefs=None): self.typedefs.update(typedefs) self.enums = {} self.spec = {} - self.current_module = None - - def enter_module(self, modulename): - self.current_module = modulename - - def exit_module(self): - self.current_module = None def attach_specialization(self, spec): self.spec = spec diff --git a/pywrap/template_data/setup.template b/pywrap/template_data/setup.template index 5e73bad..a504342 100644 --- a/pywrap/template_data/setup.template +++ b/pywrap/template_data/setup.template @@ -20,9 +20,7 @@ if __name__ == '__main__': Extension( "{{ modulename }}", [ -{%- for module in modules %} - "{{ module }}.pyx", -{%- endfor %} + "{{ modulename }}.pyx", {%- for filename in filenames %} "{{ filename }}", {%- endfor %} diff --git a/pywrap/type_conversion.py b/pywrap/type_conversion.py index 33c6a15..b99ce9c 100644 --- a/pywrap/type_conversion.py +++ b/pywrap/type_conversion.py @@ -94,7 +94,7 @@ def find_all_subtypes(tname): return list(result) -def lookup_module(type_info, tname): +def lookup_module(type_info, tname): # TODO move to TypeInfo tname = _remove_pointer(tname) return type_info.get_modulename(tname) @@ -103,15 +103,7 @@ def _remove_pointer(tname): return tname.replace(" *", "") -def cython_type_prefix(type_info, tname): - modulename = lookup_module(type_info, tname) - if modulename is None or modulename == type_info.current_module: - return "" - else: - return modulename + "." - - -def cpp_type_prefix(type_info, tname): +def cpp_type_prefix(type_info, tname): # TODO move to TypeInfo modulename = lookup_module(type_info, tname) if modulename is None: return "" @@ -403,7 +395,7 @@ def n_cpp_args(self): return 1 def add_includes(self, includes): - includes.add_custom_module(lookup_module(self.type_info, self.tname)) + pass def python_to_cpp(self): return "" @@ -433,7 +425,6 @@ def n_cpp_args(self): def add_includes(self, includes): includes.add_include_for_deref() - includes.add_custom_module(lookup_module(self.type_info, self.tname)) def python_to_cpp(self): cython_argname = "cpp_" + self.python_argname @@ -446,15 +437,13 @@ def cpp_call_args(self): def return_output(self, copy=True): # TODO only works with default and assignment operator return lines( - "ret = %s%s()" % (cython_type_prefix(self.type_info, self.tname), - self.tname), + "ret = %s()" % self.tname, "ret.thisptr[0] = result", "return ret") def python_type_decl(self): spec = self.type_info.get_specialization(self.tname) - return "%s%s %s" % ( - cython_type_prefix(self.type_info, self.tname), + return "%s %s" % ( typedef_prefix(spec, self.type_info.typedefs, self.type_info), self.python_argname) @@ -482,7 +471,7 @@ def n_cpp_args(self): return 1 def add_includes(self, includes): - includes.add_custom_module(lookup_module(self.type_info, self.tname)) + pass def python_to_cpp(self): cython_argname = "cpp_" + self.python_argname @@ -494,8 +483,7 @@ def cpp_call_args(self): def return_output(self, copy=True): # TODO only works with default constructor - l = ["ret = %s%s()" % (cython_type_prefix(self.type_info, self.tname), - self.tname_wo_ptr), + l = ["ret = %s()" % self.tname_wo_ptr, "ret.thisptr = result"] if not copy: l.append("ret.delete_thisptr = False") @@ -503,8 +491,7 @@ def return_output(self, copy=True): return lines(*l) def python_type_decl(self): - return "%s%s %s" % ( - cython_type_prefix(self.type_info, self.tname), + return "%s %s" % ( typedef_prefix(self.tname_wo_ptr, self.type_info.typedefs, self.type_info), self.python_argname) @@ -524,8 +511,6 @@ def n_cpp_args(self): def add_includes(self, includes): includes.add_include_for_deref() - # TODO - #includes.add_custom_module(lookup_module(self.type_info, self.tname)) def cpp_call_args(self): return ["cpp_" + self.python_argname] From 942279112af787360cc4e085b0dbddafc704aa3f Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Sun, 14 May 2017 21:48:33 +0200 Subject: [PATCH 17/22] Add module prefix in declaration --- pywrap/cython.py | 6 +++--- pywrap/exporter.py | 28 +++++++++++++++++++++++++--- pywrap/parser.py | 2 +- pywrap/templates.py | 2 +- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/pywrap/cython.py b/pywrap/cython.py index d72df27..fb82f19 100644 --- a/pywrap/cython.py +++ b/pywrap/cython.py @@ -99,7 +99,7 @@ def make_cython_wrapper(filenames, sources, modulename=None, target=".", extensions = _make_extension( modulename, asts, includes, type_info, config) - declarations = _make_declarations(asts, includes, config) + declarations = _make_declarations(asts, includes, type_info, config) setup = _make_setup(sources, modulename, asts.keys(), target, incdirs, compiler_flags, config) results = dict(declarations + [extensions, setup]) @@ -136,10 +136,10 @@ def _make_extension(modulename, asts, includes, type_info, config): return pyx_filename, extension -def _make_declarations(asts, includes, config): +def _make_declarations(asts, includes, type_info, config): cdes = {} for modulename, ast in asts.iteritems(): - cde = CythonDeclarationExporter(includes, config) + cde = CythonDeclarationExporter(modulename, includes, type_info, config) ast.accept(cde) cdes[modulename] = cde results = [] diff --git a/pywrap/exporter.py b/pywrap/exporter.py index 80eb148..2672f0e 100644 --- a/pywrap/exporter.py +++ b/pywrap/exporter.py @@ -10,7 +10,7 @@ from .template_specialization import (ClassSpecializer, FunctionSpecializer, MethodSpecializer) from .templates import render -from .type_conversion import create_type_converter +from .type_conversion import create_type_converter, cpp_type_prefix from .utils import from_camel_case, replace_keyword_argnames @@ -177,12 +177,18 @@ class CythonDeclarationExporter(AstExporter): includes : Includes, optional Collects information about required import statements from the exporter + type_info : TypeInfo, optional + Contains names of custom C++ types that have been defined in the code + config : Config, optional Configuration that controls e.g. template specializations """ - def __init__(self, includes=Includes(), config=Config()): + def __init__(self, modulename, includes=Includes(), type_info=TypeInfo(), + config=Config()): super(CythonDeclarationExporter, self).__init__() + self.modulename = modulename self.includes = includes + self.type_info = type_info self.config = config def visit_ast(self, ast): @@ -221,7 +227,12 @@ def _visit_class(self, clazz, additional_args=None): def visit_field(self, field): if not field.ignored: - self.fields.append(templates.field_decl % field.__dict__) + field_dict = {} + field_dict.update(field.__dict__) + field_dict["tipe"] = ( + cpp_type_prefix(self.type_info, field_dict["tipe"]) + + field_dict["tipe"]) + self.fields.append(templates.field_decl % field_dict) def visit_constructor(self, ctor): if not ctor.ignored: @@ -243,6 +254,9 @@ def _visit_method(self, method, template, additional_args=None): if not method.ignored: method_dict = {"args": ", ".join(self.arguments)} method_dict.update(method.__dict__) + method_dict["result_type"] = ( + cpp_type_prefix(self.type_info, method_dict["result_type"]) + + method_dict["result_type"]) if additional_args is not None: method_dict.update(additional_args) method_dict["name"] = replace_operator_decl( @@ -256,6 +270,9 @@ def visit_function(self, function): if not function.ignored: function_dict = {"args": ", ".join(self.arguments)} function_dict.update(function.__dict__) + function_dict["result_type"] = ( + cpp_type_prefix(self.type_info, function_dict["result_type"]) + + function_dict["result_type"]) function_str = templates.function_decl % function_dict function_str += self._exception_suffix(function.result_type) self.functions.append(function_str) @@ -267,6 +284,9 @@ def visit_template_function(self, template_function): "args": ", ".join(self.arguments), "types": ", ".join(template_function.template_types)} function_dict.update(template_function.__dict__) + function_dict["result_type"] = ( + cpp_type_prefix(self.type_info, function_dict["result_type"]) + + function_dict["result_type"]) function_str = templates.template_function_decl % function_dict function_str += self._exception_suffix( template_function.result_type) @@ -276,6 +296,8 @@ def visit_template_function(self, template_function): def visit_param(self, param): param_dict = param.__dict__ param_dict["name"] = replace_keyword_argnames(param.name) + param_dict["type_prefix"] = cpp_type_prefix( + self.type_info, param_dict["tipe"]) self.arguments.append(templates.arg_decl % param_dict) def _exception_suffix(self, result_type): diff --git a/pywrap/parser.py b/pywrap/parser.py index b671353..b9dada0 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -116,7 +116,7 @@ def get_modulename(self, tname): elif tname in self.enums: filename = self.enums[tname] else: - raise KeyError("No module for %s" % tname) + return None return derive_module_name_from(filename) diff --git a/pywrap/templates.py b/pywrap/templates.py index 3317bc2..357150b 100644 --- a/pywrap/templates.py +++ b/pywrap/templates.py @@ -38,7 +38,7 @@ def render(template, **kwargs): method_decl = "%(result_type)s %(name)s(%(args)s)" template_method_decl = "%(result_type)s %(name)s[%(types)s](%(args)s)" constructor_decl = "%(class_name)s(%(args)s)" -arg_decl = "%(tipe)s %(name)s" +arg_decl = "%(type_prefix)s%(tipe)s %(name)s" field_decl = "%(tipe)s %(name)s" # member definitions From 7616764e7a0d65f99e601d6da93094a75c6c8a40 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Mon, 15 May 2017 22:43:11 +0200 Subject: [PATCH 18/22] Fix additional declarations --- pywrap/cython.py | 4 ++-- pywrap/defaultconfig.py | 10 +++++++--- pywrap/parser.py | 1 + test/test_custom_conversions.py | 7 ++++--- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/pywrap/cython.py b/pywrap/cython.py index fb82f19..3399613 100644 --- a/pywrap/cython.py +++ b/pywrap/cython.py @@ -147,9 +147,9 @@ def _make_declarations(asts, includes, type_info, config): pxd_filename = "_%s.%s" % (modulename, config.pxd_file_ending) body = cde.export() declarations = includes.declarations_import() + body - for decl in config.additional_declarations: - declarations += decl results.append((pxd_filename, declarations)) + for modulename, decl in config.additional_declarations.iteritems(): + results.append(("_%s.%s" % (modulename, config.pxd_file_ending), decl)) return results diff --git a/pywrap/defaultconfig.py b/pywrap/defaultconfig.py index 915ada0..75cadea 100644 --- a/pywrap/defaultconfig.py +++ b/pywrap/defaultconfig.py @@ -54,20 +54,24 @@ def __init__(self): self.registered_converters = [] self.registered_template_specializations = {} - self.additional_declarations = [] + self.additional_declarations = {} self.ignored = [] self.library_dirs = [] self.libraries = [] + self.class_to_module = {} + def cpp_to_py_operator(self, name): if name.startswith("operator") and name not in self.operators: raise NotImplementedError("Cannot convert C++ operator '%s' to " "Python operator." % name) return self.operators.get(name, name) - def add_declaration(self, decl): - self.additional_declarations.append(decl) + def add_declaration(self, modulename, decl, defined_classes=()): + self.additional_declarations[modulename] = decl + for clazz in defined_classes: + self.class_to_module[clazz] = modulename def register_class_specialization(self, cpp_classname, python_classname, template_to_type): diff --git a/pywrap/parser.py b/pywrap/parser.py index b9dada0..9b60882 100644 --- a/pywrap/parser.py +++ b/pywrap/parser.py @@ -87,6 +87,7 @@ class TypeInfo: def __init__(self, config=Config(), typedefs=None): self.config = config self.classes = {} + self.classes.update(self.config.class_to_module) self.typedefs = {} if typedefs is not None: self.typedefs.update(typedefs) diff --git a/test/test_custom_conversions.py b/test/test_custom_conversions.py index 5cddd3e..4e3a47d 100644 --- a/test/test_custom_conversions.py +++ b/test/test_custom_conversions.py @@ -44,11 +44,12 @@ def n_cpp_args(self): def add_includes(self, includes): includes.numpy = True + includes.add_custom_module("eigen_vector") def python_to_cpp(self): return lines( "cdef int %(python_argname)s_length = %(python_argname)s.shape[0]", - "cdef cpp.VectorXd %(cpp_argname)s = cpp.VectorXd(%(python_argname)s_length)", + "cdef _eigen_vector.VectorXd %(cpp_argname)s = _eigen_vector.VectorXd(%(python_argname)s_length)", "cdef int %(python_argname)s_idx", "for %(python_argname)s_idx in range(%(python_argname)s_length):", " %(cpp_argname)s.data()[%(python_argname)s_idx] = %(python_argname)s[%(python_argname)s_idx]" @@ -72,11 +73,11 @@ def python_type_decl(self): return "np.ndarray[double, ndim=1] " + self.python_argname def cpp_type_decl(self): - return "cdef cpp.VectorXd" + return "cdef _eigen_vector.VectorXd" config = Config() config.registered_converters.append(EigenConverter) - config.add_declaration(eigen_vector_decl) + config.add_declaration("eigen_vector", eigen_vector_decl, ["VectorXd"]) with cython_extension_from("eigen.hpp", config=config, incdirs=eigen3_incdir): From f7d2ff5ba8fcdb8b15437353f6114c2703f8df7f Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Mon, 15 May 2017 22:53:10 +0200 Subject: [PATCH 19/22] Fix unit tests --- pywrap/test/test_cython.py | 4 ++-- pywrap/test/test_type_conversion.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pywrap/test/test_cython.py b/pywrap/test/test_cython.py index 9718eec..18d53c8 100644 --- a/pywrap/test/test_cython.py +++ b/pywrap/test/test_cython.py @@ -23,11 +23,11 @@ def test_load_config(): from pywrap.defaultconfig import Config config = Config() -config.add_declaration("asd") +config.add_declaration("mymodule", "asd") """) try: config = load_config(configfile) - assert_equal(config.additional_declarations[0], "asd") + assert_equal(config.additional_declarations["mymodule"], "asd") finally: os.remove(configfile) diff --git a/pywrap/test/test_type_conversion.py b/pywrap/test/test_type_conversion.py index 21f8f6d..9f1f926 100644 --- a/pywrap/test/test_type_conversion.py +++ b/pywrap/test/test_type_conversion.py @@ -56,8 +56,11 @@ def test_const_chars(): def test_typedef_prefix(): - assert_equal(typedef_prefix("tdef", {}), "tdef") - assert_equal(typedef_prefix("tdef", {"tdef": "float"}), "cpp.tdef") + type_info = TypeInfo() + assert_equal(typedef_prefix("tdef", {}, type_info), "tdef") + type_info.classes["tdef"] = "cpp" + assert_equal(typedef_prefix("tdef", {"tdef": "float"}, type_info), + "_cpp.tdef") def test_find_subtypes_of_primitive(): @@ -92,5 +95,6 @@ def test_find_subtypes_of_complex_hierarchy(): def test_converter_not_available(): + config = Config() assert_raises(NotImplementedError, create_type_converter, - "UnknownType", "unknownType", TypeInfo([]), Config()) + "UnknownType", "unknownType", TypeInfo(config, []), config) From 969f07cf844b8196f21b85c61109cfb4075da859 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 16 May 2017 23:03:25 +0200 Subject: [PATCH 20/22] Fix unit tests --- pywrap/test/test_exporter.py | 37 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/pywrap/test/test_exporter.py b/pywrap/test/test_exporter.py index 80eebc5..859d73f 100644 --- a/pywrap/test/test_exporter.py +++ b/pywrap/test/test_exporter.py @@ -10,9 +10,10 @@ def test_simple_function_def(): + config = Config() method = MethodDefinition( - "Testclass", "", "testfun", [], Includes(), - "void", TypeInfo({}), Config()).make() + "mymodule", "Testclass", "", "testfun", [], Includes(), + "void", TypeInfo(config, {}), config).make() assert_multi_line_equal( method, lines("cpdef testfun(Testclass self):", @@ -21,10 +22,11 @@ def test_simple_function_def(): def test_array_arg_function_def(): + config = Config() method = MethodDefinition( - "Testclass", "", "testfun", [Param("a", "double *"), - Param("aSize", "unsigned int")], - Includes(), "void", TypeInfo({}), Config()).make() + "mymodule", "Testclass", "", "testfun", + [Param("a", "double *"), Param("aSize", "unsigned int")], + Includes(), "void", TypeInfo(config, {}), config).make() assert_multi_line_equal( method, lines("cpdef testfun(Testclass self, np.ndarray[double, ndim=1] a):", @@ -35,7 +37,7 @@ def test_array_arg_function_def(): def test_setter_definition(): field = Field("myField", "double", "MyClass") setter = SetterDefinition( - "MyClass", field, Includes(), TypeInfo(), Config()).make() + "mymodule", "MyClass", field, Includes(), TypeInfo(), Config()).make() assert_multi_line_equal( setter, lines( @@ -49,7 +51,7 @@ def test_setter_definition(): def test_getter_definition(): field = Field("myField", "double", "MyClass") getter = GetterDefinition( - "MyClass", field, Includes(), TypeInfo(), Config()).make() + "mymodule", "MyClass", field, Includes(), TypeInfo(), Config()).make() assert_multi_line_equal( getter, lines( @@ -60,37 +62,40 @@ def test_getter_definition(): ) def test_default_ctor_def(): - ctor = ConstructorDefinition("MyClass", "", [], Includes(), TypeInfo(), - Config(), "MyClass").make() + ctor = ConstructorDefinition( + "mymodule", "MyClass", "", [], Includes(), TypeInfo(), Config(), + "MyClass").make() assert_multi_line_equal( ctor, lines( "def __init__(MyClass self):", - " self.thisptr = new cpp.MyClass()" + " self.thisptr = new _mymodule.MyClass()" ) ) def test_function_def(): - fun = FunctionDefinition("myFun", "", [], Includes(), "void", TypeInfo(), - Config()).make() + fun = FunctionDefinition( + "mymodule", "myFun", "", [], Includes(), "void", TypeInfo(), Config() + ).make() assert_multi_line_equal( fun, lines( "cpdef my_fun():", - " cpp.myFun()" + " _mymodule.myFun()" ) ) def test_function_def_with_another_cppname(): - fun = FunctionDefinition("myFunInt", "", [], Includes(), "void", TypeInfo(), - Config(), cppname="myFun").make() + fun = FunctionDefinition( + "mymodule", "myFunInt", "", [], Includes(), "void", TypeInfo(), + Config(), cppname="myFun").make() assert_multi_line_equal( fun, lines( "cpdef my_fun_int():", - " cpp.myFun()" + " _mymodule.myFun()" ) ) From f51df8ba473741d48b8b385fd41fd1b8133a5569 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 16 May 2017 23:37:33 +0200 Subject: [PATCH 21/22] Document config functions --- pywrap/defaultconfig.py | 113 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 106 insertions(+), 7 deletions(-) diff --git a/pywrap/defaultconfig.py b/pywrap/defaultconfig.py index 75cadea..4727ae5 100644 --- a/pywrap/defaultconfig.py +++ b/pywrap/defaultconfig.py @@ -69,22 +69,80 @@ def cpp_to_py_operator(self, name): return self.operators.get(name, name) def add_declaration(self, modulename, decl, defined_classes=()): + """Add declaration manually. + + Parameters + ---------- + modulename : str + Name of the module. A declaration file with the name + '_modulename.pxd' will be created. + + decl : str + Content of the declaration file. + + defined_classes : iterable, optional (default: ()) + A list of classes that are defined in the declaration and will + be available in '_modulename'. + """ self.additional_declarations[modulename] = decl for clazz in defined_classes: self.class_to_module[clazz] = modulename - def register_class_specialization(self, cpp_classname, python_classname, - template_to_type): + def register_class_specialization( + self, cpp_classname, python_classname, template_to_type): + """Register a specialization for a template class. + + Parameters + ---------- + cpp_classname : str + Name of the template class + + python_classname : str + Name of the specialized template in Python + + template_to_type : dict + Maps template type names to actual type + """ self._register_specialization(cpp_classname, python_classname, template_to_type) - def register_function_specialization(self, cpp_functionname, - python_classname, template_to_type): - self._register_specialization(cpp_functionname, python_classname, + def register_function_specialization( + self, cpp_functionname, python_functionname, template_to_type): + """Register a specialization for a template function. + + Parameters + ---------- + cpp_functionname : str + Name of the template function + + python_functionname : str + Name of the specialized template in Python + + template_to_type : dict + Maps template type names to actual type + """ + self._register_specialization(cpp_functionname, python_functionname, template_to_type) - def register_method_specialization(self, cpp_classname, cpp_methodname, - python_methodname, template_to_type): + def register_method_specialization( + self, cpp_classname, cpp_methodname, python_methodname, + template_to_type): + """Register a specialization for a template method. + + Parameters + ---------- + cpp_classname : str + Name of the class + + cpp_methodname : str + Name of the template method + + python_methodname : str + Name of the specialized template in Python + + template_to_type : dict + Maps template type names to actual type + """ self._register_specialization(cpp_classname + "::" + cpp_methodname, python_methodname, template_to_type) @@ -95,12 +153,32 @@ def _register_specialization(self, key, name, template_to_type): (name, template_to_type)) def ignore_class(self, filename, class_name): + """Ignore class during generation of the bindings. + + Parameters + ---------- + filename : str + Header file + + class_name : str + Name of the class that will be ignored + """ self.ignore(filename, class_name) def is_ignored_class(self, filename, class_name): return self.is_ignored(filename, class_name) def ignore_method(self, class_name, method_name): + """Ignore method during generation of the bindings. + + Parameters + ---------- + class_name : str + Name of the class + + method_name : str + Name of the method that will be ignored + """ self.ignore(class_name, method_name) def is_ignored_method(self, class_name, method_name): @@ -113,13 +191,34 @@ def is_ignored(self, *args): return ":".join(args) in self.ignored def abstract_class(self, class_name): + """Prevent class constructor generation by declaring it abstract. + + Parameters + ---------- + class_name : str + Name of the abstract class + """ self.ignore(class_name, "__init__") def is_abstract_class(self, class_name): return self.is_ignored(class_name, "__init__") def add_library_dir(self, library_dir): + """Add library directory that is required to build the extension. + + Parameters + ---------- + library_dir : str + Directory + """ self.library_dirs.append(library_dir) def add_library(self, library): + """Add library that is required to build the extension. + + Parameters + ---------- + library : str + Library name + """ self.libraries.append(library) From 24758f706285db5070428c66e92117c61b5d6605 Mon Sep 17 00:00:00 2001 From: Alexander Fabisch Date: Tue, 16 May 2017 23:38:13 +0200 Subject: [PATCH 22/22] Document breaking changes --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 8819fb5..95e68c2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -25,6 +25,9 @@ Not released yet. * Fixed typo in interface of Config: renamed all occurences of 'decleration' to 'declaration'. In particular, the method Config.add_decleration() has been renamed to Config.add_declaration(). +* A declaration file for each C++ header will be generated. +* Config.add_declaration() takes the arguments 'modulename', 'decl', + and 'defined_classes'. ## Version 0.1