Support both Cython >3 and Cython < 3 (#669)

* Remove .pxi include files from setup.py

* show pip list

* dont allow cython 3

* Revert "show pip list"

This reverts commit b3f88b0443.

* separate FILES into PYX_FILES and PXI_FILES

* testing for cython 3

* fixes __cls_storage in Cython 3

* detect Cython 3 at compile time, compile appropriately

* force cython version using sed. expected to fail on windows.

* i think this will work on the Windows sed on GHA

* cleanup __cls_storage name consistently

* migrate to getattr() to make code more elegant

* fix GHA for macos by separating seds

* add actual extension!

* access all CLS_STORAGE through the cdef 'constant'

.github/workflows/push.yml

@@ -26,6 +26,9 @@ jobs:
         architecture:
           - 'x64'
           - 'x86'
+        cython:
+          - '<3'
+          - '>=3'
 
         # exclude problematic combinations
         exclude:
@@ -67,6 +70,16 @@ jobs:
     - name: Build test classes via ant
       run: ant all
 
+    - name: (Windows) Force Cython version
+      # Windows sed doesnt accept .bak filename extensions
+      if: matrix.os == 'windows-latest'
+      run: sed -i 's/"Cython"/"Cython${{matrix.cython}}"/' pyproject.toml
+    
+    - name: (Linux, macOS) Force Cython version
+      # macOS sed requires .bak filename extensions
+      if: (matrix.os == 'ubuntu-latest') || (matrix.os == 'macos-latest')
+      run: sed -i.bak 's/"Cython"/"Cython${{matrix.cython}}"/' pyproject.toml
+
     - name: Install pyjnius with [dev, ci] extras
       run: |
         pip install --timeout=120 .[dev,ci]

README.md

@@ -119,7 +119,7 @@ You can use the `signatures` method of `JavaMethod` and `JavaMultipleMethod`, to
 ```python
 >>> String = autoclass('java.lang.String')
 >>> dir(String)
-['CASE_INSENSITIVE_ORDER', '__class__', '__cls_storage', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__javaclass__', '__javaconstructor__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__pyx_vtable__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', 'charAt', 'checkBounds', 'clone', 'codePointAt', 'codePointBefore', 'codePointCount', 'compareTo', 'compareToIgnoreCase', 'concat', 'contains', 'contentEquals', 'copyValueOf', 'empty', 'endsWith', 'equals', 'equalsIgnoreCase', 'finalize', 'format', 'getBytes', 'getChars', 'getClass', 'hashCode', 'indexOf', 'indexOfSupplementary', 'intern', 'isEmpty', 'join', 'lastIndexOf', 'lastIndexOfSupplementary', 'length', 'matches', 'nonSyncContentEquals', 'notify', 'notifyAll', 'offsetByCodePoints', 'regionMatches', 'registerNatives', 'replace', 'replaceAll', 'replaceFirst', 'split', 'startsWith', 'subSequence', 'substring', 'toCharArray', 'toLowerCase', 'toString', 'toUpperCase', 'trim', 'valueOf', 'wait']
+['CASE_INSENSITIVE_ORDER', '__class__', '_JavaClass__cls_storage', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__javaclass__', '__javaconstructor__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__pyx_vtable__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', 'charAt', 'checkBounds', 'clone', 'codePointAt', 'codePointBefore', 'codePointCount', 'compareTo', 'compareToIgnoreCase', 'concat', 'contains', 'contentEquals', 'copyValueOf', 'empty', 'endsWith', 'equals', 'equalsIgnoreCase', 'finalize', 'format', 'getBytes', 'getChars', 'getClass', 'hashCode', 'indexOf', 'indexOfSupplementary', 'intern', 'isEmpty', 'join', 'lastIndexOf', 'lastIndexOfSupplementary', 'length', 'matches', 'nonSyncContentEquals', 'notify', 'notifyAll', 'offsetByCodePoints', 'regionMatches', 'registerNatives', 'replace', 'replaceAll', 'replaceFirst', 'split', 'startsWith', 'subSequence', 'substring', 'toCharArray', 'toLowerCase', 'toString', 'toUpperCase', 'trim', 'valueOf', 'wait']
 >>> String.format.signatures()
 [(['java/util/Locale', 'java/lang/String', 'java/lang/Object...'], 'java/lang/String'), (['java/lang/String', 'java/lang/Object...'], 'java/lang/String')]
 ```

jnius/jnius.pyx

@@ -106,6 +106,10 @@ ELIF JNIUS_PLATFORM == "win32":
 ELSE:
     include "jnius_jvm_dlopen.pxi"
 
+# from Cython 3.0, in the MetaJavaClass, this is accessed as _JavaClass__cls_storage
+# see https://cython.readthedocs.io/en/latest/src/userguide/migrating_to_cy30.html#class-private-name-mangling
+cdef CLS_STORAGE_NAME = '_JavaClass__cls_storage' if JNIUS_CYTHON_3 else '__cls_storage'
+
 include "jnius_env.pxi"
 include "jnius_utils.pxi"
 include "jnius_conversion.pxi"

jnius/jnius_conversion.pxi

@@ -88,7 +88,7 @@ cdef void populate_args(JNIEnv *j_env, tuple definition_args, jvalue *j_args, ar
                 jo = py_arg
                 j_args[index].l = jo.obj
             elif isinstance(py_arg, MetaJavaClass):
-                jcs = py_arg.__cls_storage
+                jcs = getattr(py_arg, CLS_STORAGE_NAME)
                 j_args[index].l = jcs.j_cls
             elif isinstance(py_arg, PythonJavaClass):
                 # from python class, get the proxy/python class
@@ -515,7 +515,7 @@ cdef jobject convert_python_to_jobject(JNIEnv *j_env, definition, obj) except *:
             jo = obj
             return jo.obj
         elif isinstance(obj, MetaJavaClass):
-            jcs = obj.__cls_storage
+            jcs = getattr(obj, CLS_STORAGE_NAME)
             return jcs.j_cls
         elif isinstance(obj, PythonJavaClass):
             # from python class, get the proxy/python class

jnius/jnius_export_class.pxi

@@ -1,7 +1,6 @@
 from cpython cimport PyObject
 from warnings import warn
 
-
 class JavaException(Exception):
     '''Can be a real java exception, or just an exception from the wrapper.
     '''
@@ -43,7 +42,7 @@ cdef class JavaClassStorage:
 class MetaJavaBase(type):
     def __instancecheck__(cls, value):
         cdef JNIEnv *j_env = get_jnienv()
-        cdef JavaClassStorage meta = getattr(cls, '__cls_storage', None)
+        cdef JavaClassStorage meta = getattr(cls, CLS_STORAGE_NAME, None)
         cdef JavaObject jo
         cdef JavaClass jc
         cdef PythonJavaClass pc
@@ -132,7 +131,7 @@ class MetaJavaClass(MetaJavaBase):
 
     def __subclasscheck__(cls, value):
         cdef JNIEnv *j_env = get_jnienv()
-        cdef JavaClassStorage me = getattr(cls, '__cls_storage')
+        cdef JavaClassStorage me = getattr(cls, CLS_STORAGE_NAME)
         cdef JavaClassStorage jcs
         cdef JavaClass jc
         cdef jclass obj = NULL
@@ -141,7 +140,7 @@ class MetaJavaClass(MetaJavaBase):
             jc = value
             obj = jc.j_self.obj
         else:
-            jcs = getattr(value, '__cls_storage', None)
+            jcs = getattr(value, CLS_STORAGE_NAME, None)
             if jcs is not None:
                 obj = jcs.j_cls
 
@@ -218,7 +217,7 @@ class MetaJavaClass(MetaJavaBase):
         #    in the section Local and Global References
         jcs.j_cls = j_env[0].NewGlobalRef(j_env, jcs.j_cls)
 
-        classDict['__cls_storage'] = jcs
+        classDict[CLS_STORAGE_NAME] = jcs
 
         # search all the static JavaMethod within our class, and resolve them
         cdef JavaMethod jm
@@ -263,7 +262,7 @@ cdef class JavaClass(object):
     def __init__(self, *args, **kwargs):
         super(JavaClass, self).__init__()
         # copy the current attribute in the storage to our class
-        cdef JavaClassStorage jcs = self.__cls_storage
+        cdef JavaClassStorage jcs = getattr(self, CLS_STORAGE_NAME, None)
         self.j_cls = jcs.j_cls
 
         if 'noinstance' not in kwargs:

pyproject.toml

@@ -2,5 +2,5 @@
 requires = [
     "setuptools>=58.0.0",
     "wheel",
-    "Cython>=0.29.30"
-]
+    "Cython"
+]

setup.py

@@ -33,8 +33,10 @@ def getenv(key):
     return val
 
 
-FILES = [
-    'jni.pxi',
+PYX_FILES = [
+    'jnius.pyx',
+]
+PXI_FILES = [
     'jnius_compat.pxi',
     'jnius_conversion.pxi',
     'jnius_export_class.pxi',
@@ -46,7 +48,7 @@ FILES = [
     'jnius_nativetypes3.pxi',
     'jnius_proxy.pxi',
     'jnius.pyx',
-    'jnius_utils.pxi',
+    'jnius_utils.pxi'
 ]
 
 EXTRA_LINK_ARGS = []
@@ -59,7 +61,7 @@ if NDKPLATFORM is not None and getenv('LIBLINK'):
 
 # detect platform
 if PLATFORM == 'android':
-    FILES = [fn[:-3] + 'c' for fn in FILES if fn.endswith('pyx')]
+    PYX_FILES = [fn[:-3] + 'c' for fn in PYX_FILES]
 
 JAVA=get_java_setup(PLATFORM)
 
@@ -85,14 +87,20 @@ compile_native_invocation_handler(JAVA)
 
 # generate the config.pxi
 with open(join(dirname(__file__), 'jnius', 'config.pxi'), 'w') as fd:
+    import Cython
+    cython3 = Cython.__version__.startswith('3.')
     fd.write('DEF JNIUS_PLATFORM = {0!r}\n\n'.format(PLATFORM))
+    # record the Cython version, to address #669
+    fd.write(f'DEF JNIUS_CYTHON_3 = {cython3}')
 
 # pop setup.py from included files in the installed package
 SETUP_KWARGS['py_modules'].remove('setup')
 
 ext_modules = [
     Extension(
-        'jnius', [join('jnius', x) for x in FILES],
+        'jnius', 
+        [join('jnius', x) for x in PYX_FILES],
+        depends=[join('jnius', x) for x in PXI_FILES],
         libraries=JAVA.get_libraries(),
         library_dirs=JAVA.get_library_dirs(),
         include_dirs=JAVA.get_include_dirs(),