push sheeet

pkgs/development/python-modules/cython/0.nix

@@ -0,0 +1,106 @@
+{
+  lib,
+  stdenv,
+  buildPythonPackage,
+  fetchFromGitHub,
+  fetchpatch,
+  setuptools,
+  python,
+  pkg-config,
+  pythonAtLeast,
+  gdb,
+  numpy,
+  ncurses,
+}:
+
+let
+  excludedTests = [
+    "reimport_from_subinterpreter"
+  ]
+  # cython's testsuite is not working very well with libc++
+  # We are however optimistic about things outside of testsuite still working
+  ++ lib.optionals (stdenv.cc.isClang or false) [
+    "cpdef_extern_func"
+    "libcpp_algo"
+  ]
+  # Some tests in the test suite isn't working on aarch64. Disable them for
+  # now until upstream finds a workaround.
+  # Upstream issue here: https://github.com/cython/cython/issues/2308
+  ++ lib.optionals stdenv.hostPlatform.isAarch64 [ "numpy_memoryview" ]
+  ++ lib.optionals stdenv.hostPlatform.isi686 [
+    "future_division"
+    "overflow_check_longlong"
+  ];
+in
+buildPythonPackage rec {
+  pname = "cython";
+  version = "0.29.37.1";
+  pyproject = true;
+
+  # error: too few arguments to function '_PyLong_AsByteArray'
+  disabled = pythonAtLeast "3.13";
+
+  src = fetchFromGitHub {
+    owner = "cython";
+    repo = "cython";
+    rev = "refs/tags/${version}";
+    hash = "sha256-XsEy2NrG7hq+VXRCRbD4BRaBieU6mVoE0GT52L3mMhs=";
+  };
+
+  nativeBuildInputs = [
+    pkg-config
+    setuptools
+  ];
+
+  nativeCheckInputs = [
+    gdb
+    numpy
+    ncurses
+  ];
+
+  LC_ALL = "en_US.UTF-8";
+
+  patches = [
+    # backport Cython 3.0 trashcan support (https://github.com/cython/cython/pull/2842) to 0.X series.
+    # it does not affect Python code unless the code explicitly uses the feature.
+    # trashcan support is needed to avoid stack overflows during object deallocation in sage (https://trac.sagemath.org/ticket/27267)
+    ./trashcan.patch
+    # The above commit introduces custom trashcan macros, as well as
+    # compiler changes to use them in Cython-emitted code. The latter
+    # change is still useful, but the former has been upstreamed as of
+    # Python 3.8, and the patch below makes Cython use the upstream
+    # trashcan macros whenever available. This is needed for Python
+    # 3.11 support, because the API used in Cython's implementation
+    # changed: https://github.com/cython/cython/pull/4475
+    (fetchpatch {
+      name = "disable-trashcan.patch";
+      url = "https://github.com/cython/cython/commit/e337825cdcf5e94d38ba06a0cb0188e99ce0cc92.patch";
+      hash = "sha256-q0f63eetKrDpmP5Z4v8EuGxg26heSyp/62OYqhRoSso=";
+    })
+  ];
+
+  checkPhase = ''
+    export HOME="$NIX_BUILD_TOP"
+    ${python.interpreter} runtests.py -j$NIX_BUILD_CORES \
+      --no-code-style \
+      ${lib.optionalString (
+        builtins.length excludedTests != 0
+      ) ''--exclude="(${builtins.concatStringsSep "|" excludedTests})"''}
+  '';
+
+  # https://github.com/cython/cython/issues/2785
+  # Temporary solution
+  doCheck = false;
+  # doCheck = !stdenv.hostPlatform.isDarwin;
+
+  # force regeneration of generated code in source distributions
+  # https://github.com/cython/cython/issues/5089
+  setupHook = ./setup-hook.sh;
+
+  meta = {
+    changelog = "https://github.com/cython/cython/blob/${version}/CHANGES.rst";
+    description = "Optimising static compiler for both the Python programming language and the extended Cython programming language";
+    homepage = "https://cython.org";
+    license = lib.licenses.asl20;
+  };
+}

pkgs/development/python-modules/cython/default.nix

@@ -0,0 +1,128 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  gdb,
+  isPyPy,
+  ncurses,
+  numpy,
+  pkg-config,
+  pygame-ce,
+  python,
+  sage, # Reverse dependency
+  setuptools,
+  stdenv,
+}:
+
+buildPythonPackage rec {
+  pname = "cython";
+  version = "3.1.2";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "cython";
+    repo = "cython";
+    tag = version;
+    hash = "sha256-lP8ILCzAZuoPzFhCqGXwIpifN8XoWz93SJ7c3XVe69Y=";
+  };
+
+  build-system = [
+    pkg-config
+    setuptools
+  ];
+
+  nativeCheckInputs = [
+    gdb
+    numpy
+    ncurses
+  ];
+
+  env = lib.optionalAttrs (!isPyPy) {
+    LC_ALL = "en_US.UTF-8";
+  };
+
+  # https://github.com/cython/cython/issues/2785
+  # Temporary solution
+  doCheck = false;
+
+  strictDeps = true;
+
+  checkPhase =
+    let
+      excludedTests = [
+        "reimport_from_subinterpreter"
+      ]
+      # cython's testsuite is not working very well with libc++
+      # We are however optimistic about things outside of testsuite still working
+      ++ lib.optionals (stdenv.cc.isClang or false) [
+        "cpdef_extern_func"
+        "libcpp_algo"
+      ]
+      # Some tests in the test suite aren't working on aarch64.
+      # Disable them for now until upstream finds a workaround.
+      # Upstream issue: https://github.com/cython/cython/issues/2308
+      ++ lib.optionals stdenv.hostPlatform.isAarch64 [ "numpy_memoryview" ]
+      ++ lib.optionals stdenv.hostPlatform.isi686 [
+        "future_division"
+        "overflow_check_longlong"
+      ];
+      commandline = builtins.concatStringsSep " " (
+        [
+          "-j$NIX_BUILD_CORES"
+          "--no-code-style"
+        ]
+        ++ lib.optionals (builtins.length excludedTests != 0) [
+          ''--exclude="(${builtins.concatStringsSep "|" excludedTests})"''
+        ]
+      );
+    in
+    ''
+      runHook preCheck
+      export HOME="$NIX_BUILD_TOP"
+      ${python.interpreter} runtests.py ${commandline}
+      runHook postCheck
+    '';
+
+  passthru.tests = {
+    inherit pygame-ce sage;
+  };
+
+  # Force code regeneration in source distributions
+  # https://github.com/cython/cython/issues/5089
+  setupHook = ./setup-hook.sh;
+
+  meta = {
+    homepage = "https://cython.org";
+    description = "Optimising static compiler for both the Python and the extended Cython programming languages";
+    longDescription = ''
+      Cython is an optimising static compiler for both the Python programming
+      language and the extended Cython programming language (based on Pyrex). It
+      makes writing C extensions for Python as easy as Python itself.
+
+      Cython gives you the combined power of Python and C to let you:
+
+      - write Python code that calls back and forth from and to C or C++ code
+        natively at any point.
+      - easily tune readable Python code into plain C performance by adding
+        static type declarations, also in Python syntax.
+      - use combined source code level debugging to find bugs in your Python,
+        Cython and C code.
+      - interact efficiently with large data sets, e.g. using multi-dimensional
+        NumPy arrays.
+      - quickly build your applications within the large, mature and widely used
+        CPython ecosystem.
+      - integrate natively with existing code and data from legacy, low-level or
+        high-performance libraries and applications.
+
+      The Cython language is a superset of the Python language that additionally
+      supports calling C functions and declaring C types on variables and class
+      attributes. This allows the compiler to generate very efficient C code
+      from Cython code.
+    '';
+    changelog = "https://github.com/cython/cython/blob/${version}/CHANGES.rst";
+    license = lib.licenses.asl20;
+    mainProgram = "cython";
+    maintainers = [ ];
+  };
+}
+# TODO: investigate recursive loop when doCheck is true

pkgs/development/python-modules/cython/setup-hook.sh

@@ -0,0 +1,3 @@
+if [ -z "${dontForceRegenCython-}"]; then
+  export CYTHON_FORCE_REGEN=1
+fi

pkgs/development/python-modules/cython/trashcan.patch

@@ -0,0 +1,354 @@
+From 1b77e35d848340f2c5f4c9b82965c25a0572d48f Mon Sep 17 00:00:00 2001
+From: Jeroen Demeyer <J.Demeyer@UGent.be>
+Date: Thu, 14 Feb 2019 10:02:41 +0100
+Subject: [PATCH] @cython.trashcan directive to enable the Python trashcan for
+ deallocations
+
+---
+ Cython/Compiler/ModuleNode.py   |  10 +++
+ Cython/Compiler/Options.py      |   2 +
+ Cython/Compiler/PyrexTypes.py   |   8 +-
+ Cython/Compiler/Symtab.py       |  18 +++-
+ Cython/Utility/ExtensionTypes.c |  43 ++++++++++
+ tests/run/trashcan.pyx          | 148 ++++++++++++++++++++++++++++++++
+ 6 files changed, 227 insertions(+), 2 deletions(-)
+ create mode 100644 tests/run/trashcan.pyx
+
+diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
+index 56845330d..3a3e8a956 100644
+--- a/Cython/Compiler/ModuleNode.py
++++ b/Cython/Compiler/ModuleNode.py
+@@ -1443,6 +1443,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
+ 
+         is_final_type = scope.parent_type.is_final_type
+         needs_gc = scope.needs_gc()
++        needs_trashcan = scope.needs_trashcan()
+ 
+         weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None
+         if weakref_slot not in scope.var_entries:
+@@ -1481,6 +1482,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
+             # running this destructor.
+             code.putln("PyObject_GC_UnTrack(o);")
+ 
++        if needs_trashcan:
++            code.globalstate.use_utility_code(
++                UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c"))
++            code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname)
++
+         # call the user's __dealloc__
+         self.generate_usr_dealloc_call(scope, code)
+ 
+@@ -1554,6 +1560,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
+             code.putln("(*Py_TYPE(o)->tp_free)(o);")
+             if freelist_size:
+                 code.putln("}")
++
++        if needs_trashcan:
++            code.putln("__Pyx_TRASHCAN_END")
++
+         code.putln(
+             "}")
+ 
+diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py
+index d03119fca..05a728135 100644
+--- a/Cython/Compiler/Options.py
++++ b/Cython/Compiler/Options.py
+@@ -319,6 +319,7 @@ directive_types = {
+     'freelist': int,
+     'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'),
+     'c_string_encoding': normalise_encoding_name,
++    'trashcan': bool,
+     'cpow': bool
+ }
+ 
+@@ -362,6 +363,7 @@ directive_scopes = {  # defaults to available everywhere
+     'np_pythran': ('module',),
+     'fast_gil': ('module',),
+     'iterable_coroutine': ('module', 'function'),
++    'trashcan' : ('cclass',),
+ }
+ 
+ 
+diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
+index c309bd04b..9231130b5 100644
+--- a/Cython/Compiler/PyrexTypes.py
++++ b/Cython/Compiler/PyrexTypes.py
+@@ -1129,6 +1129,7 @@ class PyObjectType(PyrexType):
+     is_extern = False
+     is_subclassed = False
+     is_gc_simple = False
++    builtin_trashcan = False  # builtin type using trashcan
+ 
+     def __str__(self):
+         return "Python object"
+@@ -1183,10 +1184,14 @@ class PyObjectType(PyrexType):
+ 
+ 
+ builtin_types_that_cannot_create_refcycles = set([
+-    'bool', 'int', 'long', 'float', 'complex',
++    'object', 'bool', 'int', 'long', 'float', 'complex',
+     'bytearray', 'bytes', 'unicode', 'str', 'basestring'
+ ])
+ 
++builtin_types_with_trashcan = set([
++    'dict', 'list', 'set', 'frozenset', 'tuple', 'type',
++])
++
+ 
+ class BuiltinObjectType(PyObjectType):
+     #  objstruct_cname  string           Name of PyObject struct
+@@ -1211,6 +1216,7 @@ class BuiltinObjectType(PyObjectType):
+         self.typeptr_cname = "(&%s)" % cname
+         self.objstruct_cname = objstruct_cname
+         self.is_gc_simple = name in builtin_types_that_cannot_create_refcycles
++        self.builtin_trashcan = name in builtin_types_with_trashcan
+         if name == 'type':
+             # Special case the type type, as many C API calls (and other
+             # libraries) actually expect a PyTypeObject* for type arguments.
+diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py
+index 7361a55ae..f0c311ba6 100644
+--- a/Cython/Compiler/Symtab.py
++++ b/Cython/Compiler/Symtab.py
+@@ -2043,7 +2043,7 @@ class PyClassScope(ClassScope):
+ class CClassScope(ClassScope):
+     #  Namespace of an extension type.
+     #
+-    #  parent_type           CClassType
++    #  parent_type           PyExtensionType
+     #  #typeobj_cname        string or None
+     #  #objstruct_cname      string
+     #  method_table_cname    string
+@@ -2087,6 +2087,22 @@ class CClassScope(ClassScope):
+             return not self.parent_type.is_gc_simple
+         return False
+ 
++    def needs_trashcan(self):
++        # If the trashcan directive is explicitly set to False,
++        # unconditionally disable the trashcan.
++        directive = self.directives.get('trashcan')
++        if directive is False:
++            return False
++        # If the directive is set to True and the class has Python-valued
++        # C attributes, then it should use the trashcan in tp_dealloc.
++        if directive and self.has_cyclic_pyobject_attrs:
++            return True
++        # Use the trashcan if the base class uses it
++        base_type = self.parent_type.base_type
++        if base_type and base_type.scope is not None:
++            return base_type.scope.needs_trashcan()
++        return self.parent_type.builtin_trashcan
++
+     def needs_tp_clear(self):
+         """
+         Do we need to generate an implementation for the tp_clear slot? Can
+diff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.c
+index dc187ab49..f359165df 100644
+--- a/Cython/Utility/ExtensionTypes.c
++++ b/Cython/Utility/ExtensionTypes.c
+@@ -119,6 +119,49 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) {
+     return r;
+ }
+ 
++/////////////// PyTrashcan.proto ///////////////
++
++// These macros are taken from https://github.com/python/cpython/pull/11841
++// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they
++// allow dealing correctly with subclasses.
++
++// This requires CPython version >= 2.7.4
++// (or >= 3.2.4 but we don't support such old Python 3 versions anyway)
++#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400
++#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \
++    do { \
++        PyThreadState *_tstate = NULL; \
++        // If "cond" is false, then _tstate remains NULL and the deallocator
++        // is run normally without involving the trashcan
++        if (cond) { \
++            _tstate = PyThreadState_GET(); \
++            if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \
++                // Store the object (to be deallocated later) and jump past
++                // Py_TRASHCAN_END, skipping the body of the deallocator
++                _PyTrash_thread_deposit_object((PyObject*)(op)); \
++                break; \
++            } \
++            ++_tstate->trash_delete_nesting; \
++        }
++        // The body of the deallocator is here.
++#define __Pyx_TRASHCAN_END \
++        if (_tstate) { \
++            --_tstate->trash_delete_nesting; \
++            if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \
++                _PyTrash_thread_destroy_chain(); \
++        } \
++    } while (0);
++
++#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \
++        Py_TYPE(op)->tp_dealloc == (destructor)(dealloc))
++
++#else
++// The trashcan is a no-op on other Python implementations
++// or old CPython versions
++#define __Pyx_TRASHCAN_BEGIN(op, dealloc)
++#define __Pyx_TRASHCAN_END
++#endif
++
+ /////////////// CallNextTpDealloc.proto ///////////////
+ 
+ static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc);
+diff --git a/tests/run/trashcan.pyx b/tests/run/trashcan.pyx
+new file mode 100644
+index 000000000..93a501ff8
+--- /dev/null
++++ b/tests/run/trashcan.pyx
+@@ -0,0 +1,148 @@
++# mode: run
++
++cimport cython
++
++
++# Count number of times an object was deallocated twice. This should remain 0.
++cdef int double_deallocations = 0
++def assert_no_double_deallocations():
++    global double_deallocations
++    err = double_deallocations
++    double_deallocations = 0
++    assert not err
++
++
++# Compute x = f(f(f(...(None)...))) nested n times and throw away the result.
++# The real test happens when exiting this function: then a big recursive
++# deallocation of x happens. We are testing two things in the tests below:
++# that Python does not crash and that no double deallocation happens.
++# See also https://github.com/python/cpython/pull/11841
++def recursion_test(f, int n=2**20):
++    x = None
++    cdef int i
++    for i in range(n):
++        x = f(x)
++
++
++@cython.trashcan(True)
++cdef class Recurse:
++    """
++    >>> recursion_test(Recurse)
++    >>> assert_no_double_deallocations()
++    """
++    cdef public attr
++    cdef int deallocated
++
++    def __init__(self, x):
++        self.attr = x
++
++    def __dealloc__(self):
++        # Check that we're not being deallocated twice
++        global double_deallocations
++        double_deallocations += self.deallocated
++        self.deallocated = 1
++
++
++cdef class RecurseSub(Recurse):
++    """
++    >>> recursion_test(RecurseSub)
++    >>> assert_no_double_deallocations()
++    """
++    cdef int subdeallocated
++
++    def __dealloc__(self):
++        # Check that we're not being deallocated twice
++        global double_deallocations
++        double_deallocations += self.subdeallocated
++        self.subdeallocated = 1
++
++
++@cython.freelist(4)
++@cython.trashcan(True)
++cdef class RecurseFreelist:
++    """
++    >>> recursion_test(RecurseFreelist)
++    >>> recursion_test(RecurseFreelist, 1000)
++    >>> assert_no_double_deallocations()
++    """
++    cdef public attr
++    cdef int deallocated
++
++    def __init__(self, x):
++        self.attr = x
++
++    def __dealloc__(self):
++        # Check that we're not being deallocated twice
++        global double_deallocations
++        double_deallocations += self.deallocated
++        self.deallocated = 1
++
++
++# Subclass of list => uses trashcan by default
++# As long as https://github.com/python/cpython/pull/11841 is not fixed,
++# this does lead to double deallocations, so we skip that check.
++cdef class RecurseList(list):
++    """
++    >>> RecurseList(42)
++    [42]
++    >>> recursion_test(RecurseList)
++    """
++    def __init__(self, x):
++        super().__init__((x,))
++
++
++# Some tests where the trashcan is NOT used. When the trashcan is not used
++# in a big recursive deallocation, the __dealloc__s of the base classs are
++# only run after the __dealloc__s of the subclasses.
++# We use this to detect trashcan usage.
++cdef int base_deallocated = 0
++cdef int trashcan_used = 0
++def assert_no_trashcan_used():
++    global base_deallocated, trashcan_used
++    err = trashcan_used
++    trashcan_used = base_deallocated = 0
++    assert not err
++
++
++cdef class Base:
++    def __dealloc__(self):
++        global base_deallocated
++        base_deallocated = 1
++
++
++# Trashcan disabled by default
++cdef class Sub1(Base):
++    """
++    >>> recursion_test(Sub1, 100)
++    >>> assert_no_trashcan_used()
++    """
++    cdef public attr
++
++    def __init__(self, x):
++        self.attr = x
++
++    def __dealloc__(self):
++        global base_deallocated, trashcan_used
++        trashcan_used += base_deallocated
++
++
++@cython.trashcan(True)
++cdef class Middle(Base):
++    cdef public foo
++
++
++# Trashcan disabled explicitly
++@cython.trashcan(False)
++cdef class Sub2(Middle):
++    """
++    >>> recursion_test(Sub2, 1000)
++    >>> assert_no_trashcan_used()
++    """
++    cdef public attr
++
++    def __init__(self, x):
++        self.attr = x
++
++    def __dealloc__(self):
++        global base_deallocated, trashcan_used
++        trashcan_used += base_deallocated
+-- 
+2.39.0
+