commit 711b9979b731745dbeff3dd47ea67eedf8603449 Author: Blake Griffith Date: Sat Aug 31 16:02:05 2013 -0500 DOC: Update release notes. diff --git a/doc/release/1.9.0-notes.rst b/doc/release/1.9.0-notes.rst index 7d0a7b9..0b1ab77 100644 --- a/doc/release/1.9.0-notes.rst +++ b/doc/release/1.9.0-notes.rst @@ -23,6 +23,14 @@ Compatibility notes New Features ============ +Ufunc and Dot Overrides +~~~~~~~~~~~~~~~~~~~~~~~ + +For better compatibility with external objects you can now override universal +functions (ufuncs), ``numpy.core._dotblas.dot``, and +``numpy.core.multiarray.dot`` (the numpy.dot functions). By defining a +``__numpy_ufunc__`` method. + Improvements ============ commit 332057144d91e227f4e619e6dc8d8701d23a9ec5 Author: Blake Griffith Date: Thu Aug 15 12:35:27 2013 -0500 BLD TRAVIS: Added libatlas to travis build so it can test BLAS stuff. sudo apt-get install -qq libatlas-dev libatlas-base-dev diff --git a/.travis.yml b/.travis.yml index e3829de..38b1ec0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,9 @@ before_install: - pip install nose # pip install coverage - python -V + - sudo apt-get install -qq libatlas-dev libatlas-base-dev - popd + install: # We used to use 'setup.py install' here, but that has the terrible # behaviour that if a copy of the package is already installed in commit 022aea859a01a0f11372b28267a74aff9b8880f9 Author: Blake Griffith Date: Thu Jul 18 21:26:58 2013 -0500 BLD: Added ufunc_override.h to setup.py. diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 3937b23..1c8cea4 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -881,7 +881,8 @@ def configuration(parent_package='',top_path=None): umath_deps = [ generate_umath_py, join('src', 'umath', 'simd.inc.src'), - join(codegen_dir, 'generate_ufunc_api.py')] + join(codegen_dir, 'generate_ufunc_api.py'), + join('src', 'private', 'ufunc_override.h')] if not ENABLE_SEPARATE_COMPILATION: umath_deps.extend(umath_src) commit 5c630f06f9012cdfff9b35cf96cea4a696c6b66c Author: Blake Griffith Date: Mon Jul 22 23:22:55 2013 -0500 TST: Add ufunc override tests. diff --git a/numpy/core/tests/test_blasdot.py b/numpy/core/tests/test_blasdot.py index 624c617..caa576a 100644 --- a/numpy/core/tests/test_blasdot.py +++ b/numpy/core/tests/test_blasdot.py @@ -151,3 +151,21 @@ def test_dot_array_order(): assert_almost_equal(c.T.dot(b.T).T, b.dot(c), decimal=prec) assert_almost_equal(b.dot(c), _dot(b, c), decimal=prec) assert_almost_equal(c.T.dot(b.T), _dot(c.T, b.T), decimal=prec) + +def test_dot_override(): + class A(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + return "A" + + class B(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + return NotImplemented + + a = A() + b = B() + c = np.array([[1]]) + + assert_equal(np.dot(a, b), "A") + assert_equal(c.dot(a), "A") + assert_raises(TypeError, np.dot, b, c) + assert_raises(TypeError, c.dot, b) diff --git a/numpy/core/tests/test_multiarray.py b/numpy/core/tests/test_multiarray.py index 9870d44..0e7f6ec 100644 --- a/numpy/core/tests/test_multiarray.py +++ b/numpy/core/tests/test_multiarray.py @@ -1324,6 +1324,24 @@ class TestMethods(TestCase): a.dot(b=b, out=c) assert_equal(c, np.dot(a, b)) + def test_dot_override(self): + class A(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + return "A" + + class B(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + return NotImplemented + + a = A() + b = B() + c = np.array([[1]]) + + assert_equal(np.dot(a, b), "A") + assert_equal(c.dot(a), "A") + assert_raises(TypeError, np.dot, b, c) + assert_raises(TypeError, c.dot, b) + def test_diagonal(self): a = np.arange(12).reshape((3, 4)) assert_equal(a.diagonal(), [0, 5, 10]) diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index eb18304..3706cfa 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -868,6 +868,187 @@ class TestSpecialMethods(TestCase): assert_equal(ncu.maximum(a, B()), 0) assert_equal(ncu.maximum(a, C()), 0) + def test_ufunc_override(self): + class A(object): + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + return self, func, method, pos, inputs, kwargs + + a = A() + + b = np.matrix([1]) + c = np.array([1]) + res0 = np.multiply(a, b) + res1 = np.dot(a, b) + + # self + assert_equal(res0[0], a) + assert_equal(res1[0], a) + assert_equal(res0[1], np.multiply) + assert_equal(res1[1], np.dot) + assert_equal(res0[2], '__call__') + assert_equal(res1[2], '__call__') + assert_equal(res0[3], 0) + assert_equal(res1[3], 0) + assert_equal(res0[4], (a, b)) + assert_equal(res1[4], (a, b)) + assert_equal(res0[5], {}) + assert_equal(res1[5], {}) + + def test_ufunc_override_mro(self): + + # Some multi arg functions for testing. + def tres_mul(a, b, c): + return a * b * c + + def quatro_mul(a, b, c, d): + return a * b * c * d + + # Make these into ufuncs. + three_mul_ufunc = np.frompyfunc(tres_mul, 3, 1) + four_mul_ufunc = np.frompyfunc(quatro_mul, 4, 1) + + class A(object): + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + return "A" + + class ASub(A): + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + return "ASub" + + class B(object): + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + return "B" + + class C(object): + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + return NotImplemented + + class CSub(object): + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + return NotImplemented + + + + a = A() + a_sub = ASub() + b = B() + c = C() + c_sub = CSub() + + # Standard + res = np.multiply(a, a_sub) + assert_equal(res, "ASub") + res = np.multiply(a_sub, b) + assert_equal(res, "ASub") + + # With 1 NotImplemented + res = np.multiply(c, a) + assert_equal(res, "A") + + # Both NotImplemented. + assert_raises(TypeError, np.multiply, c, c_sub) + assert_raises(TypeError, np.multiply, c_sub, c) + assert_raises(TypeError, np.multiply, 2, c) + + # Ternary testing. + assert_equal(three_mul_ufunc(a, 1, 2), "A") + assert_equal(three_mul_ufunc(1, a, 2), "A") + assert_equal(three_mul_ufunc(1, 2, a), "A") + + assert_equal(three_mul_ufunc(a, a, 6), "A") + assert_equal(three_mul_ufunc(a, 2, a), "A") + assert_equal(three_mul_ufunc(a, 2, b), "A") + assert_equal(three_mul_ufunc(a, 2, a_sub), "ASub") + assert_equal(three_mul_ufunc(a, a_sub, 3), "ASub") + assert_equal(three_mul_ufunc(c, a_sub, 3), "ASub") + assert_equal(three_mul_ufunc(1, a_sub, c), "ASub") + + assert_equal(three_mul_ufunc(a, b, c), "A") + assert_equal(three_mul_ufunc(a, b, c_sub), "A") + assert_equal(three_mul_ufunc(1, 2, b), "B") + + assert_raises(TypeError, three_mul_ufunc, 1, 2, c) + assert_raises(TypeError, three_mul_ufunc, c_sub, 2, c) + assert_raises(TypeError, three_mul_ufunc, c_sub, 2, 3) + + # Quaternary testing. + assert_equal(four_mul_ufunc(a, 1, 2, 3), "A") + assert_equal(four_mul_ufunc(1, a, 2, 3), "A") + assert_equal(four_mul_ufunc(1, 1, a, 3), "A") + assert_equal(four_mul_ufunc(1, 1, 2, a), "A") + + assert_equal(four_mul_ufunc(a, b, 2, 3), "A") + assert_equal(four_mul_ufunc(1, a, 2, b), "A") + assert_equal(four_mul_ufunc(b, 1, a, 3), "B") + assert_equal(four_mul_ufunc(a_sub, 1, 2, a), "ASub") + assert_equal(four_mul_ufunc(a, 1, 2, a_sub), "ASub") + + assert_raises(TypeError, four_mul_ufunc, 1, 2, 3, c) + assert_raises(TypeError, four_mul_ufunc, 1, 2, c_sub, c) + assert_raises(TypeError, four_mul_ufunc, 1, c, c_sub, c) + + def test_ufunc_override_methods(self): + class A(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + if method == "__call__": + return method + if method == "reduce": + return method + if method == "accumulate": + return method + if method == "reduceat": + return method + + a = A() + res = np.multiply(1, a) + assert_equal(res, "__call__") + + res = np.multiply.reduce(1, a) + assert_equal(res, "reduce") + + res = np.multiply.accumulate(1, a) + assert_equal(res, "accumulate") + + res = np.multiply.reduceat(1, a) + assert_equal(res, "reduceat") + + res = np.multiply(a, 1) + assert_equal(res, "__call__") + + res = np.multiply.reduce(a, 1) + assert_equal(res, "reduce") + + res = np.multiply.accumulate(a, 1) + assert_equal(res, "accumulate") + + res = np.multiply.reduceat(a, 1) + assert_equal(res, "reduceat") + + def test_ufunc_override_out(self): + class A(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + return kwargs + + + class B(object): + def __numpy_ufunc__(self, ufunc, method, pos, inputs, **kwargs): + return kwargs + + a = A() + b = B() + res0 = np.multiply(a, b, 'out_arg') + res1 = np.multiply(a, b, out='out_arg') + res2 = np.multiply(2, b, 'out_arg') + res3 = np.multiply(3, b, out='out_arg') + res4 = np.multiply(a, 4, 'out_arg') + res5 = np.multiply(a, 5, out='out_arg') + + assert_equal(res0['out'], 'out_arg') + assert_equal(res1['out'], 'out_arg') + assert_equal(res2['out'], 'out_arg') + assert_equal(res3['out'], 'out_arg') + assert_equal(res4['out'], 'out_arg') + assert_equal(res5['out'], 'out_arg') class TestChoose(TestCase): def test_mixed(self): commit 21976ca31eda537824ad877d432634eaf103567b Author: Blake Griffith Date: Sun Aug 25 11:44:53 2013 -0500 ENH: Add ufunc override functionality to ufuncs and dots. diff --git a/numpy/core/blasdot/_dotblas.c b/numpy/core/blasdot/_dotblas.c index ae6b1b1..7a9f858 100644 --- a/numpy/core/blasdot/_dotblas.c +++ b/numpy/core/blasdot/_dotblas.c @@ -2,11 +2,14 @@ * This module provides a BLAS optimized\nmatrix multiply, * inner product and dot for numpy arrays */ -#define NPY_NO_DEPRECATED_API NPY_API_VERSION +#define NPY_NO_DEPRECATED_API NPY_API_VERSION #include "Python.h" -#include "npy_config.h" + #include "numpy/arrayobject.h" +#include "npy_config.h" +#include "npy_pycompat.h" +#include "private/ufunc_override.h" #ifndef CBLAS_HEADER #define CBLAS_HEADER "cblas.h" #endif @@ -215,8 +218,12 @@ _bad_strides(PyArrayObject *ap) static PyObject * dotblas_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwargs) { + static PyObject *cached_npy_dot = NULL; + PyObject *override = NULL; + PyObject *module; PyObject *op1, *op2; PyArrayObject *ap1 = NULL, *ap2 = NULL, *out = NULL, *ret = NULL; + int errval; int j, l, lda, ldb, ldc; int typenum, nd; npy_intp ap1stride = 0; @@ -232,6 +239,23 @@ dotblas_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwa MatrixShape ap1shape, ap2shape; char* kwords[] = {"a", "b", "out", NULL }; + if (cached_npy_dot == NULL) { + module = PyImport_ImportModule("numpy.core._dotblas"); + cached_npy_dot = PyDict_GetItemString(PyModule_GetDict(module), "dot"); + + Py_INCREF(cached_npy_dot); + Py_DECREF(module); + } + + errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", args, kwargs, + &override, 2); + if (errval) { + return NULL; + } + else if (override) { + return override; + } + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|O", kwords, &op1, &op2, &out)) { return NULL; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 85dd8ab..f4ceecd 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -53,6 +53,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "ctors.h" #include "array_assign.h" #include "common.h" +#include "private/ufunc_override.h" /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; @@ -2079,8 +2080,29 @@ array_innerproduct(PyObject *NPY_UNUSED(dummy), PyObject *args) static PyObject * array_matrixproduct(PyObject *NPY_UNUSED(dummy), PyObject *args, PyObject* kwds) { + int errval; + static PyObject *cached_npy_dot = NULL; + PyObject *override = NULL; PyObject *v, *a, *o = NULL; char* kwlist[] = {"a", "b", "out", NULL }; + PyObject *module; + + if (cached_npy_dot == NULL) { + module = PyImport_ImportModule("numpy.core.multiarray"); + cached_npy_dot = PyDict_GetItemString(PyModule_GetDict(module), "dot"); + + Py_INCREF(cached_npy_dot); + Py_DECREF(module); + } + + errval = PyUFunc_CheckOverride(cached_npy_dot, "__call__", args, kwds, + &override, 2); + if (errval) { + return NULL; + } + else if (override) { + return override; + } if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|O", kwlist, &a, &v, &o)) { return NULL; diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index a71777c..062bf16 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -44,6 +44,7 @@ #include "reduction.h" #include "ufunc_object.h" +#include "ufunc_override.h" /********** PRINTF DEBUG TRACING **************/ #define NPY_UF_DBG_TRACING 0 @@ -707,7 +708,6 @@ fail: return -1; } - /********* GENERIC UFUNC USING ITERATOR *********/ /* @@ -4043,6 +4043,7 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) PyObject *retobj[NPY_MAXARGS]; PyObject *wraparr[NPY_MAXARGS]; PyObject *res; + PyObject *override = NULL; int errval; /* @@ -4053,6 +4054,18 @@ ufunc_generic_call(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) mps[i] = NULL; } + errval = PyUFunc_CheckOverride(ufunc, "__call__", args, kwds, &override, + ufunc->nin); + if (errval) { + return NULL; + } + else if (override) { + for (i = 0; i < ufunc->nargs; i++) { + PyArray_XDECREF_ERR(mps[i]); + } + return override; + } + errval = PyUFunc_GenericFunction(ufunc, args, kwds, mps); if (errval < 0) { for (i = 0; i < ufunc->nargs; i++) { @@ -4834,18 +4847,51 @@ ufunc_outer(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) static PyObject * ufunc_reduce(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) { + int errval; + PyObject *override = NULL; + + errval = PyUFunc_CheckOverride(ufunc, "reduce", args, kwds, &override, + ufunc->nin); + if (errval) { + return NULL; + } + else if (override) { + return override; + } return PyUFunc_GenericReduction(ufunc, args, kwds, UFUNC_REDUCE); } static PyObject * ufunc_accumulate(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) { + int errval; + PyObject *override = NULL; + + errval = PyUFunc_CheckOverride(ufunc, "accumulate", args, kwds, &override, + ufunc->nin); + if (errval) { + return NULL; + } + else if (override) { + return override; + } return PyUFunc_GenericReduction(ufunc, args, kwds, UFUNC_ACCUMULATE); } static PyObject * ufunc_reduceat(PyUFuncObject *ufunc, PyObject *args, PyObject *kwds) { + int errval; + PyObject *override = NULL; + + errval = PyUFunc_CheckOverride(ufunc, "reduceat", args, kwds, &override, + ufunc->nin); + if (errval) { + return NULL; + } + else if (override) { + return override; + } return PyUFunc_GenericReduction(ufunc, args, kwds, UFUNC_REDUCEAT); } commit 536cd36c1ba45c42ad306fda5f9c4d12ee0f5afd Author: Blake Griffith Date: Sun Aug 25 11:44:14 2013 -0500 ENH: Add ufunc override API. diff --git a/numpy/core/src/private/ufunc_override.h b/numpy/core/src/private/ufunc_override.h new file mode 100644 index 0000000..5c3cbdb --- /dev/null +++ b/numpy/core/src/private/ufunc_override.h @@ -0,0 +1,195 @@ +#ifndef __UFUNC_OVERRIDE_H +#define __UFUNC_OVERRIDE_H +#include +#include "numpy/arrayobject.h" +#include "multiarray/common.h" + +/* + * Check a set of args for the `__numpy_ufunc__` method. If more than one of + * the input arguments implements `__numpy_ufunc__`, they are tried in the + * order: subclasses before superclasses, otherwise left to right. The first + * routine returning something other than `NotImplemented` determines the + * result. If all of the `__numpy_ufunc__` operations returns `NotImplemented`, + * a `TypeError` is raised. + */ +static int +PyUFunc_CheckOverride(PyObject *ufunc, char *method, + PyObject *args, PyObject *kwds, + PyObject **result, + int nin) +{ + int i; + int override_pos; /* Position of override in args.*/ + int j; + int pos_in_with_override; /* Position of override in with_override.*/ + + int nargs = PyTuple_GET_SIZE(args); + int noa = 0; /* Number of overriding args.*/ + int normalized = 0; /* Is normalized flag.*/ + + PyObject *obj; + PyObject *other_obj; + PyObject *override_args; + + PyObject *method_name = PyUString_FromString(method); + PyObject *normal_args = NULL; /* normal_* holds normalized arguments. */ + PyObject *normal_kwds = NULL; + PyObject *override_obj = NULL; /* overriding object */ + PyObject *numpy_ufunc = NULL; /* the __numpy_ufunc__ method */ + + PyObject *with_override[NPY_MAXARGS]; + /* Pos of each override in args */ + int with_override_pos[NPY_MAXARGS]; + + /* Checks */ + if (!PyTuple_Check(args)) { + goto fail; + } + if (PyTuple_GET_SIZE(args) > NPY_MAXARGS) { + goto fail; + } + + for (i = 0; i < nargs; ++i) { + obj = PyTuple_GET_ITEM(args, i); + if (PyArray_CheckExact(obj) || PyArray_IsAnyScalar(obj)) { + continue; + } + if (PyObject_HasAttrString(obj, "__numpy_ufunc__")) { + with_override[noa] = obj; + with_override_pos[noa] = i; + ++noa; + } + } + + /* No overrides, bail out.*/ + if (noa == 0) { + Py_DECREF(method_name); + return 0; + } + + while (1) { + obj = NULL; + override_obj = NULL; + *result = NULL; + + /* Choose an overriding argument */ + for (i = 0; i < noa; i++) { + obj = with_override[i]; + if (obj == NULL) { + continue; + } + /* Get the first instance of an overriding arg.*/ + override_pos = with_override_pos[i]; + override_obj = obj; + pos_in_with_override = i; + + /* Check for sub-types to the right of obj. */ + for (j = i + 1; j < noa; j++) { + other_obj = with_override[j]; + if (PyObject_Type(other_obj) != PyObject_Type(obj) && + PyObject_IsInstance(other_obj, + PyObject_Type(override_obj))) { + override_obj = NULL; + break; + } + } + /* override_obj had no subtypes to the right. */ + if (override_obj) { + break; + } + } + /* No good override_obj */ + if (!override_obj) { + break; + } + /* + * Normalize the ufuncs arguments. Returns a tuple of + * (args, kwds). + * + * Test with and without kwds. + */ + if (!normalized) { + PyObject *out_arg; + + /* If we have more args than nin, the last one must be `out`.*/ + if (nargs > nin) { + out_arg = PyTuple_GET_ITEM(args, nargs - 1); + + /* Build new args.*/ + normal_args = PyTuple_GetSlice(args, 0, nin); + + /* Build new kwds with out arg.*/ + if (kwds && PyDict_CheckExact(kwds)) { + normal_kwds = PyDict_Copy(kwds); + PyDict_SetItemString(normal_kwds, "out", out_arg); + } + else { + normal_kwds = PyDict_New(); + PyDict_SetItemString(normal_kwds, "out", out_arg); + } + + normalized = 1; + } + else { + /* Copy args */ + normal_args = PyTuple_GetSlice(args, 0, nin); + if (kwds && PyDict_CheckExact(kwds)) { + normal_kwds = PyDict_Copy(kwds); + } + else { + normal_kwds = PyDict_New(); + } + + normalized = 1; + } + } + + /* Calculate a result if we have a override. */ + if (override_obj) { + numpy_ufunc = PyObject_GetAttrString(override_obj, + "__numpy_ufunc__"); + override_args = Py_BuildValue("OOiO", ufunc, method_name, + override_pos, normal_args); + *result = PyObject_Call(numpy_ufunc, override_args, normal_kwds); + + Py_DECREF(numpy_ufunc); + Py_DECREF(override_args); + + /* Remove this arg if it gives not implemented */ + if (*result == Py_NotImplemented) { + with_override[pos_in_with_override] = NULL; + Py_DECREF(*result); + continue; + } + /* Good result. */ + else { + break; + } + } + + /* All overrides checked. */ + else { + break; + } + } + /* No acceptable override found. */ + if (!*result) { + PyErr_SetString(PyExc_TypeError, + "__numpy_ufunc__ not implemented for this type."); + Py_XDECREF(normal_args); + Py_XDECREF(normal_kwds); + goto fail; + } + /* Override found, return it. */ + Py_DECREF(method_name); + Py_XDECREF(normal_args); + Py_XDECREF(normal_kwds); + return 0; + +fail: + Py_DECREF(method_name); + return 1; + +} + +#endif commit 6fe8eb607127b554195ed25f8636f5caefd477c3 Author: Blake Griffith Date: Sun Aug 25 09:35:06 2013 -0500 DOC: Add NEP and documentation for ufunc overrides. diff --git a/doc/neps/ufunc-overrides.rst b/doc/neps/ufunc-overrides.rst new file mode 100644 index 0000000..1c0ab1c --- /dev/null +++ b/doc/neps/ufunc-overrides.rst @@ -0,0 +1,242 @@ +================================= +A Mechanism for Overriding Ufuncs +================================= + +:Author: Blake Griffith +:Contact: blake.g@utexa.edu +:Date: 2013-07-10 + +:Author: Pauli Virtanen + +:Author: Nathaniel Smith + + +Executive summary +================= + +NumPy's universal functions (ufuncs) currently have some limited +functionality for operating on user defined subclasses of ndarray using +``__array_prepare__`` and ``__array_wrap__`` [1]_, and there is little +to no support for arbitrary objects. e.g. SciPy's sparse matrices [2]_ +[3]_. + +Here we propose adding a mechanism to override ufuncs based on the ufunc +checking each of it's arguments for a ``__numpy_ufunc__`` method. +On discovery of ``__numpy_ufunc__`` the ufunc will hand off the +operation to the method. + +This covers some of the same ground as Travis Oliphant's proposal to +retro-fit NumPy with multi-methods [4]_, which would solve the same +problem. The mechanism here follows more closely the way Python enables +classes to override ``__mul__`` and other binary operations. + +.. [1] http://docs.scipy.org/doc/numpy/user/basics.subclassing.html +.. [2] https://github.com/scipy/scipy/issues/2123 +.. [3] https://github.com/scipy/scipy/issues/1569 +.. [4] http://technicaldiscovery.blogspot.com/2013/07/thoughts-after-scipy-2013-and-specific.html + + +Motivation +========== + +The current machinery for dispatching Ufuncs is generally agreed to be +insufficient. There have been lengthy discussions and other proposed +solutions [5]_. + +Using ufuncs with subclasses of ndarray is limited to ``__array_prepare__`` and +``__array_wrap__`` to prepare the arguments, but these don't allow you to for +example change the shape or the data of the arguments. Trying to ufunc things +that don't subclass ndarray is even more difficult, as the input arguments tend +to be cast to object arrays, which ends up producing surprising results. + +Take this example of ufuncs interoperability with sparse matrices.:: + + In [1]: import numpy as np + import scipy.sparse as sp + + a = np.random.randint(5, size=(3,3)) + b = np.random.randint(5, size=(3,3)) + + asp = sp.csr_matrix(a) + bsp = sp.csr_matrix(b) + + In [2]: a, b + Out[2]:(array([[0, 4, 4], + [1, 3, 2], + [1, 3, 1]]), + array([[0, 1, 0], + [0, 0, 1], + [4, 0, 1]])) + + In [3]: np.multiply(a, b) # The right answer + Out[3]: array([[0, 4, 0], + [0, 0, 2], + [4, 0, 1]]) + + In [4]: np.multiply(asp, bsp).todense() # calls __mul__ which does matrix multi + Out[4]: matrix([[16, 0, 8], + [ 8, 1, 5], + [ 4, 1, 4]], dtype=int64) + + In [5]: np.multiply(a, bsp) # Returns NotImplemented to user, bad! + Out[5]: NotImplemted + +Returning ``NotImplemented`` to user should not happen. Moreover:: + + In [6]: np.multiply(asp, b) + Out[6]: array([[ <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>, + <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>, + <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>], + [ <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>, + <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>, + <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>], + [ <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>, + <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>, + <3x3 sparse matrix of type '' + with 8 stored elements in Compressed Sparse Row format>]], dtype=object) + +Here, it appears that the sparse matrix was converted to a object array +scalar, which was then multiplied with all elements of the ``b`` array. +However, this behavior is more confusing than useful, and having a +``TypeError`` would be preferable. + +Adding the ``__numpy_ufunc__`` functionality fixes this and would +deprecate the other ufunc modifying functions. + +.. [5] http://mail.scipy.org/pipermail/numpy-discussion/2011-June/056945.html + + +Proposed interface +================== + +Objects that want to override Ufuncs can define a ``__numpy_ufunc__`` method. +The method signature is:: + + def __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs) + +Here: + +- *ufunc* is the ufunc object that was called. +- *method* is a string indicating which Ufunc method was called + (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, + ``"accumulate"``, ``"outer"``, ``"inner"``). +- *i* is the index of *self* in *inputs*. +- *inputs* is a tuple of the input arguments to the ``ufunc`` +- *kwargs* are the keyword arguments passed to the function. The ``out`` + argument is always contained in *kwargs*, if given. + +The ufunc's arguments are first normalized into a tuple of input data +(``inputs``), and dict of keyword arguments. If the output argument is +passed as a positional argument it is moved to the keyword argmunets. + +The function dispatch proceeds as follows: + +- If one of the input arguments implements ``__numpy_ufunc__`` it is + executed instead of the Ufunc. + +- If more than one of the input arguments implements ``__numpy_ufunc__``, + they are tried in the following order: subclasses before superclasses, + otherwise left to right. The first ``__numpy_ufunc__`` method returning + something else than ``NotImplemented`` determines the return value of + the Ufunc. + +- If all ``__numpy_ufunc__`` methods of the input arguments return + ``NotImplemented``, a ``TypeError`` is raised. + +- If a ``__numpy_ufunc__`` method raises an error, the error is propagated + immediately. + +If none of the input arguments has a ``__numpy_ufunc__`` method, the +execution falls back on the default ufunc behaviour. + + +Demo +==== + +A pull request[6]_ has been made including the changes proposed in this NEP. +Here is a demo highlighting the functionality.:: + + In [1]: import numpy as np; + + In [2]: a = np.array([1]) + + In [3]: class B(): + ...: def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + ...: return "B" + ...: + + In [4]: b = B() + + In [5]: np.dot(a, b) + Out[5]: 'B' + + In [6]: np.multiply(a, b) + Out[6]: 'B' + +A simple ``__numpy_ufunc__`` has been added to SciPy's sparse matrices +Currently this only handles ``np.dot`` and ``np.multiply`` because it was the +two most common cases where users would attempt to use sparse matrices with ufuncs. +The method is defined below:: + + def __numpy_ufunc__(self, func, method, pos, inputs, **kwargs): + """Method for compatibility with NumPy's ufuncs and dot + functions. + """ + + without_self = list(inputs) + del without_self[pos] + without_self = tuple(without_self) + + if func == np.multiply: + return self.multiply(*without_self) + + elif func == np.dot: + if pos == 0: + return self.__mul__(inputs[1]) + if pos == 1: + return self.__rmul__(inputs[0]) + else: + return NotImplemented + +So we now get the expected behavior when using ufuncs with sparse matrices.:: + + In [1]: import numpy as np; import scipy.sparse as sp + + In [2]: a = np.random.randint(3, size=(3,3)) + + In [3]: b = np.random.randint(3, size=(3,3)) + + In [4]: asp = sp.csr_matrix(a); bsp = sp.csr_matrix(b) + + In [5]: np.dot(a,b) + Out[5]: + array([[2, 4, 8], + [2, 4, 8], + [2, 2, 3]]) + + In [6]: np.dot(asp,b) + Out[6]: + array([[2, 4, 8], + [2, 4, 8], + [2, 2, 3]], dtype=int64) + + In [7]: np.dot(asp, bsp).A + Out[7]: + array([[2, 4, 8], + [2, 4, 8], + [2, 2, 3]], dtype=int64) + +.. Local Variables: +.. mode: rst +.. coding: utf-8 +.. fill-column: 72 +.. End: + diff --git a/doc/source/reference/arrays.classes.rst b/doc/source/reference/arrays.classes.rst index 5cdadd4..82f9508 100644 --- a/doc/source/reference/arrays.classes.rst +++ b/doc/source/reference/arrays.classes.rst @@ -38,6 +38,40 @@ Special attributes and methods Numpy provides several hooks that subclasses of :class:`ndarray` can customize: +.. function:: __numpy_ufunc__(self, ufunc, method, i, inputs, **kwargs) + + Any class (ndarray subclass or not) can define this method to + override behavior of Numpy's ufuncs. This works quite similarly to + Python's ``__mul__`` and other binary operation routines. + + - *ufunc* is the ufunc object that was called. + - *method* is a string indicating which Ufunc method was called + (one of ``"__call__"``, ``"reduce"``, ``"reduceat"``, + ``"accumulate"``, ``"outer"``, ``"inner"``). + - *i* is the index of *self* in *inputs*. + - *inputs* is a tuple of the input arguments to the ``ufunc`` + - *kwargs* is a dictionary containing the optional input arguments + of the ufunc. The ``out`` argument is always contained in + *kwargs*, if given. + + The method should return either the result of the operation, or + :obj:`NotImplemented` if the operation requested is not + implemented. + + If one of the arguments has a :func:`__numpy_ufunc__` method, it is + executed *instead* of the ufunc. If more than one of the input + arguments implements :func:`__numpy_ufunc__`, they are tried in the + order: subclasses before superclasses, otherwise left to right. The + first routine returning something else than :obj:`NotImplemented` + determines the result. If all of the :func:`__numpy_ufunc__` + operations returns :obj:`NotImplemented`, a :exc:`TypeError` is + raised. + + If an :class:`ndarray` subclass defines the :func:`__numpy_ufunc__` + method, this disables the :func:`__array_wrap__`, + :func:`__array_prepare__`, :data:`__array_priority__` mechanism + described below. + .. function:: __array_finalize__(self) This method is called whenever the system internally allocates a