Skip to content

Commit 18a6e3e

Browse files
authored
Merge pull request #16200 from seberg/dtypemeta-array-coercion
ENH: Rewrite of array-coercion to support new dtypes
2 parents 90177d6 + 22ee971 commit 18a6e3e

34 files changed

Lines changed: 2491 additions & 1654 deletions
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
NumPy Scalars are cast when assigned to arrays
2+
----------------------------------------------
3+
4+
When creating or assigning to arrays, in all relevant cases NumPy
5+
scalars will now be cast identically to NumPy arrays. In particular
6+
this changes the behaviour in some cases which previously raised an
7+
error::
8+
9+
np.array([np.float64(np.nan)], dtype=np.int64)
10+
11+
will succeed at this time (this may change) and return an undefined result
12+
(usually the smallest possible integer). This also affects assignments::
13+
14+
arr[0] = np.float64(np.nan)
15+
16+
Note, this already happened for ``np.array(np.float64(np.nan), dtype=np.int64)``
17+
and that the behaviour is unchanged for ``np.nan`` itself which is a Python
18+
float.
19+
To avoid backward compatibility issues, at this time assignment from
20+
``datetime64`` scalar to strings of too short length remains supported.
21+
This means that ``np.asarray(np.datetime64("2020-10-10"), dtype="S5")``
22+
succeeds now, when it failed before. In the long term this may be
23+
deprecated or the unsafe cast may be allowed generally to make assignment
24+
of arrays and scalars behave consistently.
25+
26+
27+
Array coercion changes when Strings and other types are mixed
28+
-------------------------------------------------------------
29+
30+
When stringss and other types are mixed, such as::
31+
32+
np.array(["string", np.float64(3.)], dtype="S")
33+
34+
The results will change, which may lead to string dtypes with longer strings
35+
in some cases. In particularly, if ``dtype="S"`` is not provided any numerical
36+
value will lead to a string results long enough to hold all possible numerical
37+
values. (e.g. "S32" for floats). Note that you should always provide
38+
``dtype="S"`` when converting non-strings to strings.
39+
40+
If ``dtype="S"`` is provided the results will be largely identical to before,
41+
but NumPy scalars (not a Python float like ``1.0``), will still enforce
42+
a uniform string length::
43+
44+
np.array([np.float64(3.)], dtype="S") # gives "S32"
45+
np.array([3.0], dtype="S") # gives "S3"
46+
47+
while previously the first version gave the same result as the second.
48+
49+
50+
Array coercion restructure
51+
--------------------------
52+
53+
Array coercion has been restructured. In general, this should not affect
54+
users. In extremely rare corner cases where array-likes are nested::
55+
56+
np.array([array_like1])
57+
58+
things will now be more consistent with::
59+
60+
np.array([np.array(array_like1)])
61+
62+
which could potentially change output subtly for badly defined array-likes.
63+
We are not aware of any such case where the results were not clearly
64+
incorrect previously.

numpy/core/code_generators/genapi.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@
2121

2222
# The files under src/ that are scanned for API functions
2323
API_FILES = [join('multiarray', 'alloc.c'),
24+
join('multiarray', 'abstractdtypes.c'),
2425
join('multiarray', 'arrayfunction_override.c'),
2526
join('multiarray', 'array_assign_array.c'),
2627
join('multiarray', 'array_assign_scalar.c'),
28+
join('multiarray', 'array_coercion.c'),
2729
join('multiarray', 'arrayobject.c'),
2830
join('multiarray', 'arraytypes.c.src'),
2931
join('multiarray', 'buffer.c'),

numpy/core/include/numpy/ndarraytypes.h

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1547,11 +1547,15 @@ PyArray_GETITEM(const PyArrayObject *arr, const char *itemptr)
15471547
(void *)itemptr, (PyArrayObject *)arr);
15481548
}
15491549

1550+
/*
1551+
* SETITEM should only be used if it is known that the value is a scalar
1552+
* and of a type understood by the arrays dtype.
1553+
* Use `PyArray_Pack` if the value may be of a different dtype.
1554+
*/
15501555
static NPY_INLINE int
15511556
PyArray_SETITEM(PyArrayObject *arr, char *itemptr, PyObject *v)
15521557
{
1553-
return ((PyArrayObject_fields *)arr)->descr->f->setitem(
1554-
v, itemptr, arr);
1558+
return ((PyArrayObject_fields *)arr)->descr->f->setitem(v, itemptr, arr);
15551559
}
15561560

15571561
#else
@@ -1820,18 +1824,33 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
18201824
/* TODO: Make this definition public in the API, as soon as its settled */
18211825
NPY_NO_EXPORT extern PyTypeObject PyArrayDTypeMeta_Type;
18221826

1827+
typedef struct PyArray_DTypeMeta_tag PyArray_DTypeMeta;
1828+
1829+
typedef PyArray_Descr *(discover_descr_from_pyobject_function)(
1830+
PyArray_DTypeMeta *cls, PyObject *obj);
1831+
1832+
/*
1833+
* Before making this public, we should decide whether it should pass
1834+
* the type, or allow looking at the object. A possible use-case:
1835+
* `np.array(np.array([0]), dtype=np.ndarray)`
1836+
* Could consider arrays that are not `dtype=ndarray` "scalars".
1837+
*/
1838+
typedef int (is_known_scalar_type_function)(
1839+
PyArray_DTypeMeta *cls, PyTypeObject *obj);
1840+
1841+
typedef PyArray_Descr *(default_descr_function)(PyArray_DTypeMeta *cls);
1842+
18231843
/*
18241844
* While NumPy DTypes would not need to be heap types the plan is to
1825-
* make DTypes available in Python at which point we will probably want
1826-
* them to be.
1845+
* make DTypes available in Python at which point they will be heap types.
18271846
* Since we also wish to add fields to the DType class, this looks like
18281847
* a typical instance definition, but with PyHeapTypeObject instead of
18291848
* only the PyObject_HEAD.
18301849
* This must only be exposed very extremely careful consideration, since
18311850
* it is a fairly complex construct which may be better to allow
18321851
* refactoring of.
18331852
*/
1834-
typedef struct _PyArray_DTypeMeta {
1853+
struct PyArray_DTypeMeta_tag {
18351854
PyHeapTypeObject super;
18361855

18371856
/*
@@ -1870,9 +1889,12 @@ typedef void (PyDataMem_EventHookFunc)(void *inp, void *outp, size_t size,
18701889
* NOTE: We could make a copy to detect changes to `f`.
18711890
*/
18721891
PyArray_ArrFuncs *f;
1873-
} PyArray_DTypeMeta;
18741892

1875-
#define NPY_DTYPE(descr) ((PyArray_DTypeMeta *)Py_TYPE(descr))
1893+
/* DType methods, these could be moved into its own struct */
1894+
discover_descr_from_pyobject_function *discover_descr_from_pyobject;
1895+
is_known_scalar_type_function *is_known_scalar_type;
1896+
default_descr_function *default_descr;
1897+
};
18761898

18771899
#endif /* NPY_INTERNAL_BUILD */
18781900

numpy/core/setup.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -774,9 +774,11 @@ def get_mathlib_info(*args):
774774
#######################################################################
775775

776776
multiarray_deps = [
777+
join('src', 'multiarray', 'abstractdtypes.h'),
777778
join('src', 'multiarray', 'arrayobject.h'),
778779
join('src', 'multiarray', 'arraytypes.h'),
779780
join('src', 'multiarray', 'arrayfunction_override.h'),
781+
join('src', 'multiarray', 'array_coercion.h'),
780782
join('src', 'multiarray', 'npy_buffer.h'),
781783
join('src', 'multiarray', 'calculation.h'),
782784
join('src', 'multiarray', 'common.h'),
@@ -825,9 +827,11 @@ def get_mathlib_info(*args):
825827
] + npysort_sources + npymath_sources
826828

827829
multiarray_src = [
830+
join('src', 'multiarray', 'abstractdtypes.c'),
828831
join('src', 'multiarray', 'alloc.c'),
829832
join('src', 'multiarray', 'arrayobject.c'),
830833
join('src', 'multiarray', 'arraytypes.c.src'),
834+
join('src', 'multiarray', 'array_coercion.c'),
831835
join('src', 'multiarray', 'array_assign_scalar.c'),
832836
join('src', 'multiarray', 'array_assign_array.c'),
833837
join('src', 'multiarray', 'arrayfunction_override.c'),

numpy/core/src/multiarray/_datetime.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@ create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit);
3838
NPY_NO_EXPORT PyArray_DatetimeMetaData *
3939
get_datetime_metadata_from_dtype(PyArray_Descr *dtype);
4040

41+
NPY_NO_EXPORT int
42+
find_string_array_datetime64_type(PyArrayObject *arr,
43+
PyArray_DatetimeMetaData *meta);
44+
4145
/*
4246
* Both type1 and type2 must be either NPY_DATETIME or NPY_TIMEDELTA.
4347
* Applies the type promotion rules between the two types, returning
Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
#define PY_SSIZE_T_CLEAN
2+
#include <Python.h>
3+
#include "structmember.h"
4+
5+
6+
#define NPY_NO_DEPRECATED_API NPY_API_VERSION
7+
#define _MULTIARRAYMODULE
8+
#include "numpy/ndarraytypes.h"
9+
#include "numpy/arrayobject.h"
10+
11+
#include "abstractdtypes.h"
12+
#include "array_coercion.h"
13+
#include "common.h"
14+
15+
16+
static PyArray_Descr *
17+
discover_descriptor_from_pyint(
18+
PyArray_DTypeMeta *NPY_UNUSED(cls), PyObject *obj)
19+
{
20+
assert(PyLong_Check(obj));
21+
/*
22+
* We check whether long is good enough. If not, check longlong and
23+
* unsigned long before falling back to `object`.
24+
*/
25+
long long value = PyLong_AsLongLong(obj);
26+
if (error_converting(value)) {
27+
PyErr_Clear();
28+
}
29+
else {
30+
if (NPY_MIN_LONG <= value && value <= NPY_MAX_LONG) {
31+
return PyArray_DescrFromType(NPY_LONG);
32+
}
33+
return PyArray_DescrFromType(NPY_LONGLONG);
34+
}
35+
36+
unsigned long long uvalue = PyLong_AsUnsignedLongLong(obj);
37+
if (uvalue == (unsigned long long)-1 && PyErr_Occurred()){
38+
PyErr_Clear();
39+
}
40+
else {
41+
return PyArray_DescrFromType(NPY_ULONGLONG);
42+
}
43+
44+
return PyArray_DescrFromType(NPY_OBJECT);
45+
}
46+
47+
48+
static PyArray_Descr*
49+
discover_descriptor_from_pyfloat(
50+
PyArray_DTypeMeta* NPY_UNUSED(cls), PyObject *obj)
51+
{
52+
assert(PyFloat_CheckExact(obj));
53+
return PyArray_DescrFromType(NPY_DOUBLE);
54+
}
55+
56+
57+
static PyArray_Descr*
58+
discover_descriptor_from_pycomplex(
59+
PyArray_DTypeMeta* NPY_UNUSED(cls), PyObject *obj)
60+
{
61+
assert(PyComplex_CheckExact(obj));
62+
return PyArray_DescrFromType(NPY_COMPLEX128);
63+
}
64+
65+
66+
NPY_NO_EXPORT int
67+
initialize_and_map_pytypes_to_dtypes()
68+
{
69+
PyArrayAbstractObjDTypeMeta_Type.tp_base = &PyArrayDTypeMeta_Type;
70+
if (PyType_Ready(&PyArrayAbstractObjDTypeMeta_Type) < 0) {
71+
return -1;
72+
}
73+
((PyTypeObject *)&PyArray_PyIntAbstractDType)->tp_base = &PyArrayDTypeMeta_Type;
74+
PyArray_PyIntAbstractDType.scalar_type = &PyLong_Type;
75+
if (PyType_Ready((PyTypeObject *)&PyArray_PyIntAbstractDType) < 0) {
76+
return -1;
77+
}
78+
((PyTypeObject *)&PyArray_PyFloatAbstractDType)->tp_base = &PyArrayDTypeMeta_Type;
79+
PyArray_PyFloatAbstractDType.scalar_type = &PyFloat_Type;
80+
if (PyType_Ready((PyTypeObject *)&PyArray_PyFloatAbstractDType) < 0) {
81+
return -1;
82+
}
83+
((PyTypeObject *)&PyArray_PyComplexAbstractDType)->tp_base = &PyArrayDTypeMeta_Type;
84+
PyArray_PyComplexAbstractDType.scalar_type = &PyComplex_Type;
85+
if (PyType_Ready((PyTypeObject *)&PyArray_PyComplexAbstractDType) < 0) {
86+
return -1;
87+
}
88+
89+
/* Register the new DTypes for discovery */
90+
if (_PyArray_MapPyTypeToDType(
91+
&PyArray_PyIntAbstractDType, &PyLong_Type, NPY_FALSE) < 0) {
92+
return -1;
93+
}
94+
if (_PyArray_MapPyTypeToDType(
95+
&PyArray_PyFloatAbstractDType, &PyFloat_Type, NPY_FALSE) < 0) {
96+
return -1;
97+
}
98+
if (_PyArray_MapPyTypeToDType(
99+
&PyArray_PyComplexAbstractDType, &PyComplex_Type, NPY_FALSE) < 0) {
100+
return -1;
101+
}
102+
103+
/*
104+
* Map str, bytes, and bool, for which we do not need abstract versions
105+
* to the NumPy DTypes. This is done here using the `is_known_scalar_type`
106+
* function.
107+
* TODO: The `is_known_scalar_type` function is considered preliminary,
108+
* the same could be achieved e.g. with additional abstract DTypes.
109+
*/
110+
PyArray_DTypeMeta *dtype;
111+
dtype = NPY_DTYPE(PyArray_DescrFromType(NPY_UNICODE));
112+
if (_PyArray_MapPyTypeToDType(dtype, &PyUnicode_Type, NPY_FALSE) < 0) {
113+
return -1;
114+
}
115+
116+
dtype = NPY_DTYPE(PyArray_DescrFromType(NPY_STRING));
117+
if (_PyArray_MapPyTypeToDType(dtype, &PyBytes_Type, NPY_FALSE) < 0) {
118+
return -1;
119+
}
120+
dtype = NPY_DTYPE(PyArray_DescrFromType(NPY_BOOL));
121+
if (_PyArray_MapPyTypeToDType(dtype, &PyBool_Type, NPY_FALSE) < 0) {
122+
return -1;
123+
}
124+
125+
return 0;
126+
}
127+
128+
129+
130+
/* Note: This is currently largely not used, but will be required eventually. */
131+
NPY_NO_EXPORT PyTypeObject PyArrayAbstractObjDTypeMeta_Type = {
132+
PyVarObject_HEAD_INIT(NULL, 0)
133+
.tp_name = "numpy._AbstractObjDTypeMeta",
134+
.tp_basicsize = sizeof(PyArray_DTypeMeta),
135+
.tp_flags = Py_TPFLAGS_DEFAULT,
136+
.tp_doc = "Helper MetaClass for value based casting AbstractDTypes.",
137+
};
138+
139+
NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyIntAbstractDType = {{{
140+
PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0)
141+
.tp_basicsize = sizeof(PyArray_DTypeMeta),
142+
.tp_name = "numpy._PyIntBaseAbstractDType",
143+
},},
144+
.abstract = 1,
145+
.discover_descr_from_pyobject = discover_descriptor_from_pyint,
146+
.kind = 'i',
147+
};
148+
149+
NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyFloatAbstractDType = {{{
150+
PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0)
151+
.tp_basicsize = sizeof(PyArray_DTypeMeta),
152+
.tp_name = "numpy._PyFloatBaseAbstractDType",
153+
},},
154+
.abstract = 1,
155+
.discover_descr_from_pyobject = discover_descriptor_from_pyfloat,
156+
.kind = 'f',
157+
};
158+
159+
NPY_NO_EXPORT PyArray_DTypeMeta PyArray_PyComplexAbstractDType = {{{
160+
PyVarObject_HEAD_INIT(&PyArrayAbstractObjDTypeMeta_Type, 0)
161+
.tp_basicsize = sizeof(PyArray_DTypeMeta),
162+
.tp_name = "numpy._PyComplexBaseAbstractDType",
163+
},},
164+
.abstract = 1,
165+
.discover_descr_from_pyobject = discover_descriptor_from_pycomplex,
166+
.kind = 'c',
167+
};
168+
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#ifndef _NPY_ABSTRACTDTYPES_H
2+
#define _NPY_ABSTRACTDTYPES_H
3+
4+
#include "dtypemeta.h"
5+
6+
/*
7+
* These are mainly needed for value based promotion in ufuncs. It
8+
* may be necessary to make them (partially) public, to allow user-defined
9+
* dtypes to perform value based casting.
10+
*/
11+
NPY_NO_EXPORT extern PyTypeObject PyArrayAbstractObjDTypeMeta_Type;
12+
NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyIntAbstractDType;
13+
NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyFloatAbstractDType;
14+
NPY_NO_EXPORT extern PyArray_DTypeMeta PyArray_PyComplexAbstractDType;
15+
16+
NPY_NO_EXPORT int
17+
initialize_and_map_pytypes_to_dtypes();
18+
19+
#endif /*_NPY_ABSTRACTDTYPES_H */

0 commit comments

Comments
 (0)