Merge pull request #25158 from seberg/loadtxt-encoding

charris · web-flow · commit 32fad571b948 · 2023-11-16T16:54:03.000-07:00
API: Make `encoding=None` the default in loadtxt
diff --git a/doc/release/upcoming_changes/25158.compatibility.rst b/doc/release/upcoming_changes/25158.compatibility.rst
@@ -0,0 +1,8 @@
+``loadtxt`` and ``genfromtxt`` default to ``encoding=None``
+-----------------------------------------------------------
+``loadtxt`` and ``genfromtxt`` now both default to ``encoding=None``
+which may mainly modifies how ``converters`` work.
+These will now be passed ``str`` rather than ``bytes``, pass the
+encoding explicitly to always get the new or old behavior.
+For ``genfromtxt`` the change also means that returned values will now be
+unicode strings rather than bytes.
diff --git a/doc/source/user/basics.io.genfromtxt.rst b/doc/source/user/basics.io.genfromtxt.rst
@@ -58,7 +58,7 @@ Quite often, a single character marks the separation between columns.  For
 example, comma-separated files (CSV) use a comma (``,``) or a semicolon
 (``;``) as delimiter::
 
-   >>> data = u"1, 2, 3\n4, 5, 6"
+   >>> data = "1, 2, 3\n4, 5, 6"
    >>> np.genfromtxt(StringIO(data), delimiter=",")
    array([[1.,  2.,  3.],
           [4.,  5.,  6.]])
@@ -74,12 +74,12 @@ defined as a given number of characters.  In that case, we need to set
 ``delimiter`` to a single integer (if all the columns have the same
 size) or to a sequence of integers (if columns can have different sizes)::
 
-   >>> data = u"  1  2  3\n  4  5 67\n890123  4"
+   >>> data = "  1  2  3\n  4  5 67\n890123  4"
    >>> np.genfromtxt(StringIO(data), delimiter=3)
    array([[  1.,    2.,    3.],
           [  4.,    5.,   67.],
           [890.,  123.,    4.]])
-   >>> data = u"123456789\n   4  7 9\n   4567 9"
+   >>> data = "123456789\n   4  7 9\n   4567 9"
    >>> np.genfromtxt(StringIO(data), delimiter=(4, 3, 2))
    array([[1234.,   567.,    89.],
           [   4.,     7.,     9.],
@@ -94,7 +94,7 @@ individual entries are not stripped of leading nor trailing white spaces.
 This behavior can be overwritten by setting the optional argument
 ``autostrip`` to a value of ``True``::
 
-   >>> data = u"1, abc , 2\n 3, xxx, 4"
+   >>> data = "1, abc , 2\n 3, xxx, 4"
    >>> # Without autostrip
    >>> np.genfromtxt(StringIO(data), delimiter=",", dtype="|U5")
    array([['1', ' abc ', ' 2'],
@@ -114,7 +114,7 @@ string that marks the beginning of a comment.  By default,
 occur anywhere on the line.  Any character present after the comment
 marker(s) is simply ignored::
 
-   >>> data = u"""#
+   >>> data = """#
    ... # Skip me !
    ... # Skip me too !
    ... 1, 2
@@ -154,7 +154,7 @@ of lines to skip at the beginning of the file, before any other action is
 performed.  Similarly, we can skip the last ``n`` lines of the file by
 using the ``skip_footer`` attribute and giving it a value of ``n``::
 
-   >>> data = u"\n".join(str(i) for i in range(10))
+   >>> data = "\n".join(str(i) for i in range(10))
    >>> np.genfromtxt(StringIO(data),)
    array([0.,  1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9.])
    >>> np.genfromtxt(StringIO(data),
@@ -178,7 +178,7 @@ integers behave the same as regular Python negative indexes.
 For example, if we want to import only the first and the last columns, we
 can use ``usecols=(0, -1)``::
 
-   >>> data = u"1 2 3\n4 5 6"
+   >>> data = "1 2 3\n4 5 6"
    >>> np.genfromtxt(StringIO(data), usecols=(0, -1))
    array([[1.,  3.],
           [4.,  6.]])
@@ -187,7 +187,7 @@ If the columns have names, we can also select which columns to import by
 giving their name to the ``usecols`` argument, either as a sequence
 of strings or a comma-separated string::
 
-   >>> data = u"1 2 3\n4 5 6"
+   >>> data = "1 2 3\n4 5 6"
    >>> np.genfromtxt(StringIO(data),
    ...               names="a, b, c", usecols=("a", "c"))
    array([(1., 3.), (4., 6.)], dtype=[('a', '<f8'), ('c', '<f8')])
@@ -370,8 +370,8 @@ single element of the wanted type.
 In the following example, the second column is converted from as string
 representing a percentage to a float between 0 and 1::
 
-   >>> convertfunc = lambda x: float(x.strip(b"%"))/100.
-   >>> data = u"1, 2.3%, 45.\n6, 78.9%, 0"
+   >>> convertfunc = lambda x: float(x.strip("%"))/100.
+   >>> data = "1, 2.3%, 45.\n6, 78.9%, 0"
    >>> names = ("i", "p", "n")
    >>> # General case .....
    >>> np.genfromtxt(StringIO(data), delimiter=",", names=names)
@@ -405,7 +405,7 @@ string into the corresponding float or into -999 if the string is empty.
 We need to explicitly strip the string from white spaces as it is not done
 by default::
 
-   >>> data = u"1, , 3\n 4, 5, 6"
+   >>> data = "1, , 3\n 4, 5, 6"
    >>> convert = lambda x: float(x.strip() or -999)
    >>> np.genfromtxt(StringIO(data), delimiter=",",
    ...               converters={1: convert})
@@ -483,7 +483,7 @@ with ``"N/A"`` in the first column and by ``"???"`` in the third column.
 We wish to transform these missing values to 0 if they occur in the first
 and second column, and to -999 if they occur in the last column::
 
-    >>> data = u"N/A, 2, 3\n4, ,???"
+    >>> data = "N/A, 2, 3\n4, ,???"
     >>> kwargs = dict(delimiter=",",
     ...               dtype=int,
     ...               names="a,b,c",
diff --git a/numpy/_core/code_generators/ufunc_docstrings.py b/numpy/_core/code_generators/ufunc_docstrings.py
@@ -4279,10 +4279,10 @@ def add_newdoc(place, name, doc):
     >>> a = np.array(['Grace Hopper Conference', 'Open Source Day'])
     >>> np.char.str_len(a)
     array([23, 15])
-    >>> a = np.array([u'\u0420', u'\u043e'])
+    >>> a = np.array(['\u0420', '\u043e'])
     >>> np.char.str_len(a)
     array([1, 1])
-    >>> a = np.array([['hello', 'world'], [u'\u0420', u'\u043e']])
+    >>> a = np.array([['hello', 'world'], ['\u0420', '\u043e']])
     >>> np.char.str_len(a)
     array([[5, 5], [1, 1]])
 
diff --git a/numpy/_core/defchararray.py b/numpy/_core/defchararray.py
@@ -285,10 +285,10 @@ def str_len(a):
     >>> a = np.array(['Grace Hopper Conference', 'Open Source Day'])
     >>> np.char.str_len(a)
     array([23, 15])
-    >>> a = np.array([u'\u0420', u'\u043e'])
+    >>> a = np.array(['\u0420', '\u043e'])
     >>> np.char.str_len(a)
     array([1, 1])
-    >>> a = np.array([['hello', 'world'], [u'\u0420', u'\u043e']])
+    >>> a = np.array([['hello', 'world'], ['\u0420', '\u043e']])
     >>> np.char.str_len(a)
     array([[5, 5], [1, 1]])
     """
diff --git a/numpy/lib/_function_base_impl.py b/numpy/lib/_function_base_impl.py
@@ -1976,7 +1976,7 @@ def disp(mesg, device=None, linefeed=True):
 
     >>> from io import StringIO
     >>> buf = StringIO()
-    >>> np.disp(u'"Display" in a file', device=buf)
+    >>> np.disp('"Display" in a file', device=buf)
     >>> buf.getvalue()
     '"Display" in a file\\n'
 
diff --git a/numpy/lib/_npyio_impl.py b/numpy/lib/_npyio_impl.py
@@ -815,7 +815,7 @@ def _preprocess_comments(iterable, comments, encoding):
 def _read(fname, *, delimiter=',', comment='#', quote='"',
           imaginary_unit='j', usecols=None, skiplines=0,
           max_rows=None, converters=None, ndmin=None, unpack=False,
-          dtype=np.float64, encoding="bytes"):
+          dtype=np.float64, encoding=None):
     r"""
     Read a NumPy array from a text file.
     This is a helper function for loadtxt.
@@ -1073,7 +1073,7 @@ def _read(fname, *, delimiter=',', comment='#', quote='"',
 @set_module('numpy')
 def loadtxt(fname, dtype=float, comments='#', delimiter=None,
             converters=None, skiprows=0, usecols=None, unpack=False,
-            ndmin=0, encoding='bytes', max_rows=None, *, quotechar=None,
+            ndmin=0, encoding=None, max_rows=None, *, quotechar=None,
             like=None):
     r"""
     Load data from a text file.
@@ -1145,6 +1145,10 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
         the system default is used. The default value is 'bytes'.
 
         .. versionadded:: 1.14.0
+        .. versionchanged:: 2.0
+            Before NumPy 2, the default was ``'bytes'`` for Python 2
+            compatibility. The default is now ``None``.
+
     max_rows : int, optional
         Read `max_rows` rows of content after `skiprows` lines. The default is
         to read all the rows. Note that empty rows containing no data such as
@@ -1262,24 +1266,22 @@ def loadtxt(fname, dtype=float, comments='#', delimiter=None,
     array([1.e+00, 2.7e+00, 1.e+05])
 
     This idea can be extended to automatically handle values specified in
-    many different formats:
+    many different formats, such as hex values:
 
     >>> def conv(val):
     ...     try:
     ...         return float(val)
     ...     except ValueError:
     ...         return float.fromhex(val)
     >>> s = StringIO("1, 2.5, 3_000, 0b4, 0x1.4000000000000p+2")
-    >>> np.loadtxt(s, delimiter=",", converters=conv, encoding=None)
+    >>> np.loadtxt(s, delimiter=",", converters=conv)
     array([1.0e+00, 2.5e+00, 3.0e+03, 1.8e+02, 5.0e+00])
 
-    Note that with the default ``encoding="bytes"``, the inputs to the
-    converter function are latin-1 encoded byte strings. To deactivate the
-    implicit encoding prior to conversion, use ``encoding=None``
+    Or a format where the ``-`` sign comes after the number:
 
     >>> s = StringIO('10.01 31.25-\n19.22 64.31\n17.57- 63.94')
     >>> conv = lambda x: -float(x[:-1]) if x.endswith('-') else float(x)
-    >>> np.loadtxt(s, converters=conv, encoding=None)
+    >>> np.loadtxt(s, converters=conv)
     array([[ 10.01, -31.25],
            [ 19.22,  64.31],
            [-17.57,  63.94]])
@@ -1715,7 +1717,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
                deletechars=''.join(sorted(NameValidator.defaultdeletechars)),
                replace_space='_', autostrip=False, case_sensitive=True,
                defaultfmt="f%i", unpack=None, usemask=False, loose=True,
-               invalid_raise=True, max_rows=None, encoding='bytes',
+               invalid_raise=True, max_rows=None, encoding=None,
                *, ndmin=0, like=None):
     """
     Load data from a text file, with missing values handled as specified.
@@ -1816,6 +1818,10 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
         The default value is 'bytes'.
 
         .. versionadded:: 1.14.0
+        .. versionchanged:: 2.0
+            Before NumPy 2, the default was ``'bytes'`` for Python 2
+            compatibility. The default is now ``None``.
+
     ndmin : int, optional
         Same parameter as `loadtxt`
 
@@ -1858,7 +1864,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
 
     Comma delimited file with mixed dtype
 
-    >>> s = StringIO(u"1,1.3,abcde")
+    >>> s = StringIO("1,1.3,abcde")
     >>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'),
     ... ('mystring','S5')], delimiter=",")
     >>> data
@@ -1885,7 +1891,7 @@ def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
 
     An example with fixed-width columns
 
-    >>> s = StringIO(u"11.3abcde")
+    >>> s = StringIO("11.3abcde")
     >>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'],
     ...     delimiter=[1,3,5])
     >>> data
diff --git a/numpy/lib/tests/test_io.py b/numpy/lib/tests/test_io.py
diff --git a/numpy/lib/tests/test_loadtxt.py b/numpy/lib/tests/test_loadtxt.py
diff --git a/numpy/testing/_private/extbuild.py b/numpy/testing/_private/extbuild.py
