Merge pull request #1278 from wireservice/pythonioencoding

Use PYTHONIOENCODING if set (--encoding takes priority)

Author: jpmckinney

.github/workflows/ci.yml

@@ -18,7 +18,6 @@ jobs:
       - run: pip install .[test] coveralls
       - env:
           LANG: en_US.UTF-8
-          PYTHONIOENCODING: utf-8
           PYTHONUTF8: 1
         run: pytest --cov csvkit
       # csvpy cannot accept input as piped data via STDIN.

csvkit/cli.py

@@ -198,7 +198,7 @@ def _init_common_parser(self):
                 help='Maximum length of a single field in the input CSV file.')
         if 'e' not in self.override_flags:
             self.argparser.add_argument(
-                '-e', '--encoding', dest='encoding', default='utf-8-sig',
+                '-e', '--encoding', dest='encoding', default=os.getenv('PYTHONIOENCODING', 'utf-8-sig'),
                 help='Specify the encoding of the input CSV file.')
         if 'L' not in self.override_flags:
             self.argparser.add_argument(
@@ -330,9 +330,10 @@ def handler(t, value, traceback):
                 # Special case handling for Unicode errors, which behave very strangely
                 # when cast with unicode()
                 if t == UnicodeDecodeError:
-                    sys.stderr.write('Your file is not "%s" encoded. Please specify the correct encoding with the -e '
-                                     'flag or with the PYTHONIOENCODING environment variable. Use the -v flag to see '
-                                     'the complete error.\n' % self.args.encoding)
+                    sys.stderr.write(
+                        'Your file is not "%s" encoded. Please specify the correct encoding with the --encoding flag.'
+                        ' Use the -v flag to see the complete error.\n' % self.args.encoding
+                    )
                 else:
                     sys.stderr.write(f'{t.__name__}: {str(value)}\n')
 

docs/common_arguments.rst

@@ -68,10 +68,6 @@ The output of csvkit's tools is always formatted with "default" formatting optio
 
 See the documentation of :doc:`/scripts/csvclean` for a description of the default formatting options.
 
-.. note::
-
-   The ``--encoding`` option has no effect if reading from standard input. Set the ``PYTHONIOENCODING`` environment variable instead.
-
 .. seealso::
 
    For a list of possible values for the ``--encoding`` option, see the `Python documentation <https://docs.python.org/3/library/codecs.html#standard-encodings>`__.