[PATCH 1/2] sb/execute.py: Open subprocess I/O handles in text mode

[Anders Montonen]

With a UTF-8 locale, _readthread() would occasionally split a multibyte
code point. When running under Python 3, this would trigger an
UnicodeDecodeError, which resulted in the build hanging. Solve this
issue by opening the subprocess streams in text mode, and let Python's
runtime deal with the nitty-gritty of returning complete code points.

Fixes:
Exception in thread _stderr[]:
Traceback (most recent call last):
  File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
    self.run()
  File "/usr/lib/python3.8/threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "/home/anders/work/rtems/rsb/source-builder/sb/execute.py", line 204, in _readthread
    data = data.decode(sys.stdout.encoding)
UnicodeDecodeError: 'utf-8' codec can't decode bytes in position 4094-4095: unexpected end of data

Signed-off-by: Anders Montonen <Anders.Montonen at iki.fi>
---
 source-builder/sb/execute.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/source-builder/sb/execute.py b/source-builder/sb/execute.py
index 06f9b7d..092267b 100755
--- a/source-builder/sb/execute.py
+++ b/source-builder/sb/execute.py
@@ -123,11 +123,6 @@ class execute(object):
             is a timeout check."""
             if trace_threads:
                 print('execute:_writethread: start')
-            encoding = True
-            try:
-                tmp = bytes('temp', sys.stdin.encoding)
-            except:
-                encoding = False
             input_types = [str, bytes]
             try:
                 # Unicode is not valid in python3, not added to the list
@@ -143,8 +138,6 @@ class execute(object):
                         print('execute:_writethread: input returned:', type(lines))
                     if type(lines) in input_types:
                         try:
-                            if encoding:
-                                lines = bytes(lines, sys.stdin.encoding)
                             fh.write(lines)
                             fh.flush()
                         except:
@@ -192,16 +185,13 @@ class execute(object):
                     # and the process is shutting down.
                     #
                     try:
-                        data = fh.read1(4096)
+                        data = fh.read(4096)
                     except:
                         data = ''
                     if len(data) == 0:
                         if len(line) > 0:
                             _output_line(line + '\n', exe, prefix, out, count)
                         break
-                    # str and bytes are the same type in Python2
-                    if type(data) is not str and type(data) is bytes:
-                        data = data.decode(sys.stdout.encoding)
                     last_ch = data[-1]
                     sd = (line + data).split('\n')
                     if last_ch != '\n':
@@ -267,7 +257,7 @@ class execute(object):
                                              name = '_stdout[%s]' % (name),
                                              args = (self,
                                                      io.open(proc.stdout.fileno(),
-                                                             mode = 'rb',
+                                                             mode = 'r',
                                                              closefd = False),
                                                      self.output,
                                                      ''))
@@ -278,7 +268,7 @@ class execute(object):
                                              name = '_stderr[%s]' % (name),
                                              args = (self,
                                                      io.open(proc.stderr.fileno(),
-                                                             mode = 'rb',
+                                                             mode = 'r',
                                                              closefd = False),
                                                      self.output,
                                                      self.error_prefix))
@@ -381,7 +371,8 @@ class execute(object):
                                     cwd = cwd, env = env,
                                     stdin = stdin, stdout = stdout,
                                     stderr = stderr,
-                                    close_fds = False)
+                                    close_fds = False,
+                                    universal_newlines=True)
             if not capture:
                 return (0, proc)
             if self.output is None:
@@ -563,10 +554,7 @@ if __name__ == "__main__":
         if ec == 0:
             print('piping input into ' + commands['pipe'][0] + ': ' + \
                   commands['pipe'][2])
-            try:
-                out = bytes(commands['pipe'][2], sys.stdin.encoding)
-            except:
-                out = commands['pipe'][2]
+            out = commands['pipe'][2]
             proc.stdin.write(out)
             proc.stdin.close()
             e.capture(proc)
-- 
2.25.1