[Scons-dev] Partial fix for mixed output with -j

[TOM TANNER (BLOOMBERG/ LONDON)]

OK, this is just a 'possible' at the moment. I'm not sure it's in the right
place (debug output still gets mixed up), it's possible this should be part of
the Task object, and it happens even with no -j setting which is silly (but I
don't know how to check for that), but it works wonderfully for parallel builds.

However, I'd like some comments.

Here's the diff output (a bit split up and mangled because I was having problems with gmane)

109,112d109
< from threading import Lock, local
< import tempfile
< import sys
< import os
114,145d110
< class Threaded_Output_Redirector:
<     """ The general idea of this class is to allow redirection of output to a
<         file for a specific thread.
<     """
<     def __init__(self, out):
<         self.old_output = out
<         self.threadvar = local()
<         
<     def write(self, string):
<         if (hasattr(self.threadvar, 'out')):
<             self.threadvar.out.write(string)
<         else:
<             self.old_output.write(string)
<     
<     def flush(self):
<         if (hasattr(self.threadvar, 'out')):
<             self.threadvar.out.flush()
<         else:
<             self.old_output.flush()
< 
<     def redirect(self, handle):
<         self.threadvar.out = handle
< 
<     def end_redirect(self):
<         del self.threadvar.out
<         
< 
< 
< sys.stdout = Threaded_Output_Redirector(sys.stdout)
< sys.stderr = Threaded_Output_Redirector(sys.stderr)
< 
< 
154,155d118
<     _lock = Lock()
< 
175d137
<         self.output = None
370,381d331
<     def pspawn_wrapper(self, sh, escape, cmd, args, env):
<         """Wrapper function for handling piped spawns.
< 
<         This looks to the calling interface (in Action.py) like a "normal"
<         spawn, but associates the call with the PSPAWN variable from
<         the construction environment and with the streams to which we
<         want the output logged.  This gets slid into the construction
<         environment as the SPAWN variable so Action.py doesn't have to
<         know or care whether it's spawning a piped command or not.
<         """
<         return self.pspawn(sh, escape, cmd, args, env, self.output, self.output)
< 
387,451c337,350
< 
<         # Stash the results in a file
<         # Try and buffer the output
<         outfile = None
< 
<         # Make sure we have a PSPAWN value, and save the current
<         # SPAWN value.
<         try:
<             self.pspawn = env['PSPAWN']
<         except KeyError:
<             raise SCons.Errors.UserError('Missing PSPAWN construction variable.')
<         try:
<             save_spawn = env['SPAWN']
<         except KeyError:
<             raise SCons.Errors.UserError('Missing SPAWN construction variable.')
< 
<         with Executor._lock:
<             print '==================== Building', str(target), '===================='
< 
<         try:
<             # Slide our wrapper into the construction environment as
<             # the SPAWN function.
<             env['SPAWN'] = self.pspawn_wrapper
< 
<             with tempfile.NamedTemporaryFile(delete = False, prefix = 'scons') as self.output:
<                 outfile = self.output.name
<                 sys.stdout.redirect(self.output)
<                 sys.stderr.redirect(self.output)
<                 
<                 for act in self.get_action_list():
<                     #args = (self.get_all_targets(), self.get_all_sources(), env)
<                     args = ([], [], env)
<                     status = act(*args, **kw)
<                     if isinstance(status, SCons.Errors.BuildError):
<                         status.executor = self
<                         raise status
<                     elif status:
<                         msg = "Error %s" % status
<                         raise SCons.Errors.BuildError(
<                             errstr=msg, 
<                             node=self.batches[0].targets,
<                             executor=self, 
<                             action=act)
<         finally:
<             env['SPAWN'] = save_spawn
<             sys.stderr.end_redirect()
<             sys.stdout.end_redirect()
<             with Executor._lock:
<                 print '\n====================',
<                 if status == 0:
<                     print 'Completed',
<                 else:
<                     print 'Failed', status,
<                 print str(target),
<                 print '===================='
< 
<                 with open(outfile) as infile:
<                     for line in infile:
<                         print line.strip()
< 
<                 print '========================================\n'
< 
<                 os.remove(outfile)
< 
<         # Isn't this always 0?
---

>         for act in self.get_action_list():

>             #args = (self.get_all_targets(), self.get_all_sources(), env)

>             args = ([], [], env)

>             status = act(*args, **kw)

>             if isinstance(status, SCons.Errors.BuildError):

>                 status.executor = self

>                 raise status

>             elif status:

>                 msg = "Error %s" % status

>                 raise SCons.Errors.BuildError(

>                     errstr=msg, 

>                     node=self.batches[0].targets,

>                     executor=self, 

>                     action=act)


Yes, it does look a bit hacky, and I'd like to avoid having 2 copies of the
execution main loop - one for single threaded, one for multi - and I'm not sure
how this'd interact with SConf.py, and even whether it's actually necessary to
recover the old version of env['SPAWN']

Help please?

Thanks