[Scons-dev] Partial fix for mixed output with -j

[Kenny, Jason L]

I do something like this in Parts.

I basically use subprocess to map output of any "task" that is called and then use wrapper much like your to get any other kind of output ( ie print statements from a pure python task). Once all this output is being piped it easy to redirect to the screen and or log files, etc

Jason 


-----Original Message-----
From: scons-dev-bounces at scons.org [mailto:scons-dev-bounces at scons.org] On Behalf Of TOM TANNER (BLOOMBERG/ LONDON)
Sent: Wednesday, June 20, 2012 4:44 AM
To: scons-dev at scons.org
Subject: [Scons-dev] Partial fix for mixed output with -j

OK, this is just a 'possible' at the moment. I'm not sure it's in the right place (debug output still gets mixed up), it's possible this should be part of the Task object, and it happens even with no -j setting which is silly (but I don't know how to check for that), but it works wonderfully for parallel builds.

However, I'd like some comments.

Here's the diff output (a bit split up and mangled because I was having problems with gmane)

109,112d109
< from threading import Lock, local
< import tempfile
< import sys
< import os
114,145d110
< class Threaded_Output_Redirector:
<     """ The general idea of this class is to allow redirection of output to a
<         file for a specific thread.
<     """
<     def __init__(self, out):
<         self.old_output = out
<         self.threadvar = local()
<         
<     def write(self, string):
<         if (hasattr(self.threadvar, 'out')):
<             self.threadvar.out.write(string)
<         else:
<             self.old_output.write(string)
<     
<     def flush(self):
<         if (hasattr(self.threadvar, 'out')):
<             self.threadvar.out.flush()
<         else:
<             self.old_output.flush()
< 
<     def redirect(self, handle):
<         self.threadvar.out = handle
< 
<     def end_redirect(self):
<         del self.threadvar.out
<         
<
<
< sys.stdout = Threaded_Output_Redirector(sys.stdout)
< sys.stderr = Threaded_Output_Redirector(sys.stderr)
<
<
154,155d118
<     _lock = Lock()
<
175d137
<         self.output = None
370,381d331
<     def pspawn_wrapper(self, sh, escape, cmd, args, env):
<         """Wrapper function for handling piped spawns.
< 
<         This looks to the calling interface (in Action.py) like a "normal"
<         spawn, but associates the call with the PSPAWN variable from
<         the construction environment and with the streams to which we
<         want the output logged.  This gets slid into the construction
<         environment as the SPAWN variable so Action.py doesn't have to
<         know or care whether it's spawning a piped command or not.
<         """
<         return self.pspawn(sh, escape, cmd, args, env, self.output, self.output)
<
387,451c337,350
< 
<         # Stash the results in a file
<         # Try and buffer the output
<         outfile = None
< 
<         # Make sure we have a PSPAWN value, and save the current
<         # SPAWN value.
<         try:
<             self.pspawn = env['PSPAWN']
<         except KeyError:
<             raise SCons.Errors.UserError('Missing PSPAWN construction variable.')
<         try:
<             save_spawn = env['SPAWN']
<         except KeyError:
<             raise SCons.Errors.UserError('Missing SPAWN construction variable.')
< 
<         with Executor._lock:
<             print '==================== Building', str(target), '===================='
< 
<         try:
<             # Slide our wrapper into the construction environment as
<             # the SPAWN function.
<             env['SPAWN'] = self.pspawn_wrapper
< 
<             with tempfile.NamedTemporaryFile(delete = False, prefix = 'scons') as self.output:
<                 outfile = self.output.name
<                 sys.stdout.redirect(self.output)
<                 sys.stderr.redirect(self.output)
<                 
<                 for act in self.get_action_list():
<                     #args = (self.get_all_targets(), self.get_all_sources(), env)
<                     args = ([], [], env)
<                     status = act(*args, **kw)
<                     if isinstance(status, SCons.Errors.BuildError):
<                         status.executor = self
<                         raise status
<                     elif status:
<                         msg = "Error %s" % status
<                         raise SCons.Errors.BuildError(
<                             errstr=msg, 
<                             node=self.batches[0].targets,
<                             executor=self, 
<                             action=act)
<         finally:
<             env['SPAWN'] = save_spawn
<             sys.stderr.end_redirect()
<             sys.stdout.end_redirect()
<             with Executor._lock:
<                 print '\n====================',
<                 if status == 0:
<                     print 'Completed',
<                 else:
<                     print 'Failed', status,
<                 print str(target),
<                 print '===================='
< 
<                 with open(outfile) as infile:
<                     for line in infile:
<                         print line.strip()
< 
<                 print '========================================\n'
< 
<                 os.remove(outfile)
< 
<         # Isn't this always 0?
---

>         for act in self.get_action_list():

>             #args = (self.get_all_targets(), self.get_all_sources(), env)

>             args = ([], [], env)

>             status = act(*args, **kw)

>             if isinstance(status, SCons.Errors.BuildError):

>                 status.executor = self

>                 raise status

>             elif status:

>                 msg = "Error %s" % status

>                 raise SCons.Errors.BuildError(

>                     errstr=msg, 

>                     node=self.batches[0].targets,

>                     executor=self, 

>                     action=act)


Yes, it does look a bit hacky, and I'd like to avoid having 2 copies of the execution main loop - one for single threaded, one for multi - and I'm not sure how this'd interact with SConf.py, and even whether it's actually necessary to recover the old version of env['SPAWN']

Help please?

Thanks
_______________________________________________
Scons-dev mailing list
Scons-dev at scons.org
http://two.pairlist.net/mailman/listinfo/scons-dev