|\ merge from release-v0.7.2, released as 0.7.2.

ChangeLog.rst

@@ -1,3 +1,14 @@
+ayrton (0.7.2) UNRELEASED; urgency=medium
+
+  * Fix running remote tests with other versions of python.
+  * Fix tests borken by a change in `ls`'s output.
+  * Fix iterating over the long output of a command à la `for line in foo(...): ...`. Currently you must add `_bg=True` to the execution options.
+  * Fix recognizing names bound by for loops.
+  * Added options `-d|--debug`, `-dd|--debug2` and `-ddd|--debug3` for enabling debug logs.
+  * Added option `-xxx|--trace-all` for tracing all python execution. Use with caution, it generates lots of output.
+
+ -- Marcos Dione <[email protected]>  Thu, 25 Feb 2016 12:40:04 +0100
+
 ayrton (0.7.1) unstable; urgency=medium
 
   * Iterable parameters to executables are expanded in situ, so `foo(..., i, ...)` is expanded to `foo (..., i[0], i[1], ...` and `foo(..., k=i, ...)` is expanded to `foo (..., k=i[0], k=i[1], ...`.

Makefile

@@ -1,5 +1,5 @@
-DEBUG_MULTI=strace -tt -T -ff -o runner -s 128
-DEBUG_SIMPLE=strace -tt -T -o runner -s 128
+DEBUG_MULTI=strace -tt -T -ff -o debug/runner -s 128
+DEBUG_SIMPLE=strace -tt -T -o debug/runner -s 128
 PYTHON=python3.4
 
 all: docs
@@ -9,7 +9,7 @@ INSTALL_DIR=$(HOME)/local
 tests:
 	LC_ALL=C $(PYTHON) -m unittest discover -v ayrton
 
-slowtest:
+slowtest: debug
 	# LC_ALL=C $(DEBUG_SIMPLE) $(PYTHON) -m unittest discover -f -v ayrton
 	LC_ALL=C $(DEBUG_MULTI) $(PYTHON) -m unittest discover -f -v ayrton
 
@@ -40,12 +40,15 @@ check:
 	flake8 --ignore E201,E211,E225,E221,E226,E202 --show-source --statistics --max-line-length 130 ayrton/*.py
 
 testclean:
-	rm -rfv ayrton.*log runner.*
+	rm -f ayrton.*log debug/runner* debug/remote*
+
+debug:
+	mkdir -pv debug
 
 debugserver:
-	# TODO: generate the server key
+	# generate an rsa server key
 	if ! [ -f rsa_server_key ]; then \
-		true; \
+		ssh-keygen -f rsa_server_key -N '' -t rsa; \
 	fi
-	# TODO: discover path?
+	# TODO: discover sshd's path?
 	/usr/sbin/sshd -dd -e -h $(shell pwd)/rsa_server_key -p 2244

ayrton/__init__.py

@@ -32,9 +32,13 @@
 log_format= "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-21s) %(levelname)-8s %(message)s"
 date_format= "%H:%M:%S"
 
-# uncomment one of these for way too much debugging :)
-# logging.basicConfig(filename='ayrton.%d.log' % os.getpid (), level=logging.DEBUG, format=log_format, datefmt=date_format)
-# logging.basicConfig(filename='ayrton.log', level=logging.DEBUG, format=log_format, datefmt=date_format)
+def debug (level=logging.DEBUG, filename='ayrton.log'):
+    logging.basicConfig(filename=filename, filemode='a', level=level,
+                        format=log_format, datefmt=date_format)
+
+# uncomment next line and change level for way too much debugging
+# during test execution
+# debug (level=logging.DEBUG, filename='ayrton.%d.log' % os.getpid ())
 logger= logging.getLogger ('ayrton')
 
 # things that have to be defined before importing ayton.execute :(
@@ -47,14 +51,16 @@
 from ayrton.parser.astcompiler.astbuilder import ast_from_node
 from ayrton.ast_pprinter import pprint
 
-__version__= '0.7.1'
+__version__= '0.7.2'
 
 
 class ExecParams:
     def __init__ (self, **kwargs):
         # defaults
         self.trace= False
         self.linenos= False
+        self.trace_all= False
+        self.debug= False
 
         self.__dict__.update (kwargs)
 
@@ -289,7 +295,11 @@ def run_code (self, code, file_name, argv=None):
     def wait_for_pending_children (self):
         for i in range (len (self.pending_children)):
             child= self.pending_children.pop (0)
-            child.wait ()
+            try:
+                child.wait ()
+            except ChildProcessError:
+                # most probably is was alredy waited
+                logger.debug ('waiting for %s failed; ignoring', child)
 
 
     def global_tracer (self, frame, event, arg):
@@ -305,24 +315,40 @@ def local_tracer (self, frame, event, arg):
         if event=='line':
             file_name= frame.f_code.co_filename
             if self.params.trace_all or file_name==self.file_name:
+                l= len (file_name)
+                if l>32:
+                    short_file_name= file_name[l-32:]
+                else:
+                    short_file_name= file_name
+
                 lineno= frame.f_lineno
 
                 line= linecache.getline (file_name, lineno).rstrip ()
                 if line=='':
                     line= self.script[lineno-1].rstrip ()  # line numbers start at 1
 
                 logger.debug2 ('trace e: %s, f: %s, n: %d, l: %s', event, file_name, lineno, line)
-                if self.params.linenos:
-                    print ("+ [%6d] %s" % (lineno, line), file=sys.stderr)
+                if self.params.trace_all:
+                    self.trace_line ("+ [%32s:%-6d] %s", short_file_name, lineno, line)
+                elif self.params.linenos:
+                    self.trace_line ("+ [%6d] %s", lineno, line)
                 else:
-                    print ("+ %s" % line, file=sys.stderr)
+                    self.trace_line ("+ %s", line)
+
+
+    def trace_line (self, msg, *args):
+        if self.params.debug and False:
+            logger.debug (msg, *args)
+        else:
+            print (msg % args, file=sys.stderr)
 
 
 def run_tree (tree, g, l):
     """main entry point for remote()"""
     runner= Ayrton (g=g, l=l)
     return runner.run_tree (tree, 'unknown_tree')
 
+
 def run_file_or_script (script=None, file_name='script_from_command_line',
                         argv=None, params=None, **kwargs):
     """Main entry point for bin/ayrton and unittests."""

ayrton/castt.py

@@ -148,6 +148,7 @@ def bind (self, o):
             name= o.id
 
         elif type (o)==str:
+            # NOTE: when?
             name= o
 
         elif type (o)==Tuple:
@@ -160,11 +161,13 @@ def bind (self, o):
                 self.bind (e)
 
         if name is not None:
+            logger.debug ('binding "%s"', name)
             self.known_names[name]+= 1
             self.defined_names[self.stack].append (name)
             self.seen_names.add (name)
 
     def unbind (self, name, remove_from_stack=False):
+        logger.debug ('unbinding "%s"', name)
         self.known_names[name]-= 1
         if remove_from_stack:
             self.defined_names[self.stack].remove (name)
@@ -244,10 +247,10 @@ def visit_For (self, node):
         # For(target=Tuple(elts=[Name(id='band', ctx=Store()), Name(id='color', ctx=Store())], ctx=Store()),
         #     iter=Tuple(elts=[...], ctx=Load()),
         #     body=[Pass()], orelse=[])
-        self.generic_visit (node)
         self.bind (node.target)
+        self.generic_visit (node)
 
-        # if iter is Command, _out=Capture
+        # if iter is Command, set _out=Capture
         # so this works as expected:
         # for line in ls(): ...
 

ayrton/execute.py

@@ -411,27 +411,16 @@ def parent (self):
         # successfully.
         if not self.options['_bg']:
             self.wait ()
+            # NOTE: uhm?
             ayrton.runner.wait_for_pending_children ()
         else:
             ayrton.runner.pending_children.append (self)
 
+
     def wait (self):
+        logger.debug (self.child_pid)
         self._exit_code= os.waitpid (self.child_pid, 0)[1] >> 8
 
-        reader_pipe= None
-
-        if self.stdout_pipe is not None:
-            # this will also read stderr if both are Capture'd
-            reader_pipe= self.stdout_pipe
-        if self.stderr_pipe is not None:
-            reader_pipe= self.stderr_pipe
-
-        if reader_pipe is not None and self.options.get ('_out', None)!=Pipe:
-            r, w= reader_pipe
-            logger.debug ('closing %d', w)
-            os.close (w)
-            self.capture_file= open (r)
-
         if self._exit_code==127:
             # NOTE: when running bash, it returns 127 when it can't find the script to run
             raise CommandNotFound (self.path)
@@ -442,11 +431,13 @@ def wait (self):
 
             raise CommandFailed (self)
 
+
     def exit_code (self):
         if self._exit_code is None:
             self.wait ()
         return self._exit_code
 
+
     def __call__ (self, *args, **kwargs):
         if self.exe is None:
             raise CommandNotFound (self.path)
@@ -494,37 +485,74 @@ def __call__ (self, *args, **kwargs):
 
         return self
 
+
     def __bool__ (self):
         if self._exit_code is None:
             self.wait ()
         return self._exit_code==0
 
+
+    def prepare_capture_file (self):
+        if self.capture_file is None:
+            reader_pipe= None
+
+            if self.stdout_pipe is not None:
+                # this will also read stderr if both are Capture'd
+                reader_pipe= self.stdout_pipe
+            if self.stderr_pipe is not None:
+                reader_pipe= self.stderr_pipe
+
+            if reader_pipe is not None and self.options.get ('_out', None)!=Pipe:
+                r, w= reader_pipe
+                logger.debug ('closing %d', w)
+                os.close (w)
+                self.capture_file= open (r, encoding=encoding)
+
+
     def __str__ (self):
         if self._exit_code is None:
             self.wait ()
+
+        self.prepare_capture_file ()
+
         return self.capture_file.read ()
 
+
     def __iter__ (self):
-        if self._exit_code is None:
-            self.wait ()
+        logger.debug ('iterating!')
+
+        self.prepare_capture_file ()
+
         if self.capture_file is not None:
             for line in self.capture_file.readlines ():
                 if self.options['_chomp']:
                     line= line.rstrip (os.linesep)
 
+                logger.debug2 ('read line: %s', line)
                 yield line
 
             # finish him!
+            logger.debug ('finished!')
             self.capture_file.close ()
+            if self._exit_code is None:
+                self.wait ()
         else:
-            # TODO
-            pass
+            logger.debug ('dunno what to do!')
+
+    def readlines (self):
+        if self._exit_code is None:
+            self.wait ()
+
+        # ugly way to not leak the file()
+        return ( line for line in self )
 
-    # BUG this method is leaking an opend file()
+    # BUG this method is leaking an opened file()
     # self.capture_file
     def readline (self):
         if self._exit_code is None:
             self.wait ()
+
+        self.prepare_capture_file ()
         line= self.capture_file.readline ()
         if self.options['_chomp']:
             line= line.rstrip (os.linesep)
@@ -536,12 +564,6 @@ def close (self):
             self.wait ()
         self.capture_file.close ()
 
-    def readlines (self):
-        if self._exit_code is None:
-            self.wait ()
-        # ugly way to not leak the file()
-        return ( line for line in self )
-
     def __del__ (self):
         # finish it
         if self._exit_code is None and self.child_pid is not None: