From a4d394b2a18cd170a0ebbaa5073a3657c74a4845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Jos=C3=A9=20Conti?= Date: Sun, 6 Sep 2015 19:04:34 -0300 Subject: [PATCH 01/81] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 029d0a4..1c88d62 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Nothing fancy, right? Let's try something slightly different: echo ('Hello, World!') -Not interested yet? What if I tell you that that `echo` function just +Not interested yet? What if I tell you that `echo` function just executed `/bin/echo`?: mdione@diablo:~/src/projects/ayrton$ strace -e process -ff ayrton doc/examples/hw.ay From 4ff2e600092a09733ecd0e7cbed006642db36964 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:24:43 +0200 Subject: [PATCH 02/81] [*] s/column/col_offset/. no wonder I was getting confused with these two names. --- ayrton/castt.py | 6 +- ayrton/parser/astcompiler/astbuilder.py | 150 ++++++++++++------------ ayrton/parser/pyparser/parser.py | 6 +- ayrton/parser/pyparser/pyparse.py | 2 +- 4 files changed, 82 insertions(+), 82 deletions(-) diff --git a/ayrton/castt.py b/ayrton/castt.py index 4c74729..9ca40fb 100644 --- a/ayrton/castt.py +++ b/ayrton/castt.py @@ -495,7 +495,7 @@ def visit_Call (self, node): if is_option (arg): kw_expr= arg.keywords[0].arg if not isinstance (kw_expr, ast.Name) and not isinstance (kw_expr, str): - raise SyntaxError (self.file_name, node.lineno, node.column, + raise SyntaxError (self.file_name, node.lineno, node.col_offset, "keyword can't be an expression") if isinstance (kw_expr, ast.Name): @@ -504,7 +504,7 @@ def visit_Call (self, node): kw_name= kw_expr # str if kw_name in used_keywords: - raise SyntaxError (self.file_name, node.lineno, node.column, + raise SyntaxError (self.file_name, node.lineno, node.col_offset, "keyword argument repeated") # convert the expr into a str @@ -514,7 +514,7 @@ def visit_Call (self, node): first_kw= True else: if first_kw: - raise SyntaxError (self.file_name, node.lineno, node.column, + raise SyntaxError (self.file_name, node.lineno, node.col_offset, "non-keyword arg after keyword arg") new_args.append (arg) diff --git a/ayrton/parser/astcompiler/astbuilder.py b/ayrton/parser/astcompiler/astbuilder.py index 6feb2ec..1460aad 100644 --- a/ayrton/parser/astcompiler/astbuilder.py +++ b/ayrton/parser/astcompiler/astbuilder.py @@ -112,7 +112,7 @@ def number_of_statements(self, n): def error(self, msg, n): """Raise a SyntaxError with the lineno and column set to n's.""" - raise SyntaxError(msg, n.lineno, n.column, + raise SyntaxError(msg, n.lineno, n.col_offset, filename=self.compile_info.filename) def error_ast(self, msg, ast_node): @@ -149,7 +149,7 @@ def handle_del_stmt(self, del_node): targets = self.handle_exprlist(del_node.children[1], ast.Del()) new_node = ast.Delete (targets) new_node.lineno = ( del_node.lineno) - new_node.column = del_node.column + new_node.col_offset = del_node.col_offset return new_node def handle_flow_stmt(self, flow_node): @@ -158,18 +158,18 @@ def handle_flow_stmt(self, flow_node): if first_child_type == syms.break_stmt: new_node = ast.Break () new_node.lineno = flow_node.lineno - new_node.column = flow_node.column + new_node.col_offset = flow_node.col_offset return new_node elif first_child_type == syms.continue_stmt: new_node = ast.Continue () new_node.lineno = flow_node.lineno - new_node.column = flow_node.column + new_node.col_offset = flow_node.col_offset return new_node elif first_child_type == syms.yield_stmt: yield_expr = self.handle_expr(first_child.children[0]) new_node = ast.Expr (yield_expr) new_node.lineno = flow_node.lineno - new_node.column = flow_node.column + new_node.col_offset = flow_node.col_offset return new_node elif first_child_type == syms.return_stmt: if len(first_child.children) == 1: @@ -178,7 +178,7 @@ def handle_flow_stmt(self, flow_node): values = self.handle_testlist(first_child.children[1]) new_node = ast.Return (values) new_node.lineno = flow_node.lineno - new_node.column = flow_node.column + new_node.col_offset = flow_node.col_offset return new_node elif first_child_type == syms.raise_stmt: exc = None @@ -190,7 +190,7 @@ def handle_flow_stmt(self, flow_node): cause = self.handle_expr(first_child.children[3]) new_node = ast.Raise (exc, cause) new_node.lineno = flow_node.lineno - new_node.column = flow_node.column + new_node.col_offset = flow_node.col_offset return new_node else: raise AssertionError("unknown flow statement") @@ -241,7 +241,7 @@ def handle_import_stmt(self, import_node): for i in range(0, len(dotted_as_names.children), 2)] new_node = ast.Import (aliases) new_node.lineno = import_node.lineno - new_node.column = import_node.column + new_node.col_offset = import_node.col_offset return new_node elif import_node.type == syms.import_from: child_count = len(import_node.children) @@ -287,7 +287,7 @@ def handle_import_stmt(self, import_node): modname = module.name new_node = ast.ImportFrom (modname, aliases, dot_count) new_node.lineno = import_node.lineno - new_node.column = import_node.column + new_node.col_offset = import_node.col_offset return new_node else: raise AssertionError("unknown import node") @@ -297,7 +297,7 @@ def handle_global_stmt(self, global_node): for i in range(1, len(global_node.children), 2)] new_node = ast.Global (names) new_node.lineno = global_node.lineno - new_node.column = global_node.column + new_node.col_offset = global_node.col_offset return new_node def handle_nonlocal_stmt(self, nonlocal_node): @@ -305,7 +305,7 @@ def handle_nonlocal_stmt(self, nonlocal_node): for i in range(1, len(nonlocal_node.children), 2)] new_node = ast.Nonlocal (names) new_node.lineno = nonlocal_node.lineno - new_node.column = nonlocal_node.column + new_node.col_offset = nonlocal_node.col_offset return new_node def handle_assert_stmt(self, assert_node): @@ -315,7 +315,7 @@ def handle_assert_stmt(self, assert_node): msg = self.handle_expr(assert_node.children[3]) new_node = ast.Assert (expr, msg) new_node.lineno = assert_node.lineno - new_node.column = assert_node.column + new_node.col_offset = assert_node.col_offset return new_node def handle_suite(self, suite_node): @@ -349,7 +349,7 @@ def handle_if_stmt(self, if_node): suite = self.handle_suite(if_node.children[3]) new_node = ast.If (test, suite, []) new_node.lineno = if_node.lineno - new_node.column = if_node.column + new_node.col_offset = if_node.col_offset return new_node otherwise_string = if_node.children[4].value if otherwise_string == "else": @@ -358,7 +358,7 @@ def handle_if_stmt(self, if_node): else_suite = self.handle_suite(if_node.children[6]) new_node = ast.If (test, suite, else_suite) new_node.lineno = if_node.lineno - new_node.column = if_node.column + new_node.col_offset = if_node.col_offset return new_node elif otherwise_string == "elif": elif_count = child_count - 4 @@ -377,7 +377,7 @@ def handle_if_stmt(self, if_node): else_body = self.handle_suite(if_node.children[-1]) new_node = ast.If(last_elif_test, elif_body, else_body) new_node.lineno = last_elif.lineno - new_node.column = last_elif.column + new_node.col_offset = last_elif.col_offset otherwise = [new_node] elif_count -= 1 else: @@ -389,13 +389,13 @@ def handle_if_stmt(self, if_node): elif_body = self.handle_suite(if_node.children[offset + 2]) new_if = ast.If (elif_test, elif_body, otherwise) new_if.lineno = elif_test_node.lineno - new_if.column = elif_test_node.column + new_if.col_offset = elif_test_node.col_offset otherwise = [new_if] expr = self.handle_expr(if_node.children[1]) body = self.handle_suite(if_node.children[3]) new_node = ast.If (expr, body, otherwise) new_node.lineno = if_node.lineno - new_node.column = if_node.column + new_node.col_offset = if_node.col_offset return new_node else: raise AssertionError("unknown if statement configuration") @@ -409,7 +409,7 @@ def handle_while_stmt(self, while_node): otherwise = [] new_node = ast.While (loop_test, body, otherwise) new_node.lineno = while_node.lineno - new_node.column = while_node.column + new_node.col_offset = while_node.col_offset return new_node def handle_for_stmt(self, for_node): @@ -420,7 +420,7 @@ def handle_for_stmt(self, for_node): else: target = ast.Tuple (target_as_exprlist, ast.Store()) target.lineno = target_node.lineno - target.column = target_node.column + target.col_offset = target_node.col_offset expr = self.handle_testlist(for_node.children[3]) body = self.handle_suite(for_node.children[5]) if len(for_node.children) == 9: @@ -429,7 +429,7 @@ def handle_for_stmt(self, for_node): otherwise = [] new_node = ast.For (target, expr, body, otherwise) new_node.lineno = for_node.lineno - new_node.column = for_node.column + new_node.col_offset = for_node.col_offset return new_node def handle_except_clause(self, exc, body): @@ -445,7 +445,7 @@ def handle_except_clause(self, exc, body): self.check_forbidden_name(name, name_node) new_node = ast.ExceptHandler (test, name, suite) new_node.lineno = exc.lineno - new_node.column = exc.column + new_node.col_offset = exc.col_offset return new_node def handle_try_stmt(self, try_node): @@ -475,7 +475,7 @@ def handle_try_stmt(self, try_node): handlers.append(self.handle_except_clause(exc, except_body)) new_node = ast.Try (body, handlers, otherwise, finally_suite) new_node.lineno = try_node.lineno - new_node.column = try_node.column + new_node.col_offset = try_node.col_offset return new_node def handle_with_stmt(self, with_node): @@ -492,7 +492,7 @@ def handle_with_stmt(self, with_node): target = None wi = ast.With (test, target, body) wi.lineno = with_node.lineno - wi.column = with_node.column + wi.col_offset = with_node.col_offset if i == 1: break body = [wi] @@ -513,7 +513,7 @@ def handle_with_stmt(self, with_node): for i in range(1, len(with_node.children)-2, 2)] new_node = ast.With (items, body) new_node.lineno = with_node.lineno - new_node.column = with_node.column + new_node.col_offset = with_node.col_offset return new_node def handle_classdef(self, classdef_node, decorators=None): @@ -527,14 +527,14 @@ def handle_classdef(self, classdef_node, decorators=None): body = self.handle_suite(classdef_node.children[3]) new_node = ast.ClassDef (name, [], [], None, None, body, decorators) new_node.lineno = classdef_node.lineno - new_node.column = classdef_node.column + new_node.col_offset = classdef_node.col_offset return new_node if classdef_node.children[3].type == tokens.RPAR: # class NAME '(' ')' ':' suite body = self.handle_suite(classdef_node.children[5]) new_node = ast.ClassDef (name, [], [], None, None, body, decorators) new_node.lineno = classdef_node.lineno - new_node.column = classdef_node.column + new_node.col_offset = classdef_node.col_offset return new_node # class NAME '(' arglist ')' ':' suite @@ -544,12 +544,12 @@ def handle_classdef(self, classdef_node, decorators=None): # and handle_suite() to parse the body call_name = ast.Name (name, ast.Load()) call_name.lineno = classdef_node.lineno - call_name.column = classdef_node.column + call_name.col_offset = classdef_node.col_offset call = self.handle_call(classdef_node.children[3], call_name) body = self.handle_suite(classdef_node.children[6]) new_node = ast.ClassDef (name, call.args, call.keywords, call.starargs, call.kwargs, body, decorators) new_node.lineno = classdef_node.lineno - new_node.column = classdef_node.column + new_node.col_offset = classdef_node.col_offset return new_node def handle_class_bases(self, bases_node): @@ -572,7 +572,7 @@ def handle_funcdef(self, funcdef_node, decorators=None): body = self.handle_suite(funcdef_node.children[suite]) new_node = ast.FunctionDef (name, args, body, decorators, returns) new_node.lineno = funcdef_node.lineno - new_node.column = funcdef_node.column + new_node.col_offset = funcdef_node.col_offset return new_node def handle_decorated(self, decorated_node): @@ -585,7 +585,7 @@ def handle_decorated(self, decorated_node): else: raise AssertionError("unkown decorated") node.lineno = decorated_node.lineno - node.col_offset = decorated_node.column + node.col_offset = decorated_node.col_offset return node def handle_decorators(self, decorators_node): @@ -598,7 +598,7 @@ def handle_decorator(self, decorator_node): elif len(decorator_node.children) == 5: dec = ast.Call (dec_name, None, None, None, None) dec.lineno = decorator_node.lineno - dec.column = decorator_node.column + dec.col_offset = decorator_node.col_offset else: dec = self.handle_call(decorator_node.children[3], dec_name) return dec @@ -607,13 +607,13 @@ def handle_dotted_name(self, dotted_name_node): base_value = self.new_identifier(dotted_name_node.children[0].value) name = ast.Name (base_value, ast.Load()) name.lineno = dotted_name_node.lineno - name.column = dotted_name_node.column + name.col_offset = dotted_name_node.col_offset for i in range(2, len(dotted_name_node.children), 2): attr = dotted_name_node.children[i].value attr = self.new_identifier(attr) name = ast.Attribute (name, attr, ast.Load()) name.lineno = dotted_name_node.lineno - name.column = dotted_name_node.column + name.col_offset = dotted_name_node.col_offset return name def handle_arguments(self, arguments_node): @@ -777,7 +777,7 @@ def handle_stmt(self, stmt): elif stmt_type == syms.pass_stmt: new_node = ast.Pass () new_node.lineno = stmt.lineno - new_node.column = stmt.column + new_node.col_offset = stmt.col_offset return new_node elif stmt_type == syms.flow_stmt: return self.handle_flow_stmt(stmt) @@ -820,7 +820,7 @@ def handle_expr_stmt(self, stmt): expression = self.handle_testlist(stmt.children[0]) new_node = ast.Expr (expression) new_node.lineno = stmt.lineno - new_node.column = stmt.column + new_node.col_offset = stmt.col_offset return new_node elif stmt.children[1].type == syms.augassign: # Augmented assignment. @@ -836,7 +836,7 @@ def handle_expr_stmt(self, stmt): operator = augassign_operator_map[op_str] new_node = ast.AugAssign (target_expr, operator(), value_expr) new_node.lineno = stmt.lineno - new_node.column = stmt.column + new_node.col_offset = stmt.col_offset return new_node else: # Normal assignment. @@ -856,7 +856,7 @@ def handle_expr_stmt(self, stmt): value_expr = self.handle_expr(value_child) new_node = ast.Assign (targets, value_expr) new_node.lineno = stmt.lineno - new_node.column = stmt.column + new_node.col_offset = stmt.col_offset return new_node def get_expression_list(self, tests): @@ -870,7 +870,7 @@ def handle_testlist(self, tests): elts = self.get_expression_list(tests) new_node = ast.Tuple (elts, ast.Load()) new_node.lineno = tests.lineno - new_node.column = tests.column + new_node.col_offset = tests.col_offset return new_node def handle_expr(self, expr_node): @@ -898,7 +898,7 @@ def handle_expr(self, expr_node): op = ast.And new_node = ast.BoolOp (op(), seq) new_node.lineno = expr_node.lineno - new_node.column = expr_node.column + new_node.col_offset = expr_node.col_offset return new_node elif expr_node_type == syms.not_test: if len(expr_node.children) == 1: @@ -907,7 +907,7 @@ def handle_expr(self, expr_node): expr = self.handle_expr(expr_node.children[1]) new_node = ast.UnaryOp (ast.Not(), expr) new_node.lineno = expr_node.lineno - new_node.column = expr_node.column + new_node.col_offset = expr_node.col_offset return new_node elif expr_node_type == syms.comparison: if len(expr_node.children) == 1: @@ -921,7 +921,7 @@ def handle_expr(self, expr_node): operands.append(self.handle_expr(expr_node.children[i + 1])) new_node = ast.Compare (expr, operators, operands) new_node.lineno = expr_node.lineno - new_node.column = expr_node.column + new_node.col_offset = expr_node.col_offset return new_node elif expr_node_type == syms.star_expr: return self.handle_star_expr(expr_node) @@ -949,11 +949,11 @@ def handle_expr(self, expr_node): if is_from: new_node = ast.YieldFrom (expr) new_node.lineno = expr_node.lineno - new_node.column = expr_node.column + new_node.col_offset = expr_node.col_offset return new_node new_node = ast.Yield (expr) new_node.lineno = expr_node.lineno - new_node.column = expr_node.column + new_node.col_offset = expr_node.col_offset return new_node elif expr_node_type == syms.factor: if len(expr_node.children) == 1: @@ -969,7 +969,7 @@ def handle_star_expr(self, star_expr_node): expr = self.handle_expr(star_expr_node.children[1]) new_node = ast.Starred (expr, ast.Load()) new_node.lineno = star_expr_node.lineno - new_node.column = star_expr_node.column + new_node.col_offset = star_expr_node.col_offset return new_node def handle_lambdef(self, lambdef_node): @@ -980,7 +980,7 @@ def handle_lambdef(self, lambdef_node): args = self.handle_arguments(lambdef_node.children[1]) new_node = ast.Lambda (args, expr) new_node.lineno = lambdef_node.lineno - new_node.column = lambdef_node.column + new_node.col_offset = lambdef_node.col_offset return new_node def handle_ifexp(self, if_expr_node): @@ -989,7 +989,7 @@ def handle_ifexp(self, if_expr_node): otherwise = self.handle_expr(if_expr_node.children[4]) new_node = ast.IfExp (expression, body, otherwise) new_node.lineno = if_expr_node.lineno - new_node.column = if_expr_node.column + new_node.col_offset = if_expr_node.col_offset return new_node def handle_comp_op(self, comp_op_node): @@ -1036,7 +1036,7 @@ def handle_binop(self, binop_node): op = operator_map(binop_node.children[1].type) result = ast.BinOp (left, op(), right) result.lineno = binop_node.lineno - result.column = binop_node.column + result.col_offset = binop_node.col_offset number_of_ops = (len(binop_node.children) - 1) // 2 for i in range(1, number_of_ops): op_node = binop_node.children[i * 2 + 1] @@ -1044,7 +1044,7 @@ def handle_binop(self, binop_node): sub_right = self.handle_expr(binop_node.children[i * 2 + 2]) result = ast.BinOp (result, op(), sub_right) result.lineno = op_node.lineno - result.column = op_node.column + result.col_offset = op_node.col_offset return result def handle_factor(self, factor_node): @@ -1060,7 +1060,7 @@ def handle_factor(self, factor_node): raise AssertionError("invalid factor node") new_node = ast.UnaryOp (op(), expr) new_node.lineno = factor_node.lineno - new_node.column = factor_node.column + new_node.col_offset = factor_node.col_offset return new_node def handle_power(self, power_node): @@ -1073,13 +1073,13 @@ def handle_power(self, power_node): break tmp_atom_expr = self.handle_trailer(trailer, atom_expr) tmp_atom_expr.lineno = atom_expr.lineno - tmp_atom_expr.column = atom_expr.column + tmp_atom_expr.col_offset = atom_expr.col_offset atom_expr = tmp_atom_expr if power_node.children[-1].type == syms.factor: right = self.handle_expr(power_node.children[-1]) atom_expr = ast.BinOp (atom_expr, ast.Pow(), right) atom_expr.lineno = power_node.lineno - atom_expr.column = power_node.column + atom_expr.col_offset = power_node.col_offset return atom_expr def handle_slice(self, slice_node): @@ -1115,7 +1115,7 @@ def handle_trailer(self, trailer_node, left_expr): if len(trailer_node.children) == 2: new_node = ast.Call (left_expr, [], [], None, None) new_node.lineno = trailer_node.lineno - new_node.column = trailer_node.column + new_node.col_offset = trailer_node.col_offset return new_node else: return self.handle_call(trailer_node.children[1], left_expr) @@ -1123,7 +1123,7 @@ def handle_trailer(self, trailer_node, left_expr): attr = self.new_identifier(trailer_node.children[1].value) new_node = ast.Attribute (left_expr, attr, ast.Load()) new_node.lineno = trailer_node.lineno - new_node.column = trailer_node.column + new_node.col_offset = trailer_node.col_offset return new_node else: middle = trailer_node.children[1] @@ -1131,7 +1131,7 @@ def handle_trailer(self, trailer_node, left_expr): slice = self.handle_slice(middle.children[0]) new_node = ast.Subscript (left_expr, slice, ast.Load()) new_node.lineno = middle.lineno - new_node.column = middle.column + new_node.col_offset = middle.col_offset return new_node slices = [] simple = True @@ -1144,7 +1144,7 @@ def handle_trailer(self, trailer_node, left_expr): ext_slice = ast.ExtSlice(slices) new_node = ast.Subscript (left_expr, ext_slice, ast.Load()) new_node.lineno = middle.lineno - new_node.column = middle.column + new_node.col_offset = middle.col_offset return new_node elts = [] for idx in slices: @@ -1152,10 +1152,10 @@ def handle_trailer(self, trailer_node, left_expr): elts.append(idx.value) tup = ast.Tuple (elts, ast.Load()) tup.lineno = middle.lineno - tup.column = middle.column + tup.col_offset = middle.col_offset new_node = ast.Subscript(left_expr, ast.Index (tup), ast.Load()) new_node.lineno = middle.lineno - new_node.column = middle.column + new_node.col_offset = middle.col_offset return new_node def handle_call(self, args_node, callable_expr): @@ -1208,13 +1208,13 @@ def handle_call(self, args_node, callable_expr): else: kw = ast.keyword(keyword, keyword_value) kw.lineno = keyword_node.lineno - kw.col_offset = keyword_node.column + kw.col_offset = keyword_node.col_offset name = ast.Name ('o', ast.Load()) name.lineno = keyword_node.lineno - name.column = keyword_node.column + name.col_offset = keyword_node.col_offset arg = ast.Call(name, [], [ kw ], None, None) arg.lineno = keyword_node.lineno - arg.column = keyword.column + arg.col_offset = keyword.col_offset args.append(arg) elif argument.type == tokens.STAR: variable_arg = self.handle_expr(args_node.children[i + 1]) @@ -1229,7 +1229,7 @@ def handle_call(self, args_node, callable_expr): keywords = [] new_node = ast.Call(callable_expr, args, keywords, variable_arg, keywords_arg) new_node.lineno = callable_expr.lineno - new_node.column = callable_expr.column + new_node.col_offset = callable_expr.col_offset return new_node def parse_number(self, raw): @@ -1242,7 +1242,7 @@ def handle_atom(self, atom_node): name = self.new_identifier(first_child.value) new_node = ast.Name (name, ast.Load()) new_node.lineno = first_child.lineno - new_node.column = first_child.column + new_node.col_offset = first_child.col_offset return new_node elif first_child_type == tokens.STRING: space = self.space @@ -1272,25 +1272,25 @@ def handle_atom(self, atom_node): node_cls = ast.Str if strdata else ast.Bytes new_node = node_cls(w_string) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node elif first_child_type == tokens.NUMBER: num_value = self.parse_number(first_child.value) new_node = ast.Num (num_value) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node elif first_child_type == tokens.ELLIPSIS: new_node = ast.Ellipsis () new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node elif first_child_type == tokens.LPAR: second_child = atom_node.children[1] if second_child.type == tokens.RPAR: new_node = ast.Tuple (None, ast.Load()) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node elif second_child.type == syms.yield_expr: return self.handle_expr(second_child) @@ -1300,14 +1300,14 @@ def handle_atom(self, atom_node): if second_child.type == tokens.RSQB: new_node = ast.List (None, ast.Load()) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node if len(second_child.children) == 1 or \ second_child.children[1].type == tokens.COMMA: elts = self.get_expression_list(second_child) new_node = ast.List (elts, ast.Load()) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node return self.handle_listcomp(second_child) elif first_child_type == tokens.LBRACE: @@ -1315,7 +1315,7 @@ def handle_atom(self, atom_node): if maker.type == tokens.RBRACE: new_node = ast.Dict (None, None) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node n_maker_children = len(maker.children) if n_maker_children == 1 or maker.children[1].type == tokens.COMMA: @@ -1324,7 +1324,7 @@ def handle_atom(self, atom_node): elts.append(self.handle_expr(maker.children[i])) new_node = ast.Set (elts) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node if maker.children[1].type == syms.comp_for: return self.handle_setcomp(maker) @@ -1338,7 +1338,7 @@ def handle_atom(self, atom_node): values.append(self.handle_expr(maker.children[i + 2])) new_node = ast.Dict (keys, values) new_node.lineno = atom_node.lineno - new_node.column = atom_node.column + new_node.col_offset = atom_node.col_offset return new_node else: raise AssertionError("unknown atom") @@ -1424,7 +1424,7 @@ def handle_genexp(self, genexp_node): comps = self.comprehension_helper(genexp_node.children[1]) new_node = ast.GeneratorExp (elt, comps) new_node.lineno = genexp_node.lineno - new_node.column = genexp_node.column + new_node.col_offset = genexp_node.col_offset return new_node def handle_listcomp(self, listcomp_node): @@ -1432,7 +1432,7 @@ def handle_listcomp(self, listcomp_node): comps = self.comprehension_helper(listcomp_node.children[1]) new_node = ast.ListComp (elt, comps) new_node.lineno = listcomp_node.lineno - new_node.column = listcomp_node.column + new_node.col_offset = listcomp_node.col_offset return new_node def handle_setcomp(self, set_maker): @@ -1440,7 +1440,7 @@ def handle_setcomp(self, set_maker): comps = self.comprehension_helper(set_maker.children[1]) new_node = ast.SetComp (elt, comps) new_node.lineno = set_maker.lineno - new_node.column = set_maker.column + new_node.col_offset = set_maker.col_offset return new_node def handle_dictcomp(self, dict_maker): @@ -1449,7 +1449,7 @@ def handle_dictcomp(self, dict_maker): comps = self.comprehension_helper(dict_maker.children[3]) new_node = ast.DictComp (key, value, comps) new_node.lineno = dict_maker.lineno - new_node.column = dict_maker.column + new_node.col_offset = dict_maker.col_offset return new_node def handle_exprlist(self, exprlist, context): diff --git a/ayrton/parser/pyparser/parser.py b/ayrton/parser/pyparser/parser.py index ed30523..373a541 100644 --- a/ayrton/parser/pyparser/parser.py +++ b/ayrton/parser/pyparser/parser.py @@ -44,14 +44,14 @@ def _freeze_(self): class Node(object): - __slots__ = "type value children lineno column".split() + __slots__ = "type value children lineno col_offset".split() def __init__(self, type, value, children, lineno, column): self.type = type self.value = value self.children = children self.lineno = lineno - self.column = column + self.col_offset = column def __eq__(self, other): # For tests. @@ -74,7 +74,7 @@ def __init__(self, msg, token_type, value, lineno, column, line, self.token_type = token_type self.value = value self.lineno = lineno - self.column = column + self.col_offset = column self.line = line self.expected = expected diff --git a/ayrton/parser/pyparser/pyparse.py b/ayrton/parser/pyparser/pyparse.py index be31dfd..7b3adce 100644 --- a/ayrton/parser/pyparser/pyparse.py +++ b/ayrton/parser/pyparser/pyparse.py @@ -210,7 +210,7 @@ def parse_source(self, bytessrc, compile_info): else: new_err = error.SyntaxError msg = "invalid syntax" - raise new_err(msg, e.lineno, e.column, e.line, + raise new_err(msg, e.lineno, e.col_offset, e.line, compile_info.filename) else: tree = self.root From 4ef8195e637b4157f1830b3be9a221a81a154413 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:26:07 +0200 Subject: [PATCH 03/81] [*] unique values for tests, so we can identify AST dumps more easily. --- ayrton/tests/test_ayrton.py | 34 +++++++++++++++++----------------- ayrton/tests/test_castt.py | 6 +++--- ayrton/tests/test_execute.py | 4 ++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py index 308954c..f128628 100644 --- a/ayrton/tests/test_ayrton.py +++ b/ayrton/tests/test_ayrton.py @@ -250,7 +250,7 @@ def testLt (self): def testLtGt (self): fd, fn1= tempfile.mkstemp () - os.write (fd, b'42\n') + os.write (fd, b'43\n') os.close (fd) fn2= tempfile.mkstemp ()[1] @@ -258,7 +258,7 @@ def testLtGt (self): contents= open (fn2).read () # read() does not return bytes! - self.assertEqual (contents, '42\n') + self.assertEqual (contents, '43\n') os.unlink (fn1) os.unlink (fn2) @@ -280,14 +280,14 @@ class MiscTests (unittest.TestCase): tearDown= tearDownMockStdout def testEnviron (self): - ayrton.main ('''export (TEST_ENV=42); + ayrton.main ('''export (TEST_ENV=44); run ("./ayrton/tests/data/test_environ.sh")''') # close stdout as per the description of setUpMockStdout() os.close (1) - self.assertEqual (self.r.read (), b'42\n') + self.assertEqual (self.r.read (), b'44\n') def testUnset (self): - ayrton.main ('''export (TEST_ENV=42) + ayrton.main ('''export (TEST_ENV=45) print (TEST_ENV) unset ("TEST_ENV") try: @@ -296,21 +296,21 @@ def testUnset (self): print ("yes")''') # close stdout as per the description of setUpMockStdout() os.close (1) - self.assertEqual (self.r.read (), b'42\nyes\n') + self.assertEqual (self.r.read (), b'45\nyes\n') def testEnvVarAsGlobalVar (self): - os.environ['testEnvVarAsLocalVar'] = '42' # envvars are strings only + os.environ['testEnvVarAsLocalVar'] = '46' # envvars are strings only ayrton.main ('print (testEnvVarAsLocalVar)') # close stdout as per the description of setUpMockStdout() os.close (1) - self.assertEqual (self.r.read (), b'42\n') + self.assertEqual (self.r.read (), b'46\n') def testExportSetsGlobalVar (self): - ayrton.main ('''export (foo=42); + ayrton.main ('''export (foo=47); print (foo)''') # close stdout as per the description of setUpMockStdout() os.close (1) - self.assertEqual (self.r.read (), b'42\n') + self.assertEqual (self.r.read (), b'47\n') def testCwdPwdRename (self): ayrton.main ('''import os.path; @@ -329,17 +329,17 @@ def testWithCd (self): def testShift (self): ayrton.main ('''a= shift (); -print (a)''', argv=['test_script.ay', '42']) +print (a)''', argv=['test_script.ay', '48']) # close stdout as per the description of setUpMockStdout() os.close (1) - self.assertEqual (self.r.read (), b'42\n') + self.assertEqual (self.r.read (), b'48\n') def testShifts (self): ayrton.main ('''a= shift (2); -print (a)''', argv=['test_script.ay', '42', '27']) +print (a)''', argv=['test_script.ay', '49', '27']) # close stdout as per the description of setUpMockStdout() os.close (1) - self.assertEqual (self.r.read (), b"['42', '27']\n") + self.assertEqual (self.r.read (), b"['49', '27']\n") def testO (self): # this should not explode @@ -483,13 +483,13 @@ def testDel (self): def testDefFun1 (self): ayrton.main ('''def foo (): - true= 42 + true= 40 true ()''') def testDefFun2 (self): self.assertRaises (ayrton.CommandFailed, ayrton.main, '''option ('errexit') def foo (): - false= 42 + false= 41 false ()''') def testDefFunFails1 (self): @@ -557,7 +557,7 @@ def testSimpleFor (self): class ReturnValues (unittest.TestCase): def testSimpleReturn (self): - self.assertEqual (ayrton.main ('''return 42'''), 42) + self.assertEqual (ayrton.main ('''return 50'''), 50) def testException (self): self.assertRaises (SystemError, ayrton.main, '''raise SystemError''') diff --git a/ayrton/tests/test_castt.py b/ayrton/tests/test_castt.py index d3e85b1..4def944 100644 --- a/ayrton/tests/test_castt.py +++ b/ayrton/tests/test_castt.py @@ -68,7 +68,7 @@ class TestVisits (unittest.TestCase): def testFunctionKeywords (self): c= castt.CrazyASTTransformer ({ 'dict': dict, 'o': o}) - t= ast.parse ("""dict (a=42)""") + t= ast.parse ("""dict (a=51)""") node= c.visit_Call (t.body[0].value) @@ -77,7 +77,7 @@ def testFunctionKeywords (self): def testFunctionOKeywords (self): c= castt.CrazyASTTransformer ({ 'dict': dict, 'o': o}) - t= ast.parse ("""dict (o (a=42))""") + t= ast.parse ("""dict (o (a=52))""") node= c.visit_Call (t.body[0].value) @@ -88,7 +88,7 @@ def testFunctionOArgs (self): # NOTE: I need to give the implementation for o(); # otherwise it will also be converted to Command() c= castt.CrazyASTTransformer ({ 'o': o}) - t= ast.parse ("""dict (o (a=42))""") + t= ast.parse ("""dict (o (a=53))""") node= c.visit_Call (t.body[0].value) diff --git a/ayrton/tests/test_execute.py b/ayrton/tests/test_execute.py index 4ff61c9..17b2ba9 100644 --- a/ayrton/tests/test_execute.py +++ b/ayrton/tests/test_execute.py @@ -110,9 +110,9 @@ def testOutNone (self): self.mock_stdout.close () def testOrderedOptions (self): - ayrton.main ("""echo (-l=True, --more=42, --ordered_options='yes!')""") + ayrton.main ("""echo (-l=True, --more=55, --ordered_options='yes!')""") tearDownMockStdOut (self) - self.assertEqual (self.mock_stdout.read (), '-l --more 42 --ordered_options yes!\n') + self.assertEqual (self.mock_stdout.read (), '-l --more 55 --ordered_options yes!\n') self.mock_stdout.close () def testEnvironment (self): From 34447f2d5fd8fd87747fd69507089c87aa718ac3 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:30:53 +0200 Subject: [PATCH 04/81] [*] more unique values for tests, so we can identify AST dumps more easily. --- ayrton/tests/test_castt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ayrton/tests/test_castt.py b/ayrton/tests/test_castt.py index 4def944..aa31730 100644 --- a/ayrton/tests/test_castt.py +++ b/ayrton/tests/test_castt.py @@ -172,8 +172,8 @@ def testDottedSubscript (self): self.assertEqual (combined, 'argv[3].split') def testDottedSubscriptComplex (self): - single, combined= castt.func_name2dotted_exec (parse_expression ('argv[3].split[:42]')) + single, combined= castt.func_name2dotted_exec (parse_expression ('argv[3].split[:54]')) self.assertEqual (single, 'argv') # this is a very strange but possible executable name - self.assertEqual (combined, 'argv[3].split[:42]') + self.assertEqual (combined, 'argv[3].split[:54]') From 7cd4617f5c90b94205ffa100d57cb778dcc70d7c Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:33:25 +0200 Subject: [PATCH 05/81] [*] some doc update. --- ChangeLog.rst | 19 ++++++++++++------- ayrton/castt.py | 2 +- ayrton/functions.py | 2 ++ ayrton/tests/test_ayrton.py | 9 +++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 05440b5..45fb021 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,10 +1,8 @@ -ayrton (0.5) UNRELEASED; urgency=medium +ayrton (0.5) unstable; urgency=medium - * source() is out. use python's import system. - * Support executing foo.py(). * Much better command detection. - * CommandNotFound exception is now a subclass of NameError. - * Allow Command keywords be named like '-l' and '--long-option', so it supports options with dashes. + * `CommandNotFound` exception is now a subclass of `NameError`. + * Allow `Command` keywords be named like `-l` and `--long-option`, so it supports options with single dashes (`-long-option`, à la `find`). * This also means that long-option is no longer passed as --long-option; you have to put the dashes explicitly. * bash() does not return a single string by default; override with single=True. * Way more tests. @@ -12,6 +10,13 @@ ayrton (0.5) UNRELEASED; urgency=medium -- Marcos Dione Sun, 30 Aug 2015 15:13:30 +0200 +ayrton (0.4.4) unstable; urgency=low + + * `source()` is out. use Python's import system. + * Support executing `foo.py()`. + + -- Marcos Dione Wed, 20 May 2015 23:44:42 +0200 + ayrton (0.4.3) unstable; urgency=medium * Let commands handle SIGPIE and SIGINT. Python does funky things to them. @@ -24,7 +29,7 @@ ayrton (0.4.3) unstable; urgency=medium ayrton (0.4.2) unstable; urgency=low * _bg allows running a command in the background. - * _fails allows a Command to fail even when option('-e') is on. + * _fails allows a Command to fail even when option('-e') is on. * Try program_name as program-name if the first failed the path lookup. * Convert all arguments to commands to str(). * chdir() is an alias of cd(). @@ -41,7 +46,7 @@ ayrton (0.4.2) unstable; urgency=low ayrton (0.4) unstable; urgency=low * >= can redirect stederr to stdout. - * o(option=argument) can be used to declare keyword params among/before + * o(option=argument) can be used to declare keyword params among/before positional ones. * bash() now returns a single string if there is only one result. * Slightly better error reporting: don't print a part of the stacktrace diff --git a/ayrton/castt.py b/ayrton/castt.py index 9ca40fb..bc06a7d 100644 --- a/ayrton/castt.py +++ b/ayrton/castt.py @@ -82,7 +82,7 @@ def func_name2dotted_exec (node): class CrazyASTTransformer (ast.NodeTransformer): def __init__ (self, environ, file_name=None): super ().__init__ () - # the whole ayrton environment; see ayrton.Environ. + # the whole ayrton instace globals self.environ= environ # names defined in the global namespace self.known_names= defaultdict (lambda: 0) diff --git a/ayrton/functions.py b/ayrton/functions.py index f47b380..17f6d24 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -151,6 +151,8 @@ def __enter__ (self): (i, o, e)= self.client.exec_command (command) else: + # to debug, run + # nc -l -s 127.0.0.1 -p 2233 -vv -e /bin/bash self.client= socket () self.client.connect ((self.hostname, 2233)) i= open (self.client.fileno (), 'wb') diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py index f128628..6b0e4ff 100644 --- a/ayrton/tests/test_ayrton.py +++ b/ayrton/tests/test_ayrton.py @@ -111,17 +111,22 @@ def setUpMockStdout (self): # I save the old stdout in a new fd self.old_stdout= os.dup (1) + # create a pipe; this gives me a read and write fd r, w= os.pipe () + # I replace the stdout with the write fd # this closes 1, but the original stdout is saved in old_stdout os.dup2 (w, 1) - # now I have to fds pointing to the write end of the pipe, stdout and w + + # now I have two fds pointing to the write end of the pipe, stdout and w # close w os.close (w) - # create me a file() from the reading fd + + # create a file() from the reading fd # this DOES NOT create a new fd or file self.r= open (r, mode='rb') + # the test will have to close stdin after performing what's testing # that's because otherwise the test locks at reading from the read end # because there's still that fd available for writing in the pipe From c5874c7ad5b0b982874566c5dbc9ff17abec1d2c Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:34:36 +0200 Subject: [PATCH 06/81] [+] check_attrs() to make sure .lineno and .col_offset are attributes of generated AST nodes. --- ayrton/tests/test_castt.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/ayrton/tests/test_castt.py b/ayrton/tests/test_castt.py index aa31730..5ba85ed 100644 --- a/ayrton/tests/test_castt.py +++ b/ayrton/tests/test_castt.py @@ -17,12 +17,27 @@ import unittest import ast -from ast import Attribute, Name, Load, Subscript, Index, Num +from ast import Attribute, Name, Load, Subscript, Index, Num, With +from ast import Break, Continue, Return, Raise, Yield +from ast import Del, Pass, Import, ImportFrom, Global, Assert from ayrton import castt from ayrton.execute import o +from ayrton.functions import cd import ayrton +flow_stmt= [ Break, Continue, Return, Raise, Yield, ] +expr_stmt= [ ] +small_stmt= expr_stmt + [ Del, Pass ] + flow_stmt + [ Import, ImportFrom, Global, + Assert ] + +def check_attrs (self, node): + for n in ast.walk (node): + if type (n) in small_stmt: + self.assertTrue (hasattr(node, 'lineno'), "%s.lineno not present" % ast.dump (node)) + self.assertTrue (hasattr(node, 'col_offset'), "%s.col_offset not present" % ast.dump (node)) + + class TestBinding (unittest.TestCase): def setUp (self): @@ -65,6 +80,7 @@ def parse_expression (s): return ast.parse (s).body[0].value class TestVisits (unittest.TestCase): + check_attrs= check_attrs def testFunctionKeywords (self): c= castt.CrazyASTTransformer ({ 'dict': dict, 'o': o}) @@ -74,6 +90,7 @@ def testFunctionKeywords (self): self.assertEqual (len (node.args), 0, ast.dump (node)) self.assertEqual (len (node.keywords), 1, ast.dump (node)) + self.check_attrs (node) def testFunctionOKeywords (self): c= castt.CrazyASTTransformer ({ 'dict': dict, 'o': o}) @@ -83,6 +100,7 @@ def testFunctionOKeywords (self): self.assertEqual (len (node.args), 0, ast.dump (node)) self.assertEqual (len (node.keywords), 1, ast.dump (node)) + self.check_attrs (node) def testFunctionOArgs (self): # NOTE: I need to give the implementation for o(); @@ -94,6 +112,7 @@ def testFunctionOArgs (self): self.assertEqual (len (node.args), 1, ast.dump (node)) self.assertEqual (len (node.keywords), 0, ast.dump (node)) + self.check_attrs (node) def testDoubleKeywordCommand (self): c= castt.CrazyASTTransformer ({ 'o': o}) @@ -109,6 +128,7 @@ def testDoubleKeywordCommand (self): self.assertEqual (node.args[0].keywords[0].arg, node.args[1].keywords[0].arg, "\n%s\n%s\n" % (ast.dump (node.args[0]), ast.dump (node.args[1]))) + self.check_attrs (node) def testDoubleKeywordFunction (self): c= castt.CrazyASTTransformer ({ 'o': o, 'dict': dict}) @@ -135,6 +155,7 @@ def testMinusMinusCommand (self): node= c.visit_Call (t.body[0].value) self.assertEqual (node.args[0].keywords[0].arg, '--p') + self.check_attrs (node) def testLongOptionCommand (self): c= castt.CrazyASTTransformer ({ 'o': o}) @@ -143,7 +164,8 @@ def testLongOptionCommand (self): node= c.visit_Call (t.body[0].value) self.assertEqual (node.args[0].keywords[0].arg, '--long-option') - + self.check_attrs (node) + class TestHelperFunctions (unittest.TestCase): def testName (self): From 0c41156348b94933c7b2d6710364879547f76c69 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:35:26 +0200 Subject: [PATCH 07/81] [+] TestWeirdErrors makes sure I don't bork again. not really functional tests. --- ayrton/tests/test_castt.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ayrton/tests/test_castt.py b/ayrton/tests/test_castt.py index 5ba85ed..4dd176d 100644 --- a/ayrton/tests/test_castt.py +++ b/ayrton/tests/test_castt.py @@ -199,3 +199,20 @@ def testDottedSubscriptComplex (self): self.assertEqual (single, 'argv') # this is a very strange but possible executable name self.assertEqual (combined, 'argv[3].split[:54]') + +class TestWeirdErrors (unittest.TestCase): + check_attrs= check_attrs + + def testWithCd (self): + code= """with cd() as e: pass""" + t= ast.parse (code) + self.check_attrs (t.body[0]) + + c= castt.CrazyASTTransformer ({ 'o': o, 'cd': cd}) + t= ayrton.parse (code) + self.check_attrs (t.body[0]) + + node= c.visit_With (t.body[0]) + + self.assertEqual (node.items[0].context_expr.func.id, 'cd') + self.check_attrs (node) From f26e94e1e6395f9f1d8d244243a2fb10408cc759 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:46:03 +0200 Subject: [PATCH 08/81] [*] better logging format. --- ayrton/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 717c84d..52bdf24 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -23,8 +23,12 @@ import ast import logging -# logging.basicConfig(filename='ayrton.%d.log' % os.getpid (),level=logging.DEBUG) -# logging.basicConfig(filename='ayrton.log',level=logging.DEBUG) +log_format= "%(asctime)s %(name)s:%(lineno)-4d %(levelname)-8s %(message)s" +date_format= "%H:%M:%S.%f" + +# uncomment one of these for way too much debugging :) +logging.basicConfig(filename='ayrton.%d.log' % os.getpid (), level=logging.DEBUG, format=log_format, datefmt=date_format) +# logging.basicConfig(filename='ayrton.log', level=logging.DEBUG, format=log_format, datefmt=date_format) logger= logging.getLogger ('ayrton') # things that have to be defined before importing ayton.execute :( From a61d68f14a4f0445f970ab73b4dea554cbc6933f Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:48:29 +0200 Subject: [PATCH 09/81] [*] better logging format. --- ayrton/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 52bdf24..eaf3ec7 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -24,7 +24,7 @@ import logging log_format= "%(asctime)s %(name)s:%(lineno)-4d %(levelname)-8s %(message)s" -date_format= "%H:%M:%S.%f" +date_format= "%H:%M:%S" # uncomment one of these for way too much debugging :) logging.basicConfig(filename='ayrton.%d.log' % os.getpid (), level=logging.DEBUG, format=log_format, datefmt=date_format) From d63f60f17168af156aa16adf47413bb57634e187 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:50:55 +0200 Subject: [PATCH 10/81] [*] s/lambda: []/list/. --- ayrton/castt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/castt.py b/ayrton/castt.py index bc06a7d..a359adf 100644 --- a/ayrton/castt.py +++ b/ayrton/castt.py @@ -92,7 +92,7 @@ def __init__ (self, environ, file_name=None): # holds the temporary namespaces in function and class definitions # key: the stack so far # value: list of names - self.defined_names= defaultdict (lambda: []) + self.defined_names= defaultdict (list) # for testing self.seen_names= set () self.file_name= file_name From d3aa096eb87af1775604eeac14bbc536d32a0ae9 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:51:33 +0200 Subject: [PATCH 11/81] [*] trailing space. --- ayrton/tests/test_castt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ayrton/tests/test_castt.py b/ayrton/tests/test_castt.py index 4dd176d..0012d76 100644 --- a/ayrton/tests/test_castt.py +++ b/ayrton/tests/test_castt.py @@ -28,7 +28,7 @@ flow_stmt= [ Break, Continue, Return, Raise, Yield, ] expr_stmt= [ ] -small_stmt= expr_stmt + [ Del, Pass ] + flow_stmt + [ Import, ImportFrom, Global, +small_stmt= expr_stmt + [ Del, Pass ] + flow_stmt + [ Import, ImportFrom, Global, Assert ] def check_attrs (self, node): @@ -36,7 +36,7 @@ def check_attrs (self, node): if type (n) in small_stmt: self.assertTrue (hasattr(node, 'lineno'), "%s.lineno not present" % ast.dump (node)) self.assertTrue (hasattr(node, 'col_offset'), "%s.col_offset not present" % ast.dump (node)) - + class TestBinding (unittest.TestCase): @@ -165,7 +165,7 @@ def testLongOptionCommand (self): self.assertEqual (node.args[0].keywords[0].arg, '--long-option') self.check_attrs (node) - + class TestHelperFunctions (unittest.TestCase): def testName (self): @@ -202,16 +202,16 @@ def testDottedSubscriptComplex (self): class TestWeirdErrors (unittest.TestCase): check_attrs= check_attrs - + def testWithCd (self): code= """with cd() as e: pass""" t= ast.parse (code) self.check_attrs (t.body[0]) - + c= castt.CrazyASTTransformer ({ 'o': o, 'cd': cd}) t= ayrton.parse (code) self.check_attrs (t.body[0]) - + node= c.visit_With (t.body[0]) self.assertEqual (node.items[0].context_expr.func.id, 'cd') From dcd2aa167c6bb3f784653927f8dd1b35f8ee38f3 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 23 Sep 2015 17:52:31 +0200 Subject: [PATCH 12/81] [*] reenabled remote tests by factoring them out from test_ayrton. --- ayrton/tests/test_ayrton.py | 81 ----------------------- ayrton/tests/test_remote.py | 125 ++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 81 deletions(-) create mode 100644 ayrton/tests/test_remote.py diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py index 6b0e4ff..73be7dc 100644 --- a/ayrton/tests/test_ayrton.py +++ b/ayrton/tests/test_ayrton.py @@ -368,87 +368,6 @@ def testBg (self): self.assertEqual (self.r.read (), b'yes!\n0\n') # ayrton.runner.wait_for_pending_children () -class RemoteTests (unittest.TestCase): - def __testRemote (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False) as s: - print (USER) - -value= s[1].readlines () - -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s[0].close () -s[1].close () -s[2].close () - -return value''') - - self.assertEqual ( output, [ ('%s\n' % os.environ['USER']) ] ) - -# SSH_CLIENT='127.0.0.1 55524 22' -# SSH_CONNECTION='127.0.0.1 55524 127.0.0.1 22' -# SSH_TTY=/dev/pts/14 - def __testRemoteEnv (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False) as s: - print (SSH_CLIENT) - -value= s[1].readlines () - -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s[0].close () -s[1].close () -s[2].close () - -return value''') - - expected1= '''127.0.0.1 ''' - expected2= ''' 22\n''' - self.assertEqual (output[0][:len (expected1)], expected1) - self.assertEqual (output[0][-len (expected2):], expected2) - - def __testRemoteVar (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: - foo= 42 - -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s[0].close () -s[1].close () -s[2].close () - -try: - return foo -except Exception as e: - return e''') - - self.assertEqual (output, '''42\n''') - - def __testRemoteReturn (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: - return 42 - -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s[0].close () -s[1].close () -#s[2].close () - -try: - return foo -except Exception as e: - return e''') - - self.assertEqual (output, '''42\n''') - class CommandDetection (unittest.TestCase): def testSimpleCase (self): diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py new file mode 100644 index 0000000..5558f95 --- /dev/null +++ b/ayrton/tests/test_remote.py @@ -0,0 +1,125 @@ +# (c) 2013 Marcos Dione + +# This file is part of ayrton. +# +# ayrton is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ayrton is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ayrton. If not, see . + +import unittest +import sys +import io +import os +import tempfile +import os.path +import time + +from ayrton.expansion import bash +import ayrton +from ayrton.execute import CommandNotFound + +import logging + +logger= logging.getLogger ('ayton.tests.remote') + +# create one of these +ayrton.runner= ayrton.Ayrton () + +class RemoteTests (unittest.TestCase): + def testRemote (self): + """This test only succeeds if you you have password/passphrase-less access + to localhost""" + output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False) as s: + print (USER) + +value= s[1].readlines () + +# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting +# this means even more that the current remote() API sucks +s[0].close () +s[1].close () +#s[2].close () + +return value''') + + self.assertEqual ( output, [ ('%s\n' % os.environ['USER']) ] ) + # give time for nc to recover + time.sleep (0.25) + +# SSH_CLIENT='127.0.0.1 55524 22' +# SSH_CONNECTION='127.0.0.1 55524 127.0.0.1 22' +# SSH_TTY=/dev/pts/14 + def testRemoteEnv (self): + """This test only succeeds if you you have password/passphrase-less access + to localhost""" + output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False) as s: + print (SSH_CLIENT) + +value= s[1].readlines () + +# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting +# this means even more that the current remote() API sucks +s[0].close () +s[1].close () +s[2].close () + +return value''') + + expected1= '''127.0.0.1 ''' + expected2= ''' 22\n''' + self.assertEqual (output[0][:len (expected1)], expected1) + self.assertEqual (output[0][-len (expected2):], expected2) + # give time for nc to recover + time.sleep (0.25) + + def testRemoteVar (self): + """This test only succeeds if you you have password/passphrase-less access + to localhost""" + output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + foo= 56 + +# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting +# this means even more that the current remote() API sucks +s[0].close () +s[1].close () +s[2].close () + +try: + return foo +except Exception as e: + return e''') + + self.assertEqual (output, '''56\n''') + # give time for nc to recover + time.sleep (0.25) + + def testRemoteReturn (self): + """This test only succeeds if you you have password/passphrase-less access + to localhost""" + output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + return 57 + +# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting +# this means even more that the current remote() API sucks +s[0].close () +s[1].close () +#s[2].close () + +try: + return foo +except Exception as e: + return e''') + + self.assertEqual (output, '''57\n''') + # give time for nc to recover + time.sleep (0.25) + From 7fbc4fb041afdd3891811b4183ce9089d0e1fc70 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 13:43:33 +0200 Subject: [PATCH 13/81] [*] use yield from instead of yielding from within a for. --- ayrton/ast_pprinter.py | 190 ++++++++++++++++++++--------------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/ayrton/ast_pprinter.py b/ayrton/ast_pprinter.py index be20f0d..a3fa59e 100644 --- a/ayrton/ast_pprinter.py +++ b/ayrton/ast_pprinter.py @@ -18,7 +18,7 @@ def pprint_body (body, level): for statement in body: yield ' '*level - for i in pprint_inner (statement, level): yield i + yield from pprint_inner (statement, level) yield '\n' def pprint_seq (seq, sep=', '): @@ -26,18 +26,18 @@ def pprint_seq (seq, sep=', '): if type (elem)==str: yield elem else: - for i in pprint_inner (elem): yield i + yield from pprint_inner (elem) if index0: yield ' '*level+'else:\n' - for i in pprint_body (orelse, level+1): yield i + yield from pprint_body (orelse, level+1) def pprint_args (args, defaults): # TODO: anotations @@ -49,7 +49,7 @@ def pprint_args (args, defaults): if index>=d_index: yield '=' - for i in pprint_inner (defaults[index-d_index]): yield i + yield from pprint_inner (defaults[index-d_index]) if index0 and (len (node.keywords)>0 or node.starargs is not None or node.kwargs is not None): yield ', ' - for i in pprint_seq (node.keywords): yield i + yield from pprint_seq (node.keywords) if ((len (node.args)>0 or len (node.keywords)>0) and (node.starargs is not None or node.kwargs is not None)): @@ -163,7 +163,7 @@ def pprint_inner (node, level=0): if node.starargs is not None: yield '*' - for i in pprint_inner (node.starargs): yield i + yield from pprint_inner (node.starargs) if ((len (node.args)>0 or len (node.keywords)>0 or @@ -172,7 +172,7 @@ def pprint_inner (node, level=0): if node.kwargs is not None: yield '**' - for i in pprint_inner (node.kwargs): yield i + yield from pprint_inner (node.kwargs) yield ')' @@ -185,47 +185,47 @@ def pprint_inner (node, level=0): # TODO: more if len (node.bases)>0: yield ' (' - for i in pprint_seq (node.bases): yield i + yield from pprint_seq (node.bases) yield ')' yield ':' - for i in pprint_body (node.body, level+1): yield i + yield from pprint_body (node.body, level+1) elif t==Compare: # Compare(left=Name(id='t', ctx=Load()), ops=[Eq()], comparators=[Name(id='Module', ctx=Load())]) # TODO: do properly - for i in pprint_inner (node.left): yield i + yield from pprint_inner (node.left) for op in node.ops: - for i in pprint_inner (op): yield i + yield from pprint_inner (op) for comparator in node.comparators: - for i in pprint_inner (comparator): yield i + yield from pprint_inner (comparator) elif t==Continue: yield 'continue' elif t==Delete: yield 'delete ' - for i in pprint_seq (node.targets): yield i + yield from pprint_seq (node.targets) elif t==Dict: yield '{ ' for k, v in zip (node.keys, node.values): - for i in pprint_inner (k): yield i + yield from pprint_inner (k) yield '=' - for i in pprint_inner (v): yield i + yield from pprint_inner (v) yield ', ' yield ' }' elif t==DictComp: # DictComp(key=Name(id='v', ctx=Load()), value=Name(id='k', ctx=Load()), generators=[comprehension(target=Tuple(elts=[Name(id='k', ctx=Store()), Name(id='v', ctx=Store())], ctx=Store()), iter=Call(func=Name(id='enumerate', ctx=Load()), args=[Name(id='_b32alphabet', ctx=Load())], keywords=[], starargs=None, kwargs=None), ifs=[])]) yield '{ ' - for i in pprint_inner (node.key): yield i + yield from pprint_inner (node.key) yield ': ' - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) yield ' for ' # TODO: more - for i in pprint_inner (node.generators[0]): yield i + yield from pprint_inner (node.generators[0]) yield ' }' elif t==Div: @@ -238,17 +238,17 @@ def pprint_inner (node, level=0): # ExceptHandler(type=Name(id='KeyError', ctx=Load()), name=None, body=[Pass()]) yield ' '*level+'except ' if node.type is not None: - for i in pprint_inner (node.type): yield i + yield from pprint_inner (node.type) if node.name is not None: yield ' as ' yield node.name yield ':' - for i in pprint_body (node.body, level+1): yield i + yield from pprint_body (node.body, level+1) elif t==Expr: # Expr(value=...) - for i in pprint_inner (node.value, level): yield i + yield from pprint_inner (node.value, level) elif t==FloorDiv: yield '\\\\' @@ -256,34 +256,34 @@ def pprint_inner (node, level=0): elif t==For: # For(target=..., iter=..., body=[...], orelse=[...]) yield 'for ' - for i in pprint_inner (node.target): yield i + yield from pprint_inner (node.target) yield ' in ' - for i in pprint_inner (node.iter): yield i + yield from pprint_inner (node.iter) yield ':\n' - for i in pprint_body (node.body, level+1): yield i - for i in pprint_orelse (node.orelse, level): yield i + yield from pprint_body (node.body, level+1) + yield from pprint_orelse (node.orelse, level) elif t==FunctionDef: # FunctionDef(name='foo', args=arguments(...), body=[ ... ], decorator_list=[], returns=None) # TODO: decorator_list # TODO: returns yield 'def ', node.name, ' (' - for i in pprint_inner (node.args): yield i + yield from pprint_inner (node.args) yield '):\n' - for i in pprint_body (node.body, level+1): yield i + yield from pprint_body (node.body, level+1) elif t==GeneratorExp: # GeneratorExp(elt=Name(id='line', ctx=Load()), generators=[...]) yield '( ' - for i in pprint_inner (node.elt): yield i + yield from pprint_inner (node.elt) yield ' for ' # TODO: more - for i in pprint_inner (node.generators[0]): yield i + yield from pprint_inner (node.generators[0]) yield ' )' elif t==Global: yield 'global ' - for i in pprint_seq (node.names): yield i + yield from pprint_seq (node.names) elif t==Gt: yield '>' @@ -294,43 +294,43 @@ def pprint_inner (node, level=0): elif t==If: # If(test=..., body=[...], orelse=[...] yield 'if ' - for i in pprint_inner (node.test): yield i + yield from pprint_inner (node.test) yield ':\n' - for i in pprint_body (node.body, level+1): yield i + yield from pprint_body (node.body, level+1) if len (node.orelse)>0: # special case for elif if len (node.orelse)==1 and type (node.orelse[0])==If: yield ' '*level+'el' - for i in pprint_inner (node.orelse[0], level): yield i + yield from pprint_inner (node.orelse[0], level) else: - for i in pprint_orelse (node.orelse, level): yield i + yield from pprint_orelse (node.orelse, level) elif t==IfExp: # IfExp(test=..., body=Str(s=''), orelse=Str(s='s')) - for i in pprint_inner (node.body): yield i + yield from pprint_inner (node.body) yield ' if ' - for i in pprint_inner (node.test): yield i + yield from pprint_inner (node.test) yield ' else ' - for i in pprint_inner (node.orelse): yield i + yield from pprint_inner (node.orelse) elif t==Import: # Import(names=[alias(name='ayrton', asname=None)]) yield "import " - for i in pprint_seq (node.names): yield i + yield from pprint_seq (node.names) elif t==ImportFrom: # ImportFrom(module='ayrton.execute', names=[alias(name='Command', asname=None)], level=0) yield "from " yield node.module yield " import " - for i in pprint_seq (node.names): yield i + yield from pprint_seq (node.names) elif t==In: yield ' in ' elif t==Index: - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) elif t==Is: yield ' is ' @@ -344,23 +344,23 @@ def pprint_inner (node, level=0): elif t==Lambda: # Lambda(args=arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=Num(n=0)) yield 'lambda ' - for i in pprint_inner (node.args): yield i + yield from pprint_inner (node.args) yield ': ' - for i in pprint_inner (node.body): yield i + yield from pprint_inner (node.body) elif t==List: yield '[ ' - for i in pprint_seq (node.elts): yield i + yield from pprint_seq (node.elts) yield ' ]' elif t==ListComp: # ListComp(elt=Name(id='i', ctx=Load()), generators=[...]) # [ i for i in self.indexes if i.right is not None ] yield '[ ' - for i in pprint_inner (node.elt): yield i + yield from pprint_inner (node.elt) yield ' for ' # TODO: more - for i in pprint_inner (node.generators[0]): yield i + yield from pprint_inner (node.generators[0]) yield ' ]' elif t==Lt: @@ -374,7 +374,7 @@ def pprint_inner (node, level=0): elif t==Module: # Module(body=[ ... ]) - for i in pprint_body (node.body, 0): yield i + yield from pprint_body (node.body, 0) elif t==Mult: yield '*' @@ -416,13 +416,13 @@ def pprint_inner (node, level=0): # cause=None) yield 'raise ' if node.exc is not None: - for i in pprint_inner (node.exc): yield i + yield from pprint_inner (node.exc) # TODO: cause? elif t==Return: yield 'return ' if node.value is not None: - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) elif t==Set: yield '{ ' @@ -432,24 +432,24 @@ def pprint_inner (node, level=0): elif t==SetComp: # SetComp(elt=Name(id='name', ctx=Load()), generators=[...]) yield '{ ' - for i in pprint_inner (node.elt): yield i + yield from pprint_inner (node.elt) yield ' for ' # TODO: more - for i in pprint_inner (node.generators[0]): yield i + yield from pprint_inner (node.generators[0]) elif t==Slice: # Slice(lower=None, upper=Name(id='left_cb', ctx=Load()), step=None) if node.lower is not None: - for i in pprint_inner (node.lower): yield i + yield from pprint_inner (node.lower) yield ':' if node.upper is not None: - for i in pprint_inner (node.upper): yield i + yield from pprint_inner (node.upper) if node.step is not None: yield ':' - for i in pprint_inner (node.step): yield i + yield from pprint_inner (node.step) elif t==Str: # Str(s='true') @@ -461,9 +461,9 @@ def pprint_inner (node, level=0): elif t==Subscript: # Subscript(value=Attribute(value=Name(id='node', ctx=Load()), attr='orelse', ctx=Load()), # slice=Index(value=Num(n=0)), ctx=Load()) - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) yield '[' - for i in pprint_inner (node.slice): yield i + yield from pprint_inner (node.slice) yield ']' elif t==Try: @@ -472,16 +472,16 @@ def pprint_inner (node, level=0): pprint_body (node.body, level+1) if len (node.handlers)>0: for handler in node.handlers: - for i in pprint_inner (handler, level): yield i + yield from pprint_inner (handler, level) - for i in pprint_orelse (node.orelse, level): yield i + yield from pprint_orelse (node.orelse, level) if len (node.finalbody)>0: yield ' '*level+'finally:\n' - for i in pprint_body (node.finalbody, level+1): yield i + yield from pprint_body (node.finalbody, level+1) elif t==Tuple: yield '( ' - for i in pprint_seq (node.elts): yield i + yield from pprint_seq (node.elts) yield ' )' elif t==UAdd: @@ -491,30 +491,30 @@ def pprint_inner (node, level=0): yield '-' elif t==UnaryOp: - for i in pprint_inner (node.op): yield i - for i in pprint_inner (node.operand): yield i + yield from pprint_inner (node.op) + yield from pprint_inner (node.operand) elif t==While: yield 'while ' - for i in pprint_inner (node.test): yield i + yield from pprint_inner (node.test) yield ':\n' - for i in pprint_body (node.body, level+1): yield i - for i in pprint_orelse (node.orelse, level): yield i + yield from pprint_body (node.body, level+1) + yield from pprint_orelse (node.orelse, level) elif t==With: yield 'with ' - for i in pprint_seq (node.items): yield i + yield from pprint_seq (node.items) yield ':\n' - for i in pprint_body (node.body, level+1): yield i + yield from pprint_body (node.body, level+1) elif t==Yield: # Yield(value=Attribute(value=Name(id='self', ctx=Load()), attr='left', ctx=Load())) yield 'yield ' - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) elif t==YieldFrom: yield 'yield from ' - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) elif t==alias_type: yield node.name @@ -547,7 +547,7 @@ def pprint_inner (node, level=0): # extra keywords is in kwarg - for i in pprint_args (node.args, node.defaults): yield i + yield from pprint_args (node.args, node.defaults) if len (node.args)>0 and (node.vararg is not None or len (node.kwonlyargs)>0 or @@ -556,14 +556,14 @@ def pprint_inner (node, level=0): if node.vararg is not None: yield '*' - for i in pprint_inner (node.vararg): yield i + yield from pprint_inner (node.vararg) if ((len (node.args)>0 or node.vararg is not None) and (len (node.kwonlyargs)>0 or node.kwarg is not None)): yield ', ' - for i in pprint_args (node.kwonlyargs, node.kw_defaults): yield i + yield from pprint_args (node.kwonlyargs, node.kw_defaults) if ((len (node.args)>0 or node.vararg is not None or @@ -572,7 +572,7 @@ def pprint_inner (node, level=0): if node.kwarg is not None: yield '**' - for i in pprint_inner (node.kwarg): yield i + yield from pprint_inner (node.kwarg) elif t==comprehension: # comprehension(target=Name(id='i', ctx=Store()), @@ -581,27 +581,27 @@ def pprint_inner (node, level=0): # ifs=[Compare(left=..., ops=[IsNot()], # comparators=[NameConstant(value=None)])]) # i in self.indexes if i.right is not None - for i in pprint_inner (node.target): yield i + yield from pprint_inner (node.target) yield ' in ' - for i in pprint_inner (node.iter): yield i + yield from pprint_inner (node.iter) if len (node.ifs)>0: # TODO: more yield ' if ' - for i in pprint_inner (node.ifs[0]): yield i + yield from pprint_inner (node.ifs[0]) elif t==keyword_type: # keyword(arg='end', value=Str(s='')) yield node.arg yield '=' - for i in pprint_inner (node.value): yield i + yield from pprint_inner (node.value) elif t==withitem: # withitem(context_expr=..., optional_vars=Name(id='f', ctx=Store())) - for i in pprint_inner (node.context_expr): yield i + yield from pprint_inner (node.context_expr) if node.optional_vars is not None: yield ' as ' - for i in pprint_inner (node.optional_vars): yield i + yield from pprint_inner (node.optional_vars) elif t==str: yield node From 3a6706dbc73a3c2ff2b20d428ed872d8584019ff Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 13:44:17 +0200 Subject: [PATCH 14/81] [*] ast.arguments() might not have kwargs attribute if it was constructed empty. --- ayrton/ast_pprinter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ayrton/ast_pprinter.py b/ayrton/ast_pprinter.py index a3fa59e..c6a323a 100644 --- a/ayrton/ast_pprinter.py +++ b/ayrton/ast_pprinter.py @@ -570,7 +570,8 @@ def pprint_inner (node, level=0): len (node.kwonlyargs)>0) and node.kwarg is not None): yield ', ' - if node.kwarg is not None: + # empty arguments() (from a lambda) do not have this attr (!!!) + if hasattr (node, 'kwarg') and node.kwarg is not None: yield '**' yield from pprint_inner (node.kwarg) From 3425f12f322a76102b0bbd98672e895d349197be Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 13:44:44 +0200 Subject: [PATCH 15/81] [*] yield a, b, c actually yields the tuple (a, b, c). --- ayrton/ast_pprinter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ayrton/ast_pprinter.py b/ayrton/ast_pprinter.py index c6a323a..c24a92f 100644 --- a/ayrton/ast_pprinter.py +++ b/ayrton/ast_pprinter.py @@ -267,7 +267,9 @@ def pprint_inner (node, level=0): # FunctionDef(name='foo', args=arguments(...), body=[ ... ], decorator_list=[], returns=None) # TODO: decorator_list # TODO: returns - yield 'def ', node.name, ' (' + yield 'def ' + yield node.name + yield ' (' yield from pprint_inner (node.args) yield '):\n' yield from pprint_body (node.body, level+1) From 4e07fe2be8f7b17511b50987500e33772098ea3a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 20:07:06 +0200 Subject: [PATCH 16/81] [*] you can tell run_file_or_script() the filename to use when running a script. --- ayrton/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index eaf3ec7..95ab911 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -132,13 +132,14 @@ def run_tree (tree, globals): runner= Ayrton (globals=globals) runner.run_tree (tree, 'unknown_tree') -def run_file_or_script (script=None, file=None, **kwargs): +def run_file_or_script (script=None, file='script_from_command_line', **kwargs): + """Main entry point for bin/ayrton and unittests.""" global runner runner= Ayrton (**kwargs) if script is None: v= runner.run_file (file) else: - v= runner.run_script (script, 'script_from_command_line') + v= runner.run_script (script, file) return v From 1f2380053b4501d6242550902ffa43f13638be95 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 20:15:53 +0200 Subject: [PATCH 17/81] [*] run_tree() returns the what Ayrton.run_tree() returns. --- ayrton/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 95ab911..c673baf 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -128,9 +128,10 @@ def polute (d, more): d.update (more) def run_tree (tree, globals): + """main entry point for remote()""" global runner runner= Ayrton (globals=globals) - runner.run_tree (tree, 'unknown_tree') + return runner.run_tree (tree, 'unknown_tree') def run_file_or_script (script=None, file='script_from_command_line', **kwargs): """Main entry point for bin/ayrton and unittests.""" From f717599911adde899b68b0c6be2e73b3acf77744 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 20:24:50 +0200 Subject: [PATCH 18/81] [*] keep locals so we can do tests with it. --- ayrton/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index c673baf..562361a 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -57,6 +57,7 @@ def __init__ (self, globals=None, **kwargs): self.options= {} self.pending_children= [] + self.locals= {} def run_file (self, file): # it's a pity that parse() does not accept a file as input @@ -74,10 +75,10 @@ def run_tree (self, tree, file_name): return self.run_code (compile (tree, file_name, 'exec')) def run_code (self, code): - locals= {} - exec (code, self.globals, locals) - - return locals['ayrton_return_value'] + exec (code, self.globals, self.locals) + result= self.locals.get ('ayrton_return_value', None) + logger.debug (result) + return result def wait_for_pending_children (self): for i in range (len (self.pending_children)): From 2c92d3ab452b0562cea8335eaa2c4d40a9c6b392 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 20:29:10 +0200 Subject: [PATCH 19/81] [*] also debug wit pprint, to see the code. --- ayrton/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 562361a..bed7136 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -39,6 +39,7 @@ from ayrton.execute import o, Command, Capture, CommandFailed from ayrton.parser.pyparser.pyparse import CompileInfo, PythonParser from ayrton.parser.astcompiler.astbuilder import ast_from_node +from ayrton.ast_pprinter import pprint __version__= '0.5' @@ -72,6 +73,7 @@ def run_script (self, script, file_name): def run_tree (self, tree, file_name): logger.debug (ast.dump (tree)) + logger.debug (pprint (tree)) return self.run_code (compile (tree, file_name, 'exec')) def run_code (self, code): @@ -91,6 +93,7 @@ def polute (d, more): for weed in ('copyright', '__doc__', 'help', '__package__', 'credits', 'license', '__name__'): del d[weed] + # envars as gobal vars, shell like d.update (os.environ) # these functions will be loaded from each module and put in the globals From e25851823171c4fc07cac656f2f075b64b202b7b Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 20:46:18 +0200 Subject: [PATCH 20/81] [+] exit with error 1 when the script borks. --- bin/ayrton | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/ayrton b/bin/ayrton index 601b434..9a229b9 100755 --- a/bin/ayrton +++ b/bin/ayrton @@ -115,3 +115,4 @@ except Exception: for i in range (6): tb= tb.tb_next traceback.print_exception (t, e, tb) + sys.exit (1) From dc8d6b665b9bf979fa9286db8c0e653ddb024c71 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 3 Oct 2015 20:48:15 +0200 Subject: [PATCH 21/81] [+] try to convert the return value to int; other wise, assume al went ok and return 0. --- bin/ayrton | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/ayrton b/bin/ayrton index 9a229b9..014de67 100755 --- a/bin/ayrton +++ b/bin/ayrton @@ -103,7 +103,16 @@ except IndexError: try: # fake argv sys.argv= script_args - ayrton.run_file_or_script (file=file, script=script) + v= ayrton.run_file_or_script (file=file, script=script) + + try: + v= int (v) + execpt ValueError: + # I can only assume it's ok + v= 0 + + sys.exit (v) + except Exception: t, e, tb= sys.exc_info () # skip ayrton's stack From c3c17d332fc3833cc4f6ba26492414fbe1848d02 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sun, 4 Oct 2015 09:59:05 +0200 Subject: [PATCH 22/81] [*] big remote() rewrite. [+] use class RemoteStub to encapsulate the fd's returned by paramiko. [+] use a side channel to pass the resulting locals(), the result and a possible Exception back to the caller. [+] update the globals with the locals from the remote. will need to check if locals is not a better target. [+] raise if there was an exception. --- ayrton/functions.py | 90 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 11 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index 17f6d24..5b4e19b 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -27,6 +27,11 @@ from socket import socket from threading import Thread import sys +import subprocess +import errno + +import logging +logger= logging.getLogger ('ayrton.functions') # NOTE: all this code is excuted in the script's environment @@ -89,7 +94,22 @@ def run (self): self.src.close () self.dst.close () -class remote (object): +class RemoteStub: + def __init__ (self, i, o, e): + self.i= i + self.o= o + self.e= e + + def close (self): + for attr in ('i', 'o', 'e'): + f= getattr (self, attr) + try: + f.close () + except OSError as e: + if e.errno!=errno.EBADF: + raise + +class remote: "Uses the same arguments as paramiko.SSHClient.connect ()" def __init__ (self, ast, hostname, *args, **kwargs): def param (p, d, v=False): @@ -108,6 +128,9 @@ def param (p, d, v=False): param ('_debug', kwargs) self.kwargs= kwargs + # socket/transport where the result is going to come back + self.result_channel= None + def __enter__ (self): # get the globals from the runtime @@ -133,14 +156,38 @@ def __enter__ (self): g= pickle.loads (sys.stdin.buffer.read (%d)) exec (code, g, {})"''' % (len (self.ast), len (global_env)) else: - command= '''python3 -c "import pickle -# names needed for unpickling -from ast import Module, Assign, Name, Store, Call, Load, Expr -import sys -import ayrton -ast= pickle.loads (sys.stdin.buffer.read (%d)) -g= pickle.loads (sys.stdin.buffer.read (%d)) -ayrton.run_tree (ast, g)"''' % (len (self.ast), len (global_env)) + command= '''python3 -c "import pickle # 1 +# names needed for unpickling # 2 +from ast import Module, Assign, Name, Store, Call, Load, Expr # 3 +import sys # 4 +from socket import socket # 5 +import ayrton # 6 + # 7 +import logging # 8 +logger= logging.getLogger ('ayrton.remote') # 9 + # 10 +ast= pickle.loads (sys.stdin.buffer.read (%d)) # 11 +g= pickle.loads (sys.stdin.buffer.read (%d)) # 12 + # 13 +logger.debug (ayrton.ast_pprinter.pprint (ast)) # 14 +logger.debug (g) # 15 + # 16 +runner= ayrton.Ayrton (g) # 17 +e= None # 18 +result= None # 19 + # 20 +try: # 21 + result= runner.run_tree (ast, 'from_remote') # 22 +except Exception as e: # 23 + pass # 24 + # 25 +logger.debug (runner.locals) # 26 + # 27 +client= socket () # 28 +client.connect (('127.0.0.1', 4227)) # 29 +client.sendall (pickle.dumps ( (runner.locals, result, e) )) # 30 +client.close () # 31 +"''' % (len (self.ast), len (global_env)) if not self._debug: self.client= paramiko.SSHClient () @@ -161,14 +208,35 @@ def __enter__ (self): i.write (command.encode ()) i.write (b'\n') + self.result_channel= socket () + # self.result_channel.setsockopt (SO_REUSEADDR, ) + self.result_channel.bind (('', 4227)) + self.result_channel.listen (1) i.write (self.ast) i.write (global_env) # TODO: setup threads with sendfile() to fix i,o,e API - return (i, o, e) + return RemoteStub(i, o, e) def __exit__ (self, *args): - pass + (conn, addr)= self.result_channel.accept () + self.result_channel.close () + + data= b'' + partial= conn.recv (8196) + while len(partial)>0: + data+= partial + partial= conn.recv (8196) + + (locals, result, e)= pickle.loads (data) + logger.debug (locals) + conn.close () + ayrton.runner.globals.update (locals) + # ayrton.runner.locals.update (locals) + logger.debug (ayrton.runner.globals) + logger.debug (ayrton.runner.locals) + if e is not None: + raise e def run (path, *args, **kwargs): c= ayrton.execute.Command (path) From 367fa1c6019b2cf002c3b0f69d35f477bc8c055a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Mon, 5 Oct 2015 09:43:06 +0200 Subject: [PATCH 23/81] [*] update tests. not all works. --- ayrton/tests/test_remote.py | 96 ++++++++++++++----------------------- 1 file changed, 35 insertions(+), 61 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 5558f95..f40aff3 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -31,95 +31,69 @@ logger= logging.getLogger ('ayton.tests.remote') -# create one of these -ayrton.runner= ayrton.Ayrton () class RemoteTests (unittest.TestCase): - def testRemote (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False) as s: - print (USER) -value= s[1].readlines () + def setUp (self): + # create one of these + # self.runner= ayrton.Ayrton () + pass -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s[0].close () -s[1].close () -#s[2].close () - -return value''') - - self.assertEqual ( output, [ ('%s\n' % os.environ['USER']) ] ) + def tearDown (self): # give time for nc to recover time.sleep (0.25) -# SSH_CLIENT='127.0.0.1 55524 22' -# SSH_CONNECTION='127.0.0.1 55524 127.0.0.1 22' -# SSH_TTY=/dev/pts/14 def testRemoteEnv (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False) as s: - print (SSH_CLIENT) - -value= s[1].readlines () + output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + user= USER # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting # this means even more that the current remote() API sucks -s[0].close () -s[1].close () -s[2].close () +s.close () -return value''') +return user''', 'testRemoteEnv') - expected1= '''127.0.0.1 ''' - expected2= ''' 22\n''' - self.assertEqual (output[0][:len (expected1)], expected1) - self.assertEqual (output[0][-len (expected2):], expected2) - # give time for nc to recover - time.sleep (0.25) + self.assertEqual (output, os.environ['USER']) - def testRemoteVar (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" + def testVar (self): output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: foo= 56 # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting # this means even more that the current remote() API sucks -s[0].close () -s[1].close () -s[2].close () +s.close () -try: - return foo -except Exception as e: - return e''') +return foo''', 'testRemoteVar') - self.assertEqual (output, '''56\n''') - # give time for nc to recover - time.sleep (0.25) + self.assertEqual (ayrton.runner.globals['foo'], 56) + # self.assertEqual (ayrton.runner.locals['foo'], 56) - def testRemoteReturn (self): - """This test only succeeds if you you have password/passphrase-less access - to localhost""" + def testReturn (self): output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: return 57 # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting # this means even more that the current remote() API sucks -s[0].close () -s[1].close () -#s[2].close () +s.close () -try: - return foo -except Exception as e: - return e''') +return foo''', 'testRemoteReturn') self.assertEqual (output, '''57\n''') - # give time for nc to recover - time.sleep (0.25) + def testRaisesInternal (self): + ayrton.main ('''raised= False +try: + with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + raise Exception() +except Exception: + raised= True + +# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting +# this means even more that the current remote() API sucks +s.close ()''', 'testRaisesInternal') + + self.assertEqual (ayrton.runner.globals['raised'], True) + + def testRaisesExternal (self): + self.assertRaises (Exception, ayrton.main, '''with remote ('127.0.0.1', allow_agent=False, _debug=True): + raise Exception()''', 'testRaisesExternal') From c64b6a83fed1adfb9d1f338b2d61f5a726999597 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 10:53:50 +0200 Subject: [PATCH 24/81] [*] PEP 3110 bitten me. --- ayrton/functions.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index 5b4e19b..fc0be60 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -173,19 +173,19 @@ def __enter__ (self): logger.debug (g) # 15 # 16 runner= ayrton.Ayrton (g) # 17 -e= None # 18 +caught= None # 18 result= None # 19 # 20 try: # 21 result= runner.run_tree (ast, 'from_remote') # 22 except Exception as e: # 23 - pass # 24 + caught= e # 24 # 25 logger.debug (runner.locals) # 26 # 27 client= socket () # 28 client.connect (('127.0.0.1', 4227)) # 29 -client.sendall (pickle.dumps ( (runner.locals, result, e) )) # 30 +client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 30 client.close () # 31 "''' % (len (self.ast), len (global_env)) From 44ea7b8e13fcce0fbfcb1f7cdcc4499163d7cf4a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 12:00:50 +0200 Subject: [PATCH 25/81] [*] nicer, more informative log format. --- ayrton/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index bed7136..e93df98 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -23,7 +23,7 @@ import ast import logging -log_format= "%(asctime)s %(name)s:%(lineno)-4d %(levelname)-8s %(message)s" +log_format= "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-18s) %(levelname)-8s %(message)s" date_format= "%H:%M:%S" # uncomment one of these for way too much debugging :) From 71cc6243d7d84326496f9826a5d336a3e3e7c5de Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 12:01:28 +0200 Subject: [PATCH 26/81] [*] make logs more self explainatory. --- ayrton/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index e93df98..d31d792 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -72,14 +72,14 @@ def run_script (self, script, file_name): return self.run_tree (tree, file_name) def run_tree (self, tree, file_name): - logger.debug (ast.dump (tree)) - logger.debug (pprint (tree)) + logger.debug ('AST: %s', ast.dump (tree)) + logger.debug ('code: \n%s', pprint (tree)) return self.run_code (compile (tree, file_name, 'exec')) def run_code (self, code): exec (code, self.globals, self.locals) result= self.locals.get ('ayrton_return_value', None) - logger.debug (result) + logger.debug ('ayrton_return_value: %r', result) return result def wait_for_pending_children (self): From 44b31e8b8a45b91ef3bb8ddbb305d3191e87bff0 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 12:16:41 +0200 Subject: [PATCH 27/81] [*] mark start of execution, so logs are more human readable. --- ayrton/__init__.py | 79 +++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index d31d792..a89e97d 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -48,45 +48,6 @@ def parse (script, file_name=''): info= CompileInfo (file_name, 'exec') return ast_from_node (None, parser.parse_source (script, info), info) -class Ayrton (object): - def __init__ (self, globals=None, **kwargs): - if globals is None: - self.globals= {} - else: - self.globals= globals - polute (self.globals, kwargs) - - self.options= {} - self.pending_children= [] - self.locals= {} - - def run_file (self, file): - # it's a pity that parse() does not accept a file as input - # so we could avoid reading the whole file - return self.run_script (open (file).read (), file) - - def run_script (self, script, file_name): - tree= parse (script, file_name) - tree= CrazyASTTransformer (self.globals, file_name).modify (tree) - - return self.run_tree (tree, file_name) - - def run_tree (self, tree, file_name): - logger.debug ('AST: %s', ast.dump (tree)) - logger.debug ('code: \n%s', pprint (tree)) - return self.run_code (compile (tree, file_name, 'exec')) - - def run_code (self, code): - exec (code, self.globals, self.locals) - result= self.locals.get ('ayrton_return_value', None) - logger.debug ('ayrton_return_value: %r', result) - return result - - def wait_for_pending_children (self): - for i in range (len (self.pending_children)): - child= self.pending_children.pop (0) - child.wait () - def polute (d, more): d.update (__builtins__) # weed out some stuff @@ -131,6 +92,45 @@ def polute (d, more): d.update (more) +class Ayrton (object): + def __init__ (self, globals=None, **kwargs): + if globals is None: + self.globals= {} + else: + self.globals= globals + polute (self.globals, kwargs) + self.locals= {} + + self.options= {} + self.pending_children= [] + + def run_file (self, file): + # it's a pity that parse() does not accept a file as input + # so we could avoid reading the whole file + return self.run_script (open (file).read (), file) + + def run_script (self, script, file_name): + tree= parse (script, file_name) + tree= CrazyASTTransformer (self.globals, file_name).modify (tree) + + return self.run_tree (tree, file_name) + + def run_tree (self, tree, file_name): + logger.debug ('AST: %s', ast.dump (tree)) + logger.debug ('code: \n%s', pprint (tree)) + return self.run_code (compile (tree, file_name, 'exec')) + + def run_code (self, code): + exec (code, self.globals, self.locals) + result= self.locals.get ('ayrton_return_value', None) + logger.debug ('ayrton_return_value: %r', result) + return result + + def wait_for_pending_children (self): + for i in range (len (self.pending_children)): + child= self.pending_children.pop (0) + child.wait () + def run_tree (tree, globals): """main entry point for remote()""" global runner @@ -139,6 +139,7 @@ def run_tree (tree, globals): def run_file_or_script (script=None, file='script_from_command_line', **kwargs): """Main entry point for bin/ayrton and unittests.""" + logger.debug ('===========================================================') global runner runner= Ayrton (**kwargs) if script is None: From cf9b7f8473a193976fb0d822f7f6db650b67c200 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 12:25:39 +0200 Subject: [PATCH 28/81] [*] fix logger name. --- ayrton/castt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/castt.py b/ayrton/castt.py index a359adf..a14a783 100644 --- a/ayrton/castt.py +++ b/ayrton/castt.py @@ -26,7 +26,7 @@ from collections import defaultdict import logging -logger= logging.getLogger ('ayton.castt') +logger= logging.getLogger ('ayrton.castt') import ayrton from ayrton.ast_pprinter import pprint From 7c316d21b86dd3dfd9f5614d50914f58a5cd6105 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 12:26:30 +0200 Subject: [PATCH 29/81] [*] make logs more self explainatory. --- ayrton/functions.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index fc0be60..574bb8b 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -169,8 +169,8 @@ def __enter__ (self): ast= pickle.loads (sys.stdin.buffer.read (%d)) # 11 g= pickle.loads (sys.stdin.buffer.read (%d)) # 12 # 13 -logger.debug (ayrton.ast_pprinter.pprint (ast)) # 14 -logger.debug (g) # 15 +logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 14 +logger.debug ('globals received: %%s', g) # 15 # 16 runner= ayrton.Ayrton (g) # 17 caught= None # 18 @@ -179,15 +179,17 @@ def __enter__ (self): try: # 21 result= runner.run_tree (ast, 'from_remote') # 22 except Exception as e: # 23 - caught= e # 24 - # 25 -logger.debug (runner.locals) # 26 - # 27 -client= socket () # 28 -client.connect (('127.0.0.1', 4227)) # 29 -client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 30 -client.close () # 31 + logger.debug ('run raised: %%r', e) # 24 + caught= e # 25 + # 26 +logger.debug ('runner.locals: %%s', runner.locals) # 27 + # 28 +client= socket () # 29 +client.connect (('127.0.0.1', 4227)) # 30 +client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 31 +client.close () # 32 "''' % (len (self.ast), len (global_env)) + logger.debug ('code to execute remote: %s', command) if not self._debug: self.client= paramiko.SSHClient () @@ -229,13 +231,14 @@ def __exit__ (self, *args): partial= conn.recv (8196) (locals, result, e)= pickle.loads (data) - logger.debug (locals) + logger.debug ('locals returned from remote: %s', locals) conn.close () ayrton.runner.globals.update (locals) # ayrton.runner.locals.update (locals) - logger.debug (ayrton.runner.globals) - logger.debug (ayrton.runner.locals) + logger.debug ('globals after remote: %s', ayrton.runner.globals) + logger.debug ('locals after remote: %s', ayrton.runner.locals) if e is not None: + logger.debug ('raised from remote: %r', e) raise e def run (path, *args, **kwargs): From 9ac921cb8c48cc6c6c9d2d555cc8b03b24eaffb8 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 12:26:57 +0200 Subject: [PATCH 30/81] [*] try to update globals with locals before sending to the remote. --- ayrton/functions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index 574bb8b..9dab28e 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -140,10 +140,14 @@ def __enter__ (self): # the imports and hold them in another ayrton.Environment attribute # or we just weed them out here. so far this is the simpler option # but forces the user to reimport what's going to be used in the remote - g= dict ([ (k, v) for (k, v) in ayrton.runner.globals.items () - if type (v)!=types.ModuleType and k not in ('stdin', 'stdout', 'stderr') ]) + g= dict ([ (k, v) for k, v in ayrton.runner.globals.items () + if type (v)!=types.ModuleType and k not in ('stdin', 'stdout', 'stderr') ]) # special treatment for argv g['argv']= ayrton.runner.globals['argv'] + # also locals! + g.update ( dict ([ (k, v) for k, v in ayrton.runner.locals.items () + if k not in ('ayrton_main',) ]) ) + logger.debug ('globals passed to remote: %s', g) global_env= pickle.dumps (g) if self._python_only: From b2990ae437122f1b1913eaaf07f9144cb4a80131 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 15:52:40 +0200 Subject: [PATCH 31/81] [*] locals are back, and now they're properly passed to the remote executor. --- ayrton/__init__.py | 49 ++++++++++++++++++++++++++++++++++++++------- ayrton/functions.py | 48 +++++++++++++++++++++++++++++++------------- 2 files changed, 76 insertions(+), 21 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index a89e97d..a362e52 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -23,7 +23,7 @@ import ast import logging -log_format= "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-18s) %(levelname)-8s %(message)s" +log_format= "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-21s) %(levelname)-8s %(message)s" date_format= "%H:%M:%S" # uncomment one of these for way too much debugging :) @@ -93,13 +93,18 @@ def polute (d, more): d.update (more) class Ayrton (object): - def __init__ (self, globals=None, **kwargs): - if globals is None: + def __init__ (self, g=None, l=None, **kwargs): + logger.debug ('new interpreter: %s, %s', g, l) + if g is None: self.globals= {} else: - self.globals= globals + self.globals= g polute (self.globals, kwargs) - self.locals= {} + + if l is None: + self.locals= {} + else: + self.locals= l self.options= {} self.pending_children= [] @@ -111,6 +116,7 @@ def run_file (self, file): def run_script (self, script, file_name): tree= parse (script, file_name) + # TODO: self.locals? tree= CrazyASTTransformer (self.globals, file_name).modify (tree) return self.run_tree (tree, file_name) @@ -118,10 +124,39 @@ def run_script (self, script, file_name): def run_tree (self, tree, file_name): logger.debug ('AST: %s', ast.dump (tree)) logger.debug ('code: \n%s', pprint (tree)) + return self.run_code (compile (tree, file_name, 'exec')) def run_code (self, code): + ''' + exec(): If only globals is provided, it must be a dictionary, which will + be used for both the global and the local variables. If globals and locals + are given, they are used for the global and local variables, respectively. + If provided, locals can be any mapping object. Remember that at module + level, globals and locals are the same dictionary. If exec gets two + separate objects as globals and locals, the code will be executed as if + it were embedded in a class definition. + + If the globals dictionary does not contain a value for the key __builtins__, + a reference to the dictionary of the built-in module builtins is inserted + under that key. That way you can control what builtins are available to + the executed code by inserting your own __builtins__ dictionary into + globals before passing it to exec(). + + The default locals act as described for function locals() below: + modifications to the default locals dictionary should not be attempted. + Pass an explicit locals dictionary if you need to see effects of the code + on locals after function exec() returns. + + locals(): Update and return a dictionary representing the current local + symbol table. Free variables are returned by locals() when it is called + in function blocks, but not in class blocks. + + The contents of this dictionary should not be modified; changes may not + affect the values of local and free variables used by the interpreter. + ''' exec (code, self.globals, self.locals) + result= self.locals.get ('ayrton_return_value', None) logger.debug ('ayrton_return_value: %r', result) return result @@ -131,10 +166,10 @@ def wait_for_pending_children (self): child= self.pending_children.pop (0) child.wait () -def run_tree (tree, globals): +def run_tree (tree, g, l): """main entry point for remote()""" global runner - runner= Ayrton (globals=globals) + runner= Ayrton (g=g, l=l) return runner.run_tree (tree, 'unknown_tree') def run_file_or_script (script=None, file='script_from_command_line', **kwargs): diff --git a/ayrton/functions.py b/ayrton/functions.py index 9dab28e..455ed75 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -142,13 +142,25 @@ def __enter__ (self): # but forces the user to reimport what's going to be used in the remote g= dict ([ (k, v) for k, v in ayrton.runner.globals.items () if type (v)!=types.ModuleType and k not in ('stdin', 'stdout', 'stderr') ]) + + # get the locals from the runtime + # this is not so easy: for some reason, ayrton.runner.locals is not up to + # date in the middle of the execution (remember *this* code is executed + # via exec() in Ayrton.run_code()) + # another option is go go through the frames + inception_locals= sys._getframe().f_back.f_locals + + l= dict ([ (k, v) for (k, v) in inception_locals.items () + if type (v)!=types.ModuleType and k not in ('ayrton_main', )]) + # special treatment for argv g['argv']= ayrton.runner.globals['argv'] - # also locals! - g.update ( dict ([ (k, v) for k, v in ayrton.runner.locals.items () - if k not in ('ayrton_main',) ]) ) + # l['argv']= ayrton.runner.globals['argv'] + logger.debug ('globals passed to remote: %s', g) global_env= pickle.dumps (g) + logger.debug ('locals passed to remote: %s', l) + local_env= pickle.dumps (l) if self._python_only: command= '''python3 -c "import pickle @@ -158,7 +170,8 @@ def __enter__ (self): ast= pickle.loads (sys.stdin.buffer.read (%d)) code= compile (ast, 'remote', 'exec') g= pickle.loads (sys.stdin.buffer.read (%d)) -exec (code, g, {})"''' % (len (self.ast), len (global_env)) +l= pickle.loads (sys.stdin.buffer.read (%d)) +exec (code, g, l)"''' % (len (self.ast), len (global_env), len (local_env)) else: command= '''python3 -c "import pickle # 1 # names needed for unpickling # 2 @@ -171,12 +184,13 @@ def __enter__ (self): logger= logging.getLogger ('ayrton.remote') # 9 # 10 ast= pickle.loads (sys.stdin.buffer.read (%d)) # 11 -g= pickle.loads (sys.stdin.buffer.read (%d)) # 12 - # 13 -logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 14 -logger.debug ('globals received: %%s', g) # 15 +logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 12 +g= pickle.loads (sys.stdin.buffer.read (%d)) # 13 +logger.debug ('globals received: %%s', g) # 14 +l= pickle.loads (sys.stdin.buffer.read (%d)) # 15 +logger.debug ('locals received: %%s', l) # 15 # 16 -runner= ayrton.Ayrton (g) # 17 +runner= ayrton.Ayrton (g, l) # 17 caught= None # 18 result= None # 19 # 20 @@ -192,7 +206,9 @@ def __enter__ (self): client.connect (('127.0.0.1', 4227)) # 30 client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 31 client.close () # 32 -"''' % (len (self.ast), len (global_env)) +" +''' % (len (self.ast), len (global_env), len (local_env)) + logger.debug ('code to execute remote: %s', command) if not self._debug: @@ -208,12 +224,13 @@ def __enter__ (self): # nc -l -s 127.0.0.1 -p 2233 -vv -e /bin/bash self.client= socket () self.client.connect ((self.hostname, 2233)) - i= open (self.client.fileno (), 'wb') - o= open (self.client.fileno (), 'rb') - e= open (self.client.fileno (), 'rb') + # unbuffered + i= open (self.client.fileno (), 'wb', 0) + o= open (self.client.fileno (), 'rb', 0) + e= open (self.client.fileno (), 'rb', 0) i.write (command.encode ()) - i.write (b'\n') + self.result_channel= socket () # self.result_channel.setsockopt (SO_REUSEADDR, ) self.result_channel.bind (('', 4227)) @@ -221,7 +238,10 @@ def __enter__ (self): i.write (self.ast) i.write (global_env) + i.write (local_env) + # TODO: setup threads with sendfile() to fix i,o,e API + return RemoteStub(i, o, e) def __exit__ (self, *args): From 5e3481cd48aa03d7ab2ea9bcf021c8a9888d3724 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 15:57:58 +0200 Subject: [PATCH 32/81] [*] allow_agent=False is not needed in debugging mode. --- ayrton/tests/test_remote.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index f40aff3..10e491a 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -44,7 +44,7 @@ def tearDown (self): time.sleep (0.25) def testRemoteEnv (self): - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: user= USER # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -56,7 +56,7 @@ def testRemoteEnv (self): self.assertEqual (output, os.environ['USER']) def testVar (self): - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: foo= 56 # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -69,7 +69,7 @@ def testVar (self): # self.assertEqual (ayrton.runner.locals['foo'], 56) def testReturn (self): - output= ayrton.main ('''with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: return 57 # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -83,7 +83,7 @@ def testReturn (self): def testRaisesInternal (self): ayrton.main ('''raised= False try: - with remote ('127.0.0.1', allow_agent=False, _debug=True) as s: + with remote ('127.0.0.1', _debug=True) as s: raise Exception() except Exception: raised= True From 2eb2d1087a550ab578433cecb7139d4569412816 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 15:58:40 +0200 Subject: [PATCH 33/81] [*] correct both testRaises*(). --- ayrton/tests/test_remote.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 10e491a..ec6b2ac 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -81,11 +81,14 @@ def testReturn (self): self.assertEqual (output, '''57\n''') def testRaisesInternal (self): - ayrton.main ('''raised= False + class Foo (Exception): pass + + ayrton.main ('''class Foo (Exception): pass +raised= False try: with remote ('127.0.0.1', _debug=True) as s: - raise Exception() -except Exception: + raise Foo() +except Foo: raised= True # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -95,5 +98,8 @@ def testRaisesInternal (self): self.assertEqual (ayrton.runner.globals['raised'], True) def testRaisesExternal (self): - self.assertRaises (Exception, ayrton.main, '''with remote ('127.0.0.1', allow_agent=False, _debug=True): - raise Exception()''', 'testRaisesExternal') + class Foo (Exception): pass + + self.assertRaises (Foo, ayrton.main, '''class Foo (Exception): pass +with remote ('127.0.0.1', _debug=True): + raise Foo()''', 'testRaisesExternal') From e14a3fc72a5f27c82c3990b96a47702111020fd9 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 15:59:03 +0200 Subject: [PATCH 34/81] [+] unused test to show that getting locals can be difficult. --- ayrton/tests/test_remote.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index ec6b2ac..c885ae0 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -103,3 +103,17 @@ class Foo (Exception): pass self.assertRaises (Foo, ayrton.main, '''class Foo (Exception): pass with remote ('127.0.0.1', _debug=True): raise Foo()''', 'testRaisesExternal') + + def testLocalVarToRemote (self): + ayrton.main ('''testLocalVarToRemote= True +with remote ('127.0.0.1', _debug=True): + testLocalVarToRemote''', 'testLocalVarToRemote') + + def __testLocals (self): + ayrton.main ('''import ayrton +a= True +l= locals()['a'] +# r= ayrton.runner.locals['a'] +# ayrton.main() creates a new Ayrton instance and ****s up everything +r= ayrton.runner.run_script ("""return locals()['a']""", 'inception_locals') +assert (l==r)''', 'testLocals') From 9d52e7f21999d017bb8c21bc9e41a04e1d1a1917 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:26:21 +0200 Subject: [PATCH 35/81] [+] New tests, but it's not yet possible to pass functions and classes to the remote, so they're disabled. --- ayrton/tests/test_remote.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index c885ae0..16d9238 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -109,6 +109,16 @@ def testLocalVarToRemote (self): with remote ('127.0.0.1', _debug=True): testLocalVarToRemote''', 'testLocalVarToRemote') + def __testLocalFunToRemote (self): + ayrton.main ('''def testLocalFunToRemote(): pass +with remote ('127.0.0.1', _debug=True): + testLocalFunToRemote''', 'testLocalFunToRemote') + + def __testLocalClassToRemote (self): + ayrton.main ('''class TestLocalClassToRemote: pass +with remote ('127.0.0.1', _debug=True): + TestLocalClassToRemote''', 'testLocalClassToRemote') + def __testLocals (self): ayrton.main ('''import ayrton a= True From 955ccb4c4ec21c48084140b29baae611c5400a9e Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:27:01 +0200 Subject: [PATCH 36/81] [*] use SystemError instead of custom exception to test raises in the remote. --- ayrton/tests/test_remote.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 16d9238..7f337fc 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -81,14 +81,11 @@ def testReturn (self): self.assertEqual (output, '''57\n''') def testRaisesInternal (self): - class Foo (Exception): pass - - ayrton.main ('''class Foo (Exception): pass -raised= False + ayrton.main ('''raised= False try: with remote ('127.0.0.1', _debug=True) as s: - raise Foo() -except Foo: + raise SystemError() +except SystemError: raised= True # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -98,11 +95,8 @@ class Foo (Exception): pass self.assertEqual (ayrton.runner.globals['raised'], True) def testRaisesExternal (self): - class Foo (Exception): pass - - self.assertRaises (Foo, ayrton.main, '''class Foo (Exception): pass -with remote ('127.0.0.1', _debug=True): - raise Foo()''', 'testRaisesExternal') + self.assertRaises (SystemError, ayrton.main, '''with remote ('127.0.0.1', _debug=True): + raise SystemError()''', 'testRaisesExternal') def testLocalVarToRemote (self): ayrton.main ('''testLocalVarToRemote= True From d17bd7982733a0c636bc68c8bcc242e1535b791f Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:29:54 +0200 Subject: [PATCH 37/81] [*] once more, disable logs by default. --- ayrton/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index a362e52..d5a02b8 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -27,7 +27,7 @@ date_format= "%H:%M:%S" # uncomment one of these for way too much debugging :) -logging.basicConfig(filename='ayrton.%d.log' % os.getpid (), level=logging.DEBUG, format=log_format, datefmt=date_format) +# logging.basicConfig(filename='ayrton.%d.log' % os.getpid (), level=logging.DEBUG, format=log_format, datefmt=date_format) # logging.basicConfig(filename='ayrton.log', level=logging.DEBUG, format=log_format, datefmt=date_format) logger= logging.getLogger ('ayrton') From 82511f2f2098be63a1e31722513687a0bea91886 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:31:04 +0200 Subject: [PATCH 38/81] [*] something new, something old, something blue, something borrowed... --- TODO.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TODO.rst b/TODO.rst index 484ed88..091095e 100644 --- a/TODO.rst +++ b/TODO.rst @@ -14,6 +14,8 @@ Really do: * becareful with if cat () | grep (); error codes must be carried too +* exit code of last Command should be used as return code of functions + * process substitution * https://github.com/amoffat/sh/issues/66 @@ -46,4 +48,3 @@ Think deeply about: * -f vs (-)f vs _f * commands in keywords should also be _out=Capture * which is the sanest default, bash (..., single=True) or otherwise -* foo(-l, --long-option)? From bc52e83150fb0703222740b1aa7328a9ba32a82f Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:40:58 +0200 Subject: [PATCH 39/81] [*] The Zen of Python #7: readability counts. --- ayrton/functions.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index 455ed75..7ed1d5a 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -112,11 +112,15 @@ def close (self): class remote: "Uses the same arguments as paramiko.SSHClient.connect ()" def __init__ (self, ast, hostname, *args, **kwargs): - def param (p, d, v=False): - if p in d: - v= d[p] - del d[p] - setattr (self, p, v) + def param (param, kwargs, default_value=False): + """gets a param from kwargs, or uses a default_value. if found, it's + removed from kwargs""" + if param in kwargs: + value= kwargs[param] + del kwargs[param] + else: + value= default_value + setattr (self, param, value) # actually, it's not a proper ast, it's the pickle of such thing self.ast= ast @@ -147,7 +151,7 @@ def __enter__ (self): # this is not so easy: for some reason, ayrton.runner.locals is not up to # date in the middle of the execution (remember *this* code is executed # via exec() in Ayrton.run_code()) - # another option is go go through the frames + # another option is to go through the frames inception_locals= sys._getframe().f_back.f_locals l= dict ([ (k, v) for (k, v) in inception_locals.items () From 9b8bb6b0bd06b5d4b41d68965b5dcb73a160e8e2 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:55:06 +0200 Subject: [PATCH 40/81] [-] _python_only is gone. --- ayrton/functions.py | 14 +------------- doc/source/reference.rst | 6 +----- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index 7ed1d5a..1fb0cc7 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -128,7 +128,6 @@ def param (param, kwargs, default_value=False): self.args= args self.python_only= False - param ('_python_only', kwargs) param ('_debug', kwargs) self.kwargs= kwargs @@ -166,18 +165,7 @@ def __enter__ (self): logger.debug ('locals passed to remote: %s', l) local_env= pickle.dumps (l) - if self._python_only: - command= '''python3 -c "import pickle -# names needed for unpickling -from ast import Module, Assign, Name, Store, Call, Load, Expr -import sys -ast= pickle.loads (sys.stdin.buffer.read (%d)) -code= compile (ast, 'remote', 'exec') -g= pickle.loads (sys.stdin.buffer.read (%d)) -l= pickle.loads (sys.stdin.buffer.read (%d)) -exec (code, g, l)"''' % (len (self.ast), len (global_env), len (local_env)) - else: - command= '''python3 -c "import pickle # 1 + command= '''python3 -c "import pickle # 1 # names needed for unpickling # 2 from ast import Module, Assign, Name, Store, Call, Load, Expr # 3 import sys # 4 diff --git a/doc/source/reference.rst b/doc/source/reference.rst index 8058e58..e606fad 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -69,7 +69,7 @@ Functions If set, any command that exits with a code which is not 0 will raise a :py:exc:`CommandFailed` exception. -.. py:function:: remote (..., [_python_only=False]) +.. py:function:: remote (..., ) This function is better used as a context manager:: @@ -88,10 +88,6 @@ Functions `ChannelFile `_ (there doesn't seem to be an official doc for this class). - *_python_only* declares that the body is pure Python code, so we don't try - to run it under `ayrton`. This allows remotely executing code without needing - `ayrton` installed in the remote. - For the moment imports are weeded out from the remote environment, so you will need to reimport them. From 192e24af5d3ce3e5071831eba5843620136ab22c Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 16:59:03 +0200 Subject: [PATCH 41/81] [+] testRemoteVarToLocal() proves remote vars come bac to the local process. --- ayrton/tests/test_remote.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 7f337fc..feb8d21 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -113,6 +113,11 @@ def __testLocalClassToRemote (self): with remote ('127.0.0.1', _debug=True): TestLocalClassToRemote''', 'testLocalClassToRemote') + def testRemoteVarToLocal (self): + ayrton.main ('''with remote ('127.0.0.1', _debug=True): + testRemoteVarToLocal= True +testRemoteVarToLocal''', 'testRemoteVarToLocal') + def __testLocals (self): ayrton.main ('''import ayrton a= True From 0828254b04664d8c872c0b64d5a4e2d507e766d5 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 17:03:19 +0200 Subject: [PATCH 42/81] [*] if the execution returns a boolean, convert to shellspeak values (0 for True/OK, 1 for False/NOK). --- bin/ayrton | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/ayrton b/bin/ayrton index 014de67..f18fee6 100755 --- a/bin/ayrton +++ b/bin/ayrton @@ -105,11 +105,17 @@ try: sys.argv= script_args v= ayrton.run_file_or_script (file=file, script=script) - try: - v= int (v) - execpt ValueError: - # I can only assume it's ok - v= 0 + if type(v)==bool: + if v: + v= 0 # success in shellspeak + else: + v= 1 # 'some' error + else: + try: + v= int (v) + execpt ValueError: + # I can only assume it's ok + v= 0 sys.exit (v) From 7342f58dad127950467971b979d1c0c7c49d3d67 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 20:23:08 +0200 Subject: [PATCH 43/81] [*] missing newline in ExceptHandler. --- ayrton/ast_pprinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/ast_pprinter.py b/ayrton/ast_pprinter.py index c24a92f..10cd0d3 100644 --- a/ayrton/ast_pprinter.py +++ b/ayrton/ast_pprinter.py @@ -243,7 +243,7 @@ def pprint_inner (node, level=0): yield ' as ' yield node.name - yield ':' + yield ':\n' yield from pprint_body (node.body, level+1) elif t==Expr: From ebd73274078a2fc72c1a72a95ff27d8eb96cb1ed Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 20:23:35 +0200 Subject: [PATCH 44/81] [*] try bodies were not printed. --- ayrton/ast_pprinter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/ast_pprinter.py b/ayrton/ast_pprinter.py index 10cd0d3..183b02b 100644 --- a/ayrton/ast_pprinter.py +++ b/ayrton/ast_pprinter.py @@ -471,7 +471,7 @@ def pprint_inner (node, level=0): elif t==Try: # Try(body=[...], handlers=[...], orelse=[], finalbody=[]) yield 'try:\n' - pprint_body (node.body, level+1) + yield from pprint_body (node.body, level+1) if len (node.handlers)>0: for handler in node.handlers: yield from pprint_inner (handler, level) From c683d32ade588767b229c00e5d386b7db1949529 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 20:26:21 +0200 Subject: [PATCH 45/81] [-] remove spurious 'as s' from _debug remote()s. --- ayrton/tests/test_remote.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index feb8d21..cc47e6c 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -34,17 +34,20 @@ class RemoteTests (unittest.TestCase): + def setUp (self): # create one of these # self.runner= ayrton.Ayrton () pass + def tearDown (self): # give time for nc to recover time.sleep (0.25) + def testRemoteEnv (self): - output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: + output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): user= USER # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -55,8 +58,9 @@ def testRemoteEnv (self): self.assertEqual (output, os.environ['USER']) + def testVar (self): - output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: + output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): foo= 56 # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting @@ -80,6 +84,7 @@ def testReturn (self): self.assertEqual (output, '''57\n''') + def testRaisesInternal (self): ayrton.main ('''raised= False try: @@ -98,21 +103,25 @@ def testRaisesExternal (self): self.assertRaises (SystemError, ayrton.main, '''with remote ('127.0.0.1', _debug=True): raise SystemError()''', 'testRaisesExternal') + def testLocalVarToRemote (self): ayrton.main ('''testLocalVarToRemote= True with remote ('127.0.0.1', _debug=True): testLocalVarToRemote''', 'testLocalVarToRemote') + def __testLocalFunToRemote (self): ayrton.main ('''def testLocalFunToRemote(): pass with remote ('127.0.0.1', _debug=True): testLocalFunToRemote''', 'testLocalFunToRemote') + def __testLocalClassToRemote (self): ayrton.main ('''class TestLocalClassToRemote: pass with remote ('127.0.0.1', _debug=True): TestLocalClassToRemote''', 'testLocalClassToRemote') + def testRemoteVarToLocal (self): ayrton.main ('''with remote ('127.0.0.1', _debug=True): testRemoteVarToLocal= True From b21d42b6c33a8ad79b44141f3b175300aee35728 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 20:28:01 +0200 Subject: [PATCH 46/81] [-] remove useless s.close() in _debug remote()s. --- ayrton/tests/test_remote.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index cc47e6c..7af423c 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -50,10 +50,6 @@ def testRemoteEnv (self): output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): user= USER -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s.close () - return user''', 'testRemoteEnv') self.assertEqual (output, os.environ['USER']) @@ -63,10 +59,6 @@ def testVar (self): output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): foo= 56 -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s.close () - return foo''', 'testRemoteVar') self.assertEqual (ayrton.runner.globals['foo'], 56) @@ -76,9 +68,6 @@ def testReturn (self): output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: return 57 -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s.close () return foo''', 'testRemoteReturn') @@ -93,9 +82,6 @@ def testRaisesInternal (self): except SystemError: raised= True -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s.close ()''', 'testRaisesInternal') self.assertEqual (ayrton.runner.globals['raised'], True) From 4f4bb285126c857d73f4944e01bad42c795fc10a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 6 Oct 2015 20:30:53 +0200 Subject: [PATCH 47/81] [#] disable testReturn(). most probably this will not be proper syntax anyways. --- ayrton/tests/test_remote.py | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 7af423c..a9371e4 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -64,10 +64,10 @@ def testVar (self): self.assertEqual (ayrton.runner.globals['foo'], 56) # self.assertEqual (ayrton.runner.locals['foo'], 56) - def testReturn (self): - output= ayrton.main ('''with remote ('127.0.0.1', _debug=True) as s: - return 57 + def __testReturn (self): + output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): + return 57 return foo''', 'testRemoteReturn') @@ -75,18 +75,22 @@ def testReturn (self): def testRaisesInternal (self): - ayrton.main ('''raised= False + result= ayrton.main ('''raised= False try: with remote ('127.0.0.1', _debug=True) as s: raise SystemError() except SystemError: raised= True +return raised''', 'testRaisesInternal') + + # self.assertTrue (ayrton.runner.globals['raised']) + self.assertTrue (result) - self.assertEqual (ayrton.runner.globals['raised'], True) def testRaisesExternal (self): - self.assertRaises (SystemError, ayrton.main, '''with remote ('127.0.0.1', _debug=True): + self.assertRaises (SystemError, ayrton.main, + '''with remote ('127.0.0.1', _debug=True): raise SystemError()''', 'testRaisesExternal') @@ -113,6 +117,34 @@ def testRemoteVarToLocal (self): testRemoteVarToLocal= True testRemoteVarToLocal''', 'testRemoteVarToLocal') + + def testLocalVarToRemoteToLocal (self): + result= ayrton.main ('''testLocalVarToRemoteToLocal= False +with remote ('127.0.0.1', _debug=True): + testLocalVarToRemoteToLocal= True + +return testLocalVarToRemoteToLocal''', 'testLocalVarToRemoteToLocal') + + self.assertTrue (ayrton.runner.globals['testLocalVarToRemoteToLocal']) + self.assertTrue (result) + + + def __testLocalVarToRealRemoteToLocal (self): + """This test only succeeds if you you have password/passphrase-less access + to localhost""" + ayrton.main ('''testLocalVarToRealRemote= False +with remote ('127.0.0.1', allow_agent=False) as s: + testLocalVarToRealRemote= True + +# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting +# this means even more that the current remote() API sucks +s.close () + +return testLocalVarToRealRemoteToLocal''', 'testLocalVarToRealRemoteToLocal') + + self.assertTrue (ayrton.runner.globals['testLocalVarToRealRemoteToLocal']) + + def __testLocals (self): ayrton.main ('''import ayrton a= True From 46a6af09874061bdaf5541020d462b5f068aab31 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 10 Oct 2015 19:30:05 +0200 Subject: [PATCH 48/81] [+] debug2 and debug3 logging levels. --- ayrton/__init__.py | 2 ++ ayrton/utils.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 ayrton/utils.py diff --git a/ayrton/__init__.py b/ayrton/__init__.py index d5a02b8..82de247 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -22,6 +22,8 @@ import importlib import ast import logging +# patch logging so we have debug2 and debug3 +import ayrton.utils log_format= "%(asctime)s %(name)16s:%(lineno)-4d (%(funcName)-21s) %(levelname)-8s %(message)s" date_format= "%H:%M:%S" diff --git a/ayrton/utils.py b/ayrton/utils.py new file mode 100644 index 0000000..fd8c940 --- /dev/null +++ b/ayrton/utils.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +# (c) 2015 Marcos Dione + +# This file is part of ayrton. +# +# ayrton is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ayrton is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ayrton. If not, see . + +import logging + +# TODO: there's probably a better way to do this + +def debug2 (self, msg, *args, **kwargs): + if self.manager.disable>=logging.DEBUG2: + return + + if logging.DEBUG2>=self.getEffectiveLevel (): + self._log (logging.DEBUG2, msg, args, **kwargs) + +def debug3 (self, msg, *args, **kwargs): + if self.manager.disable>=logging.DEBUG3: + return + + if logging.DEBUG3>=self.getEffectiveLevel (): + self._log (logging.DEBUG3, msg, args, **kwargs) + +def patch_logging (): + # based on https://mail.python.org/pipermail/tutor/2007-August/056243.html + logging.DEBUG2= 9 + logging.DEBUG3= 8 + + logging.addLevelName (logging.DEBUG2, 'DEBUG2') + logging.addLevelName (logging.DEBUG3, 'DEBUG3') + + logging.Logger.debug2= debug2 + logging.Logger.debug3= debug3 + +patch_logging () From 5267c728af5620ff1aa9560ca027e2894826c152 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Thu, 15 Oct 2015 17:17:45 +0200 Subject: [PATCH 49/81] [+] lots of debugging. --- ayrton/__init__.py | 15 ++++++++++++++- ayrton/functions.py | 9 +++++++-- ayrton/tests/test_remote.py | 9 +++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 82de247..c2a9a87 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -22,6 +22,8 @@ import importlib import ast import logging +import dis + # patch logging so we have debug2 and debug3 import ayrton.utils @@ -96,7 +98,9 @@ def polute (d, more): class Ayrton (object): def __init__ (self, g=None, l=None, **kwargs): - logger.debug ('new interpreter: %s, %s', g, l) + logger.debug ('new interpreter') + logger.debug2 ('globals: %s', g) + logger.debug ('locals: %s', l) if g is None: self.globals= {} else: @@ -157,8 +161,17 @@ def run_code (self, code): The contents of this dictionary should not be modified; changes may not affect the values of local and free variables used by the interpreter. ''' + logger.debug ('code dissasembled:') + if logger.parent.level==logging.DEBUG: + handler= logger.parent.handlers[0] + handler.acquire () + dis.dis (code, file=handler.stream) + handler.release () + exec (code, self.globals, self.locals) + logger.debug ('locals at script exit: %s', self.locals) + logger.debug2 ('locals: %d', id (self.locals)) result= self.locals.get ('ayrton_return_value', None) logger.debug ('ayrton_return_value: %r', result) return result diff --git a/ayrton/functions.py b/ayrton/functions.py index 1fb0cc7..c8f93bf 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -228,6 +228,7 @@ def __enter__ (self): self.result_channel.bind (('', 4227)) self.result_channel.listen (1) + logger.debug ('sending ast, globals, locals') i.write (self.ast) i.write (global_env) i.write (local_env) @@ -247,12 +248,16 @@ def __exit__ (self, *args): partial= conn.recv (8196) (locals, result, e)= pickle.loads (data) + logger.debug ('result from remote: %r', result) logger.debug ('locals returned from remote: %s', locals) conn.close () ayrton.runner.globals.update (locals) # ayrton.runner.locals.update (locals) - logger.debug ('globals after remote: %s', ayrton.runner.globals) - logger.debug ('locals after remote: %s', ayrton.runner.locals) + logger.debug ('caller name: %s', sys._getframe().f_back.f_code.co_name) + + logger.debug2 ('globals after remote: %s', ayrton.runner.globals) + logger.debug ('locals after remote: %s', inception_locals) + logger.debug2 ('locals: %d', id (ayrton.runner.locals)) if e is not None: logger.debug ('raised from remote: %r', e) raise e diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index a9371e4..2277eaf 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -123,6 +123,15 @@ def testLocalVarToRemoteToLocal (self): with remote ('127.0.0.1', _debug=True): testLocalVarToRemoteToLocal= True +import logging +import sys + +logger= logging.getLogger ('ayrton.tests.testLocalVarToRemoteToLocal') +logger.debug ('my name: %s', sys._getframe().f_code.co_name) +logger.debug ('my locals: %s', sys._getframe().f_locals) + +assert sys._getframe().f_locals['testLocalVarToRemoteToLocal'] + return testLocalVarToRemoteToLocal''', 'testLocalVarToRemoteToLocal') self.assertTrue (ayrton.runner.globals['testLocalVarToRemoteToLocal']) From 168cb381764b961f72375697a14c37c1cfba0297 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Thu, 15 Oct 2015 17:19:08 +0200 Subject: [PATCH 50/81] [+] fix UTs a little. --- ayrton/tests/test_remote.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 2277eaf..659eccd 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -84,7 +84,6 @@ def testRaisesInternal (self): return raised''', 'testRaisesInternal') - # self.assertTrue (ayrton.runner.globals['raised']) self.assertTrue (result) @@ -97,7 +96,7 @@ def testRaisesExternal (self): def testLocalVarToRemote (self): ayrton.main ('''testLocalVarToRemote= True with remote ('127.0.0.1', _debug=True): - testLocalVarToRemote''', 'testLocalVarToRemote') + assert (testLocalVarToRemote)''', 'testLocalVarToRemote') def __testLocalFunToRemote (self): @@ -113,10 +112,12 @@ def __testLocalClassToRemote (self): def testRemoteVarToLocal (self): - ayrton.main ('''with remote ('127.0.0.1', _debug=True): + result= ayrton.main ('''with remote ('127.0.0.1', _debug=True): testRemoteVarToLocal= True -testRemoteVarToLocal''', 'testRemoteVarToLocal') +return testRemoteVarToLocal''', 'testRemoteVarToLocal') + + self.assertTrue (result) def testLocalVarToRemoteToLocal (self): result= ayrton.main ('''testLocalVarToRemoteToLocal= False @@ -134,7 +135,6 @@ def testLocalVarToRemoteToLocal (self): return testLocalVarToRemoteToLocal''', 'testLocalVarToRemoteToLocal') - self.assertTrue (ayrton.runner.globals['testLocalVarToRemoteToLocal']) self.assertTrue (result) From e550b212a2698d468a532df4e28637e14351234a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Fri, 16 Oct 2015 18:07:16 +0200 Subject: [PATCH 51/81] [+] requirements.txt file. --- ayrton/tests/test_ayrton.py | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 ayrton/tests/test_ayrton.py create mode 100755 requirements.txt diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py old mode 100644 new mode 100755 index 73be7dc..902686e --- a/ayrton/tests/test_ayrton.py +++ b/ayrton/tests/test_ayrton.py @@ -47,7 +47,7 @@ def test_glob1_single (self): self.assertEqual (bash ('*.py', single=True), 'setup.py') def test_glob2 (self): - self.assertEqual (sorted (bash ([ '*.py', '*.txt' ])), [ 'LICENSE.txt', 'setup.py', ]) + self.assertEqual (sorted (bash ([ '*.py', '*.txt' ])), [ 'LICENSE.txt', 'requirements.txt', 'setup.py', ]) def test_glob_brace1 (self): self.assertEqual (sorted (bash ('s{a,*.py}')), [ 'sa', 'setup.py' ]) diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..3a7c5ad --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +paramiko>=1.15 # first one with py3 support From ba0e5a76f9badef6876dea5ce70805e74a1d2421 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Fri, 16 Oct 2015 20:22:23 +0200 Subject: [PATCH 52/81] [*] update locals from the remote. --- ayrton/functions.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index c8f93bf..d95d59a 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -29,6 +29,7 @@ import sys import subprocess import errno +import ctypes import logging logger= logging.getLogger ('ayrton.functions') @@ -247,16 +248,22 @@ def __exit__ (self, *args): data+= partial partial= conn.recv (8196) - (locals, result, e)= pickle.loads (data) + (l, result, e)= pickle.loads (data) logger.debug ('result from remote: %r', result) - logger.debug ('locals returned from remote: %s', locals) + logger.debug ('locals returned from remote: %s', l) conn.close () - ayrton.runner.globals.update (locals) - # ayrton.runner.locals.update (locals) - logger.debug ('caller name: %s', sys._getframe().f_back.f_code.co_name) + + # update locals + callers_frame= sys._getframe().f_back + logger.debug ('caller name: %s', callers_frame.f_code.co_name) + callers_frame.f_locals.update (l) + # see https://mail.python.org/pipermail/python-dev/2005-January/051018.html + ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(callers_frame), 0) + + # TODO: (and globals?) logger.debug2 ('globals after remote: %s', ayrton.runner.globals) - logger.debug ('locals after remote: %s', inception_locals) + logger.debug ('locals after remote: %s', callers_frame.f_locals) logger.debug2 ('locals: %d', id (ayrton.runner.locals)) if e is not None: logger.debug ('raised from remote: %r', e) From 05ef9a27d1143f7c94a96f88ce2e6b54cbe3193b Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 17 Oct 2015 00:16:58 +0200 Subject: [PATCH 53/81] [*] fix broken test. --- ayrton/tests/test_remote.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 659eccd..8aec862 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -43,8 +43,7 @@ def setUp (self): def tearDown (self): # give time for nc to recover - time.sleep (0.25) - + time.sleep (1) def testRemoteEnv (self): output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): @@ -76,8 +75,9 @@ def __testReturn (self): def testRaisesInternal (self): result= ayrton.main ('''raised= False + try: - with remote ('127.0.0.1', _debug=True) as s: + with remote ('127.0.0.1', _debug=True): raise SystemError() except SystemError: raised= True @@ -95,18 +95,21 @@ def testRaisesExternal (self): def testLocalVarToRemote (self): ayrton.main ('''testLocalVarToRemote= True + with remote ('127.0.0.1', _debug=True): assert (testLocalVarToRemote)''', 'testLocalVarToRemote') def __testLocalFunToRemote (self): ayrton.main ('''def testLocalFunToRemote(): pass + with remote ('127.0.0.1', _debug=True): testLocalFunToRemote''', 'testLocalFunToRemote') def __testLocalClassToRemote (self): ayrton.main ('''class TestLocalClassToRemote: pass + with remote ('127.0.0.1', _debug=True): TestLocalClassToRemote''', 'testLocalClassToRemote') @@ -121,6 +124,7 @@ def testRemoteVarToLocal (self): def testLocalVarToRemoteToLocal (self): result= ayrton.main ('''testLocalVarToRemoteToLocal= False + with remote ('127.0.0.1', _debug=True): testLocalVarToRemoteToLocal= True From a1a24b6bc39f9a1e53d5926d8eb12d683802d473 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 17 Oct 2015 21:43:38 +0200 Subject: [PATCH 54/81] [+] automatize mock remote tester. --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1679101..e7a3703 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,10 @@ all: docs INSTALL_DIR=$(HOME)/local tests: - python3 -m unittest discover -v ayrton + bash -c 'while true; do nc -l -s 127.0.0.1 -p 2233 -vv -e /bin/bash; done' & \ + pid=$!; \ + python3 -m unittest discover -v ayrton; \ + kill $pid docs: PYTHONPATH=${PWD} make -C doc html From 8f3920f1d6ba960d755d12979eb9dc94b8cbaada Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 17 Oct 2015 21:45:13 +0200 Subject: [PATCH 55/81] [*] debug. --- ayrton/functions.py | 74 +++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index d95d59a..fc5d0a0 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -161,46 +161,48 @@ def __enter__ (self): g['argv']= ayrton.runner.globals['argv'] # l['argv']= ayrton.runner.globals['argv'] - logger.debug ('globals passed to remote: %s', g) + logger.debug2 ('globals passed to remote: %s', g) global_env= pickle.dumps (g) logger.debug ('locals passed to remote: %s', l) local_env= pickle.dumps (l) - command= '''python3 -c "import pickle # 1 -# names needed for unpickling # 2 -from ast import Module, Assign, Name, Store, Call, Load, Expr # 3 -import sys # 4 -from socket import socket # 5 -import ayrton # 6 - # 7 -import logging # 8 -logger= logging.getLogger ('ayrton.remote') # 9 - # 10 -ast= pickle.loads (sys.stdin.buffer.read (%d)) # 11 -logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 12 -g= pickle.loads (sys.stdin.buffer.read (%d)) # 13 -logger.debug ('globals received: %%s', g) # 14 -l= pickle.loads (sys.stdin.buffer.read (%d)) # 15 -logger.debug ('locals received: %%s', l) # 15 - # 16 -runner= ayrton.Ayrton (g, l) # 17 -caught= None # 18 -result= None # 19 - # 20 -try: # 21 - result= runner.run_tree (ast, 'from_remote') # 22 -except Exception as e: # 23 - logger.debug ('run raised: %%r', e) # 24 - caught= e # 25 - # 26 -logger.debug ('runner.locals: %%s', runner.locals) # 27 + port= 4227 + + command= '''python3 -c "#! # 1 +import pickle # 2 +# names needed for unpickling # 3 +from ast import Module, Assign, Name, Store, Call, Load, Expr # 4 +import sys # 5 +from socket import socket # 6 +import ayrton # this means that ayrton has to be installed in the remote # 7 + # 8 +import logging # 9 +logger= logging.getLogger ('ayrton.remote') # 10 + # 11 +ast= pickle.loads (sys.stdin.buffer.read (%d)) # 12 +logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 13 +g= pickle.loads (sys.stdin.buffer.read (%d)) # 14 +logger.debug2 ('globals received: %%s', g) # 15 +l= pickle.loads (sys.stdin.buffer.read (%d)) # 16 +logger.debug ('locals received: %%s', l) # 17 + # 18 +runner= ayrton.Ayrton (g, l) # 19 +caught= None # 20 +result= None # 21 + # 22 +try: # 23 + result= runner.run_tree (ast, 'from_remote') # 24 +except Exception as e: # 25 + logger.debug ('run raised: %%r', e) # 26 + caught= e # 27 # 28 -client= socket () # 29 -client.connect (('127.0.0.1', 4227)) # 30 -client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 31 -client.close () # 32 -" -''' % (len (self.ast), len (global_env), len (local_env)) +logger.debug ('runner.locals: %%s', runner.locals) # 29 + # 30 +client= socket () # 31 +client.connect (('127.0.0.1', %d)) # 32 +client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 33 +client.close ()" # 34 +''' % (len (self.ast), len (global_env), len (local_env), port) logger.debug ('code to execute remote: %s', command) @@ -226,7 +228,7 @@ def __enter__ (self): self.result_channel= socket () # self.result_channel.setsockopt (SO_REUSEADDR, ) - self.result_channel.bind (('', 4227)) + self.result_channel.bind (('', port)) self.result_channel.listen (1) logger.debug ('sending ast, globals, locals') From 4340b13e5e5f065e986a075b68c1ad0713efc109 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Sat, 17 Oct 2015 21:45:40 +0200 Subject: [PATCH 56/81] [*] attempt to create ssh backchannel for retrieving remote locals(). --- ayrton/functions.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index fc5d0a0..1426485 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -213,6 +213,10 @@ def __enter__ (self): self.client.set_missing_host_key_policy (paramiko.WarningPolicy ()) self.client.connect (self.hostname, *self.args, **self.kwargs) + # create the backchannel + self.result_channel= self.client.get_transport () + self.result_channel.request_port_forward ('localhost', port) + (i, o, e)= self.client.exec_command (command) else: # to debug, run @@ -241,8 +245,11 @@ def __enter__ (self): return RemoteStub(i, o, e) def __exit__ (self, *args): - (conn, addr)= self.result_channel.accept () - self.result_channel.close () + if self._debug: + (conn, addr)= self.result_channel.accept () + self.result_channel.close () + else: + conn= self.result_channel.accept () data= b'' partial= conn.recv (8196) From 82b8eb77fb9ebfcc461690d37b45b1058f241c27 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 21 Oct 2015 13:10:05 +0200 Subject: [PATCH 57/81] [*] force messages language so some tests do not fail. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e7a3703..22474db 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ INSTALL_DIR=$(HOME)/local tests: bash -c 'while true; do nc -l -s 127.0.0.1 -p 2233 -vv -e /bin/bash; done' & \ pid=$!; \ - python3 -m unittest discover -v ayrton; \ + LC_ALL=C python3 -m unittest discover -v ayrton; \ kill $pid docs: From 53fad0f32cfed126d5de1af2d54040e187247611 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Thu, 22 Oct 2015 23:10:48 +0200 Subject: [PATCH 58/81] [*] no need to be verbose. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 22474db..8299737 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ all: docs INSTALL_DIR=$(HOME)/local tests: - bash -c 'while true; do nc -l -s 127.0.0.1 -p 2233 -vv -e /bin/bash; done' & \ + bash -c 'while true; do nc -l -s 127.0.0.1 -p 2233 -e /bin/bash; done' & \ pid=$!; \ LC_ALL=C python3 -m unittest discover -v ayrton; \ kill $pid From 3ecf5844e96cf1f10eddda10f7a198fa96b9ec54 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Mon, 26 Oct 2015 11:07:48 +0100 Subject: [PATCH 59/81] [*] fixed stopping the nc loop. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 Makefile diff --git a/Makefile b/Makefile old mode 100644 new mode 100755 index 8299737..6d06e45 --- a/Makefile +++ b/Makefile @@ -4,9 +4,9 @@ INSTALL_DIR=$(HOME)/local tests: bash -c 'while true; do nc -l -s 127.0.0.1 -p 2233 -e /bin/bash; done' & \ - pid=$!; \ + pid=$$!; \ LC_ALL=C python3 -m unittest discover -v ayrton; \ - kill $pid + kill $$pid docs: PYTHONPATH=${PWD} make -C doc html From e1e8ddae8fa7b3a69930ad2775fd4c0ba5ed01c0 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 13:07:00 +0100 Subject: [PATCH 60/81] [-] do not wrap the module into a function, so we maintain the module semantics. --- ayrton/castt.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/ayrton/castt.py b/ayrton/castt.py index a14a783..95ba41f 100644 --- a/ayrton/castt.py +++ b/ayrton/castt.py @@ -99,24 +99,6 @@ def __init__ (self, environ, file_name=None): def modify (self, tree): m= self.visit (tree) - - # convert Module(body=[...]) into - # def ayrton_main (): - # [...] - # ayrton_return_value= ayrton_main () - - f= FunctionDef (name='ayrton_main', body=m.body, - args=arguments (args=[], vararg=None, varargannotation=None, - kwonlyargs=[], kwargs=None, kwargannotation=None, - defaults=[], kw_defaults=[]), - decorator_list=[], returns=None) - - c= Call (func=Name (id='ayrton_main', ctx=Load ()), - args=[], keywords=[], starargs=None, kwargs=None) - - t= [Name (id='ayrton_return_value', ctx=Store ())] - - m= Module (body= [ f, Assign (targets=t, value=c) ]) ast.fix_missing_locations (m) return m @@ -136,7 +118,7 @@ def modify (self, tree): # A block is a piece of Python program text that is executed as a unit. # The following are blocks: - # [ ] a module, + # [x] a module, # [x] a function body, and # [x] a class definition. # [ ] A script file is a code block. From 98403e69045184ac2d5a1f3ad40e02d3db2fb950 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 13:08:24 +0100 Subject: [PATCH 61/81] [*] as the code is run in module semantics, fix the locals to be the same object as the globals, so the semantics is kept. --- ayrton/__init__.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index c2a9a87..879635c 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -108,7 +108,29 @@ def __init__ (self, g=None, l=None, **kwargs): polute (self.globals, kwargs) if l is None: - self.locals= {} + # If exec gets two separate objects as globals and locals, + # the code will be executed as if it were embedded in a class definition. + # and this happens: + """ + In [7]: source='''import math + ...: def foo (): + ...: math.floor (1.1, ) + ...: + ...: foo()''' + + In [8]: import ast + In [9]: t= ast.parse (source) + In [10]: c= compile (t, 'foo.py', 'exec') + In [11]: exec (c, {}, {}) + --------------------------------------------------------------------------- + NameError Traceback (most recent call last) + in () + ----> 1 exec (c, {}, {}) + /home/mdione/src/projects/ayrton/foo.py in () + /home/mdione/src/projects/ayrton/foo.py in foo() + NameError: name 'math' is not defined + """ + self.locals= self.globals else: self.locals= l From 61686d1f76aa924f5d3d5449965f300c30000f9e Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 13:13:37 +0100 Subject: [PATCH 62/81] [+] debug2 the dissasembly any first level functions definitions and fix module level debugging. --- ayrton/__init__.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 879635c..7fa3f4e 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -156,6 +156,25 @@ def run_tree (self, tree, file_name): return self.run_code (compile (tree, file_name, 'exec')) def run_code (self, code): + if logger.parent.level<=logging.DEBUG2: + logger.debug ('------------------') + logger.debug ('main (gobal) code:') + handler= logger.parent.handlers[0] + + handler.acquire () + dis.dis (code, file=handler.stream) + handler.release () + + for inst in dis.Bytecode (code): + if inst.opname=='LOAD_CONST': + if type (inst.argval)==type (code): + logger.debug ('------------------') + handler.acquire () + dis.dis (inst.argval, file=handler.stream) + handler.release () + elif type (inst.argval)==str: + logger.debug ("last function is called: %s", inst.argval) + ''' exec(): If only globals is provided, it must be a dictionary, which will be used for both the global and the local variables. If globals and locals @@ -183,17 +202,9 @@ def run_code (self, code): The contents of this dictionary should not be modified; changes may not affect the values of local and free variables used by the interpreter. ''' - logger.debug ('code dissasembled:') - if logger.parent.level==logging.DEBUG: - handler= logger.parent.handlers[0] - handler.acquire () - dis.dis (code, file=handler.stream) - handler.release () - - exec (code, self.globals, self.locals) + logger.debug2 ('globals at script exit: %s', self.globals) logger.debug ('locals at script exit: %s', self.locals) - logger.debug2 ('locals: %d', id (self.locals)) result= self.locals.get ('ayrton_return_value', None) logger.debug ('ayrton_return_value: %r', result) return result From 473db0d1317b6be341c63a3699eed104ab43cd5e Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 13:14:51 +0100 Subject: [PATCH 63/81] [*] catch any runtime Exception so I can still log exit status before reraising it. --- ayrton/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 7fa3f4e..bf900ec 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -202,11 +202,20 @@ def run_code (self, code): The contents of this dictionary should not be modified; changes may not affect the values of local and free variables used by the interpreter. ''' + error= None + try: + exec (code, self.globals, self.locals) + except Exception as e: + error= e logger.debug2 ('globals at script exit: %s', self.globals) logger.debug ('locals at script exit: %s', self.locals) result= self.locals.get ('ayrton_return_value', None) logger.debug ('ayrton_return_value: %r', result) + + if error is not None: + raise error + return result def wait_for_pending_children (self): From fe47e2401babb508fd28c94dfc508c95c22b5287 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 13:25:07 +0100 Subject: [PATCH 64/81] [+] put names to test scripts. --- ayrton/tests/test_ayrton.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py index 73be7dc..9ab080c 100644 --- a/ayrton/tests/test_ayrton.py +++ b/ayrton/tests/test_ayrton.py @@ -447,7 +447,7 @@ def testImportCallFromFunc (self): def foo (): math.floor (1.1) -foo ()''') +foo ()''', 'testImportCallFromFunc') def testImportFrom (self): ayrton.main ('''from math import floor @@ -458,7 +458,7 @@ def testImportFromCallFromFunc (self): def foo (): floor (1.1) -foo ()''') +foo ()''', 'testImportFromCallFromFunc') def testUnknown (self): try: From 8cd3121a68bc25cc0f7eccb660356338a0f6c51c Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 13:25:40 +0100 Subject: [PATCH 65/81] [#] as scripts are modules now, we can't return, so no need to test that. --- ayrton/tests/test_ayrton.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py index 9ab080c..b6c1197 100644 --- a/ayrton/tests/test_ayrton.py +++ b/ayrton/tests/test_ayrton.py @@ -480,7 +480,7 @@ def testSimpleFor (self): class ReturnValues (unittest.TestCase): - def testSimpleReturn (self): + def __testSimpleReturn (self): self.assertEqual (ayrton.main ('''return 50'''), 50) def testException (self): From 5a7f13de2de2e94c26f3b631b573da8bba056846 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Tue, 27 Oct 2015 20:47:11 +0100 Subject: [PATCH 66/81] [*] damn windows and its executable bits. --- Makefile | 0 ayrton/tests/test_ayrton.py | 0 requirements.txt | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 Makefile mode change 100755 => 100644 ayrton/tests/test_ayrton.py mode change 100755 => 100644 requirements.txt diff --git a/Makefile b/Makefile old mode 100755 new mode 100644 diff --git a/ayrton/tests/test_ayrton.py b/ayrton/tests/test_ayrton.py old mode 100755 new mode 100644 diff --git a/requirements.txt b/requirements.txt old mode 100755 new mode 100644 From de0417717b7c75e29992437b2f8294f0838ff880 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 12:57:25 +0100 Subject: [PATCH 67/81] [+] dump_dict() pretty prints dicts. --- ayrton/utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ayrton/utils.py b/ayrton/utils.py index fd8c940..930724e 100644 --- a/ayrton/utils.py +++ b/ayrton/utils.py @@ -47,3 +47,16 @@ def patch_logging (): logging.Logger.debug3= debug3 patch_logging () + +def dump_dict (d, level=0): + strings= [] + + strings.append ("%s%r: {\n" % ( ' '*level, k)) + for k, v in d.items (): + if type (v)!=dict: + strings.append ("%s%r: %r,\n" % ( ' '*(level+1), k, v )) + else: + strings.extend (dump_dict (v, level+1)) + strings.append ("%s},\n" % ( ' '*level, )) + + return ''.join (strings) From 3397b0d95f01282ded0cc5bec1d5a240ba48896d Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 13:03:28 +0100 Subject: [PATCH 68/81] [*] fix in dump_dict(). --- ayrton/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ayrton/utils.py b/ayrton/utils.py index 930724e..dcb9623 100644 --- a/ayrton/utils.py +++ b/ayrton/utils.py @@ -48,15 +48,19 @@ def patch_logging (): patch_logging () -def dump_dict (d, level=0): +def dump_dict (d, level=1): strings= [] - strings.append ("%s%r: {\n" % ( ' '*level, k)) + if level==0: + strings.append ("{\n") for k, v in d.items (): if type (v)!=dict: - strings.append ("%s%r: %r,\n" % ( ' '*(level+1), k, v )) + strings.append ("%s%r: %r,\n" % ( ' '*level, k, v )) else: + strings.append ("%s%r: {\n" % ( ' '*level, k)) strings.extend (dump_dict (v, level+1)) - strings.append ("%s},\n" % ( ' '*level, )) + strings.append ("%s},\n" % ( ' '*level, )) + if level==0: + strings.cappend ("}\n") return ''.join (strings) From 254929b18b3b6c1f9c9f3fafeff5be978562cb3a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 13:14:56 +0100 Subject: [PATCH 69/81] [*] use dump_dict() everywhere. --- ayrton/__init__.py | 4 ++-- ayrton/functions.py | 21 ++++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index bf900ec..cbd3cca 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -208,8 +208,8 @@ def run_code (self, code): except Exception as e: error= e - logger.debug2 ('globals at script exit: %s', self.globals) - logger.debug ('locals at script exit: %s', self.locals) + logger.debug2 ('globals at script exit: %s', ayrton.utils.dump_dict (self.globals)) + logger.debug ('locals at script exit: %s', ayrton.utils.dump_dict (self.locals)) result= self.locals.get ('ayrton_return_value', None) logger.debug ('ayrton_return_value: %r', result) diff --git a/ayrton/functions.py b/ayrton/functions.py index 1426485..fcdd764 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -155,15 +155,15 @@ def __enter__ (self): inception_locals= sys._getframe().f_back.f_locals l= dict ([ (k, v) for (k, v) in inception_locals.items () - if type (v)!=types.ModuleType and k not in ('ayrton_main', )]) + if type (v)!=types.ModuleType and k not in ('stdin', 'stdout', 'stderr') ]) # special treatment for argv g['argv']= ayrton.runner.globals['argv'] # l['argv']= ayrton.runner.globals['argv'] - logger.debug2 ('globals passed to remote: %s', g) + logger.debug2 ('globals passed to remote: %s', ayrton.utils.dump_dict (g)) global_env= pickle.dumps (g) - logger.debug ('locals passed to remote: %s', l) + logger.debug ('locals passed to remote: %s', ayrton.utils.dump_dict (l)) local_env= pickle.dumps (l) port= 4227 @@ -182,9 +182,9 @@ def __enter__ (self): ast= pickle.loads (sys.stdin.buffer.read (%d)) # 12 logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 13 g= pickle.loads (sys.stdin.buffer.read (%d)) # 14 -logger.debug2 ('globals received: %%s', g) # 15 +logger.debug2 ('globals received: %%s', ayrton.utils.dump_dict (g)) # 15 l= pickle.loads (sys.stdin.buffer.read (%d)) # 16 -logger.debug ('locals received: %%s', l) # 17 +logger.debug ('locals received: %%s', ayrton.utils.dump_dict (l)) # 17 # 18 runner= ayrton.Ayrton (g, l) # 19 caught= None # 20 @@ -259,7 +259,7 @@ def __exit__ (self, *args): (l, result, e)= pickle.loads (data) logger.debug ('result from remote: %r', result) - logger.debug ('locals returned from remote: %s', l) + logger.debug ('locals returned from remote: %s', ayrton.utils.dump_dict (l)) conn.close () # update locals @@ -268,12 +268,15 @@ def __exit__ (self, *args): callers_frame.f_locals.update (l) # see https://mail.python.org/pipermail/python-dev/2005-January/051018.html ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(callers_frame), 0) + if self._debug: + # this makes sure that remote locals were properly store in the fast locals + ctypes.pythonapi.PyFrame_FastToLocals(ctypes.py_object(callers_frame)) # TODO: (and globals?) - logger.debug2 ('globals after remote: %s', ayrton.runner.globals) - logger.debug ('locals after remote: %s', callers_frame.f_locals) - logger.debug2 ('locals: %d', id (ayrton.runner.locals)) + logger.debug2 ('globals after remote: %s', ayrton.utils.dump_dict (ayrton.runner.globals)) + logger.debug ('locals after remote: %s', ayrton.utils.dump_dict (callers_frame.f_locals)) + logger.debug ('co_varnames: %s', callers_frame.f_code.co_varnames) if e is not None: logger.debug ('raised from remote: %r', e) raise e From b3a9d44b43486fa83037582bf7fc31b83a2a1d51 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 13:15:36 +0100 Subject: [PATCH 70/81] [*] s/foo/testRemoteVar/.. --- ayrton/tests/test_remote.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 8aec862..76025ec 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -54,11 +54,11 @@ def testRemoteEnv (self): self.assertEqual (output, os.environ['USER']) - def testVar (self): + def testRemoteVar (self): output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): - foo= 56 + testRemoteVar= 56 -return foo''', 'testRemoteVar') +return testRemoteVar''', 'testRemoteVar') self.assertEqual (ayrton.runner.globals['foo'], 56) # self.assertEqual (ayrton.runner.locals['foo'], 56) From b0ba5eaa1719c1cc6321dbab2114defc4726db7a Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 13:30:32 +0100 Subject: [PATCH 71/81] [*] use a new Ayrton instance for each test. --- ayrton/tests/test_remote.py | 63 +++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 76025ec..4421751 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -37,7 +37,7 @@ class RemoteTests (unittest.TestCase): def setUp (self): # create one of these - # self.runner= ayrton.Ayrton () + self.runner= ayrton.Ayrton () pass @@ -49,81 +49,76 @@ def testRemoteEnv (self): output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): user= USER -return user''', 'testRemoteEnv') + def testRemoteEnv (self): + self.runner.run_script ('''with remote ('127.0.0.1', _debug=True): + user= USER''', 'testRemoteEnv.py') - self.assertEqual (output, os.environ['USER']) + self.assertEqual (self.runner.locals['user'], os.environ['USER']) def testRemoteVar (self): - output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): - testRemoteVar= 56 + self.runner.run_script ('''with remote ('127.0.0.1', _debug=True): + testRemoteVar= 56''', 'testRemoteVar.py') -return testRemoteVar''', 'testRemoteVar') - - self.assertEqual (ayrton.runner.globals['foo'], 56) - # self.assertEqual (ayrton.runner.locals['foo'], 56) + self.assertEqual (self.runner.locals['testRemoteVar'], 56) def __testReturn (self): - output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): + self.runner.run_script ('''with remote ('127.0.0.1', _debug=True): return 57 -return foo''', 'testRemoteReturn') +return foo''', 'testRemoteReturn.py') - self.assertEqual (output, '''57\n''') + self.assertEqual (self.runner.locals['foo'], 57) def testRaisesInternal (self): - result= ayrton.main ('''raised= False + self.runner.run_script ('''raised= False try: with remote ('127.0.0.1', _debug=True): raise SystemError() except SystemError: - raised= True + raised= True''', 'testRaisesInternal.py') -return raised''', 'testRaisesInternal') - - self.assertTrue (result) + self.assertTrue (self.runner.locals['raised']) def testRaisesExternal (self): - self.assertRaises (SystemError, ayrton.main, + self.assertRaises (SystemError, self.runner.run_script, '''with remote ('127.0.0.1', _debug=True): - raise SystemError()''', 'testRaisesExternal') + raise SystemError()''', 'testRaisesExternal.py') def testLocalVarToRemote (self): - ayrton.main ('''testLocalVarToRemote= True + self.runner.run_script ('''testLocalVarToRemote= True with remote ('127.0.0.1', _debug=True): assert (testLocalVarToRemote)''', 'testLocalVarToRemote') def __testLocalFunToRemote (self): - ayrton.main ('''def testLocalFunToRemote(): pass + self.runner.run_script ('''def testLocalFunToRemote(): pass with remote ('127.0.0.1', _debug=True): testLocalFunToRemote''', 'testLocalFunToRemote') def __testLocalClassToRemote (self): - ayrton.main ('''class TestLocalClassToRemote: pass + self.runner.run_script ('''class TestLocalClassToRemote: pass with remote ('127.0.0.1', _debug=True): TestLocalClassToRemote''', 'testLocalClassToRemote') def testRemoteVarToLocal (self): - result= ayrton.main ('''with remote ('127.0.0.1', _debug=True): - testRemoteVarToLocal= True - -return testRemoteVarToLocal''', 'testRemoteVarToLocal') + self.runner.run_script ('''with remote ('127.0.0.1', _debug=True): + testRemoteVarToLocal= True''', 'testRemoteVarToLocal.py') - self.assertTrue (result) + self.assertTrue (self.runner.locals['testRemoteVarToLocal']) def testLocalVarToRemoteToLocal (self): - result= ayrton.main ('''testLocalVarToRemoteToLocal= False + self.runner.run_script ('''testLocalVarToRemoteToLocal= False with remote ('127.0.0.1', _debug=True): testLocalVarToRemoteToLocal= True @@ -135,17 +130,15 @@ def testLocalVarToRemoteToLocal (self): logger.debug ('my name: %s', sys._getframe().f_code.co_name) logger.debug ('my locals: %s', sys._getframe().f_locals) -assert sys._getframe().f_locals['testLocalVarToRemoteToLocal'] - -return testLocalVarToRemoteToLocal''', 'testLocalVarToRemoteToLocal') +assert sys._getframe().f_locals['testLocalVarToRemoteToLocal']''', 'testLocalVarToRemoteToLocal') - self.assertTrue (result) + self.assertTrue (self.runner.locals['testLocalVarToRemoteToLocal']) def __testLocalVarToRealRemoteToLocal (self): """This test only succeeds if you you have password/passphrase-less access to localhost""" - ayrton.main ('''testLocalVarToRealRemote= False + self.runner.run_script ('''testLocalVarToRealRemote= False with remote ('127.0.0.1', allow_agent=False) as s: testLocalVarToRealRemote= True @@ -155,11 +148,11 @@ def __testLocalVarToRealRemoteToLocal (self): return testLocalVarToRealRemoteToLocal''', 'testLocalVarToRealRemoteToLocal') - self.assertTrue (ayrton.runner.globals['testLocalVarToRealRemoteToLocal']) + self.assertTrue (self.runner.locals['testLocalVarToRealRemoteToLocal']) def __testLocals (self): - ayrton.main ('''import ayrton + self.runner.run_script ('''import ayrton a= True l= locals()['a'] # r= ayrton.runner.locals['a'] From 3506098e3b136040343e69cde35417710605b0de Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 13:30:54 +0100 Subject: [PATCH 72/81] [*] do now wait so much for nc. --- ayrton/tests/test_remote.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 4421751..0475c93 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -43,11 +43,8 @@ def setUp (self): def tearDown (self): # give time for nc to recover - time.sleep (1) + time.sleep (0.2) - def testRemoteEnv (self): - output= ayrton.main ('''with remote ('127.0.0.1', _debug=True): - user= USER def testRemoteEnv (self): self.runner.run_script ('''with remote ('127.0.0.1', _debug=True): From 2589a7b508750e90857859534417987b271098a4 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 14:00:11 +0100 Subject: [PATCH 73/81] [*] fixed testLocalVarToRealRemoteToLocal. --- ayrton/tests/test_remote.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 0475c93..18a74c1 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -132,18 +132,16 @@ def testLocalVarToRemoteToLocal (self): self.assertTrue (self.runner.locals['testLocalVarToRemoteToLocal']) - def __testLocalVarToRealRemoteToLocal (self): + def testLocalVarToRealRemoteToLocal (self): """This test only succeeds if you you have password/passphrase-less access to localhost""" - self.runner.run_script ('''testLocalVarToRealRemote= False + self.runner.run_script ('''testLocalVarToRealRemoteToLocal= False with remote ('127.0.0.1', allow_agent=False) as s: - testLocalVarToRealRemote= True + testLocalVarToRealRemoteToLocal= True # close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting # this means even more that the current remote() API sucks -s.close () - -return testLocalVarToRealRemoteToLocal''', 'testLocalVarToRealRemoteToLocal') +s.close ()''', 'testLocalVarToRealRemoteToLocal.py') self.assertTrue (self.runner.locals['testLocalVarToRealRemoteToLocal']) From f320b9f348bbedc5ca60dc4b6de2e9f8cfeeffce Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:39:59 +0100 Subject: [PATCH 74/81] [*] remote() now closes the ssh connection. probably will need an API to leave it open. --- ayrton/functions.py | 7 +++++-- ayrton/tests/test_remote.py | 8 ++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ayrton/functions.py b/ayrton/functions.py index fcdd764..20ca7fd 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -242,12 +242,11 @@ def __enter__ (self): # TODO: setup threads with sendfile() to fix i,o,e API - return RemoteStub(i, o, e) + self.connection= RemoteStub(i, o, e) def __exit__ (self, *args): if self._debug: (conn, addr)= self.result_channel.accept () - self.result_channel.close () else: conn= self.result_channel.accept () @@ -277,6 +276,10 @@ def __exit__ (self, *args): logger.debug2 ('globals after remote: %s', ayrton.utils.dump_dict (ayrton.runner.globals)) logger.debug ('locals after remote: %s', ayrton.utils.dump_dict (callers_frame.f_locals)) logger.debug ('co_varnames: %s', callers_frame.f_code.co_varnames) + + self.result_channel.close () + self.connection.close () + if e is not None: logger.debug ('raised from remote: %r', e) raise e diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 18a74c1..82c82d9 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -136,12 +136,8 @@ def testLocalVarToRealRemoteToLocal (self): """This test only succeeds if you you have password/passphrase-less access to localhost""" self.runner.run_script ('''testLocalVarToRealRemoteToLocal= False -with remote ('127.0.0.1', allow_agent=False) as s: - testLocalVarToRealRemoteToLocal= True - -# close the fd's, otherwise the test does not finish because the paramiko.Client() is waiting -# this means even more that the current remote() API sucks -s.close ()''', 'testLocalVarToRealRemoteToLocal.py') +with remote ('127.0.0.1', allow_agent=False): + testLocalVarToRealRemoteToLocal= True''', 'testLocalVarToRealRemoteToLocal.py') self.assertTrue (self.runner.locals['testLocalVarToRealRemoteToLocal']) From b53f6aa7bb0889a6f6b261f78f3fc6166ca18480 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:41:01 +0100 Subject: [PATCH 75/81] [*] cleanup. --- ayrton/__init__.py | 2 +- ayrton/functions.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index cbd3cca..34288dc 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -36,7 +36,7 @@ logger= logging.getLogger ('ayrton') # things that have to be defined before importing ayton.execute :( -# singleton +# singleton needed so the functions can access the runner runner= None from ayrton.castt import CrazyASTTransformer diff --git a/ayrton/functions.py b/ayrton/functions.py index 20ca7fd..579dc48 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -159,7 +159,6 @@ def __enter__ (self): # special treatment for argv g['argv']= ayrton.runner.globals['argv'] - # l['argv']= ayrton.runner.globals['argv'] logger.debug2 ('globals passed to remote: %s', ayrton.utils.dump_dict (g)) global_env= pickle.dumps (g) @@ -219,8 +218,6 @@ def __enter__ (self): (i, o, e)= self.client.exec_command (command) else: - # to debug, run - # nc -l -s 127.0.0.1 -p 2233 -vv -e /bin/bash self.client= socket () self.client.connect ((self.hostname, 2233)) # unbuffered From ebd263a2cc9d9450679971c846400d31c98f826e Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:41:46 +0100 Subject: [PATCH 76/81] [*] messin' w/ debug levels and more debug. --- ayrton/__init__.py | 16 +++---- ayrton/functions.py | 87 ++++++++++++++++++++----------------- ayrton/tests/test_remote.py | 7 +-- 3 files changed, 58 insertions(+), 52 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index 34288dc..b9bbce4 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -99,8 +99,8 @@ def polute (d, more): class Ayrton (object): def __init__ (self, g=None, l=None, **kwargs): logger.debug ('new interpreter') - logger.debug2 ('globals: %s', g) - logger.debug ('locals: %s', l) + logger.debug3 ('globals: %s', ayrton.utils.dump_dict (g)) + logger.debug3 ('locals: %s', ayrton.utils.dump_dict (l)) if g is None: self.globals= {} else: @@ -150,15 +150,15 @@ def run_script (self, script, file_name): return self.run_tree (tree, file_name) def run_tree (self, tree, file_name): - logger.debug ('AST: %s', ast.dump (tree)) - logger.debug ('code: \n%s', pprint (tree)) + logger.debug2 ('AST: %s', ast.dump (tree)) + logger.debug2 ('code: \n%s', pprint (tree)) return self.run_code (compile (tree, file_name, 'exec')) def run_code (self, code): if logger.parent.level<=logging.DEBUG2: - logger.debug ('------------------') - logger.debug ('main (gobal) code:') + logger.debug2 ('------------------') + logger.debug2 ('main (gobal) code:') handler= logger.parent.handlers[0] handler.acquire () @@ -208,8 +208,8 @@ def run_code (self, code): except Exception as e: error= e - logger.debug2 ('globals at script exit: %s', ayrton.utils.dump_dict (self.globals)) - logger.debug ('locals at script exit: %s', ayrton.utils.dump_dict (self.locals)) + logger.debug3 ('globals at script exit: %s', ayrton.utils.dump_dict (self.globals)) + logger.debug3 ('locals at script exit: %s', ayrton.utils.dump_dict (self.locals)) result= self.locals.get ('ayrton_return_value', None) logger.debug ('ayrton_return_value: %r', result) diff --git a/ayrton/functions.py b/ayrton/functions.py index 579dc48..c96d294 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -160,47 +160,51 @@ def __enter__ (self): # special treatment for argv g['argv']= ayrton.runner.globals['argv'] - logger.debug2 ('globals passed to remote: %s', ayrton.utils.dump_dict (g)) + logger.debug3 ('globals passed to remote: %s', ayrton.utils.dump_dict (g)) global_env= pickle.dumps (g) - logger.debug ('locals passed to remote: %s', ayrton.utils.dump_dict (l)) + logger.debug3 ('locals passed to remote: %s', ayrton.utils.dump_dict (l)) local_env= pickle.dumps (l) port= 4227 - command= '''python3 -c "#! # 1 -import pickle # 2 -# names needed for unpickling # 3 -from ast import Module, Assign, Name, Store, Call, Load, Expr # 4 -import sys # 5 -from socket import socket # 6 -import ayrton # this means that ayrton has to be installed in the remote # 7 - # 8 -import logging # 9 -logger= logging.getLogger ('ayrton.remote') # 10 - # 11 -ast= pickle.loads (sys.stdin.buffer.read (%d)) # 12 -logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 13 -g= pickle.loads (sys.stdin.buffer.read (%d)) # 14 -logger.debug2 ('globals received: %%s', ayrton.utils.dump_dict (g)) # 15 -l= pickle.loads (sys.stdin.buffer.read (%d)) # 16 -logger.debug ('locals received: %%s', ayrton.utils.dump_dict (l)) # 17 - # 18 -runner= ayrton.Ayrton (g, l) # 19 -caught= None # 20 -result= None # 21 - # 22 -try: # 23 - result= runner.run_tree (ast, 'from_remote') # 24 -except Exception as e: # 25 - logger.debug ('run raised: %%r', e) # 26 - caught= e # 27 - # 28 -logger.debug ('runner.locals: %%s', runner.locals) # 29 - # 30 -client= socket () # 31 -client.connect (('127.0.0.1', %d)) # 32 -client.sendall (pickle.dumps ( (runner.locals, result, caught) )) # 33 -client.close ()" # 34 + command= '''python3 -c "#! # 1 +import pickle # 2 +# names needed for unpickling # 3 +from ast import Module, Assign, Name, Store, Call, Load, Expr # 4 +import sys # 5 +from socket import socket # 6 +import ayrton # this means that ayrton has to be installed in the remote # 7 + # 8 +import logging # 9 +logger= logging.getLogger ('ayrton.remote') # 10 + # 11 +ast= pickle.loads (sys.stdin.buffer.read (%d)) # 12 +logger.debug ('code to run:\\n%%s', ayrton.ast_pprinter.pprint (ast)) # 13 +g= pickle.loads (sys.stdin.buffer.read (%d)) # 14 +logger.debug2 ('globals received: %%s', ayrton.utils.dump_dict (g)) # 15 +l= pickle.loads (sys.stdin.buffer.read (%d)) # 16 +logger.debug2 ('locals received: %%s', ayrton.utils.dump_dict (l)) # 17 + # 18 +runner= ayrton.Ayrton (g, l) # 19 +caught= None # 20 +result= None # 21 + # 22 +try: # 23 + result= runner.run_tree (ast, 'from_remote') # 24 +except Exception as e: # 25 + logger.debug ('run raised: %%r', e) # 26 + caught= e # 27 + # 28 +logger.debug2 ('runner.locals: %%s', ayrton.utils.dump_dict (runner.locals)) # 29 + # 30 +client= socket () # 31 +client.connect (('127.0.0.1', %d)) # 32 +logger.debug ('about to send exit status') # 33 +data = pickle.dumps ( (runner.locals, result, caught) ) # 34 +logger.debug ('sending %%d bytes', len (data)) # 35 +client.sendall (data) # 36 +logger.debug ('exit status sent') # 37 +client.close ()" # 38 ''' % (len (self.ast), len (global_env), len (local_env), port) logger.debug ('code to execute remote: %s', command) @@ -253,14 +257,15 @@ def __exit__ (self, *args): data+= partial partial= conn.recv (8196) + logger.debug ('recieved %d bytes', len (data)) (l, result, e)= pickle.loads (data) logger.debug ('result from remote: %r', result) - logger.debug ('locals returned from remote: %s', ayrton.utils.dump_dict (l)) + logger.debug3 ('locals returned from remote: %s', ayrton.utils.dump_dict (l)) conn.close () # update locals callers_frame= sys._getframe().f_back - logger.debug ('caller name: %s', callers_frame.f_code.co_name) + logger.debug3 ('caller name: %s', callers_frame.f_code.co_name) callers_frame.f_locals.update (l) # see https://mail.python.org/pipermail/python-dev/2005-January/051018.html ctypes.pythonapi.PyFrame_LocalsToFast(ctypes.py_object(callers_frame), 0) @@ -270,9 +275,9 @@ def __exit__ (self, *args): # TODO: (and globals?) - logger.debug2 ('globals after remote: %s', ayrton.utils.dump_dict (ayrton.runner.globals)) - logger.debug ('locals after remote: %s', ayrton.utils.dump_dict (callers_frame.f_locals)) - logger.debug ('co_varnames: %s', callers_frame.f_code.co_varnames) + logger.debug3 ('globals after remote: %s', ayrton.utils.dump_dict (ayrton.runner.globals)) + logger.debug3 ('locals after remote: %s', ayrton.utils.dump_dict (callers_frame.f_locals)) + logger.debug3 ('co_varnames: %s', callers_frame.f_code.co_varnames) self.result_channel.close () self.connection.close () diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 82c82d9..0cdebd5 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -120,14 +120,15 @@ def testLocalVarToRemoteToLocal (self): with remote ('127.0.0.1', _debug=True): testLocalVarToRemoteToLocal= True +import ayrton.utils import logging import sys logger= logging.getLogger ('ayrton.tests.testLocalVarToRemoteToLocal') -logger.debug ('my name: %s', sys._getframe().f_code.co_name) -logger.debug ('my locals: %s', sys._getframe().f_locals) +logger.debug3 ('my name: %s', sys._getframe().f_code.co_name) +logger.debug3 ('my locals: %s', ayrton.utils.dump_dict (sys._getframe().f_locals)) -assert sys._getframe().f_locals['testLocalVarToRemoteToLocal']''', 'testLocalVarToRemoteToLocal') +assert sys._getframe().f_locals['testLocalVarToRemoteToLocal']''', 'testLocalVarToRemoteToLocal.py') self.assertTrue (self.runner.locals['testLocalVarToRemoteToLocal']) From ae6027a5bcbcfc0816182a9a58fcb33b3bff7c8c Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:43:13 +0100 Subject: [PATCH 77/81] [*] dump_dict() can dump Nones. --- ayrton/utils.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/ayrton/utils.py b/ayrton/utils.py index dcb9623..92592f4 100644 --- a/ayrton/utils.py +++ b/ayrton/utils.py @@ -49,18 +49,21 @@ def patch_logging (): patch_logging () def dump_dict (d, level=1): - strings= [] + if d is not None: + strings= [] - if level==0: - strings.append ("{\n") - for k, v in d.items (): - if type (v)!=dict: - strings.append ("%s%r: %r,\n" % ( ' '*level, k, v )) - else: - strings.append ("%s%r: {\n" % ( ' '*level, k)) - strings.extend (dump_dict (v, level+1)) - strings.append ("%s},\n" % ( ' '*level, )) - if level==0: - strings.cappend ("}\n") + if level==0: + strings.append ("{\n") + for k, v in d.items (): + if type (v)!=dict: + strings.append ("%s%r: %r,\n" % ( ' '*level, k, v )) + else: + strings.append ("%s%r: {\n" % ( ' '*level, k)) + strings.extend (dump_dict (v, level+1)) + strings.append ("%s},\n" % ( ' '*level, )) + if level==0: + strings.cappend ("}\n") - return ''.join (strings) + return ''.join (strings) + else: + return 'None' From 92cd4bb101c7a61b37dfe47a1b86b8e59872bb98 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:50:37 +0100 Subject: [PATCH 78/81] [*] misc. --- ayrton/__init__.py | 2 +- ayrton/functions.py | 2 +- ayrton/tests/test_remote.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ayrton/__init__.py b/ayrton/__init__.py index b9bbce4..12832b9 100644 --- a/ayrton/__init__.py +++ b/ayrton/__init__.py @@ -45,7 +45,7 @@ from ayrton.parser.astcompiler.astbuilder import ast_from_node from ayrton.ast_pprinter import pprint -__version__= '0.5' +__version__= '0.6' def parse (script, file_name=''): parser= PythonParser (None) diff --git a/ayrton/functions.py b/ayrton/functions.py index c96d294..67835ad 100644 --- a/ayrton/functions.py +++ b/ayrton/functions.py @@ -223,7 +223,7 @@ def __enter__ (self): (i, o, e)= self.client.exec_command (command) else: self.client= socket () - self.client.connect ((self.hostname, 2233)) + self.client.connect ((self.hostname, 2233)) # nc listening here # unbuffered i= open (self.client.fileno (), 'wb', 0) o= open (self.client.fileno (), 'rb', 0) diff --git a/ayrton/tests/test_remote.py b/ayrton/tests/test_remote.py index 0cdebd5..559577b 100644 --- a/ayrton/tests/test_remote.py +++ b/ayrton/tests/test_remote.py @@ -114,6 +114,7 @@ def testRemoteVarToLocal (self): self.assertTrue (self.runner.locals['testRemoteVarToLocal']) + def testLocalVarToRemoteToLocal (self): self.runner.run_script ('''testLocalVarToRemoteToLocal= False From a139f568c1cb2547f6a458bd503242e1707c59f6 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:54:38 +0100 Subject: [PATCH 79/81] [*] change doc to reflect remote() API change. --- doc/source/reference.rst | 8 -------- 1 file changed, 8 deletions(-) diff --git a/doc/source/reference.rst b/doc/source/reference.rst index e606fad..bf6393e 100644 --- a/doc/source/reference.rst +++ b/doc/source/reference.rst @@ -80,14 +80,6 @@ Functions `SSHClient.connect() `_ method. The body of the construct is executed in the remote machine. - The function returns 3 streams that represent ``stdin``, ``stdout`` and - ``stderr``. These streams have ``write()``, ``read(n)``, ``readline()`` and - ``readlines()`` methods that can be used to interact with the remote. They - only accept or return ``bytes``, not ``strings``. For more information - about them, see ``paramiko``'s - `ChannelFile `_ - (there doesn't seem to be an official doc for this class). - For the moment imports are weeded out from the remote environment, so you will need to reimport them. From c38b7b4338405816f1ce171b5374766549559be8 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 20:56:40 +0100 Subject: [PATCH 80/81] [*] typo. --- bin/ayrton | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ayrton b/bin/ayrton index f18fee6..06a655e 100755 --- a/bin/ayrton +++ b/bin/ayrton @@ -113,7 +113,7 @@ try: else: try: v= int (v) - execpt ValueError: + except ValueError: # I can only assume it's ok v= 0 From b39dcf62798aac8a5100b9b3c4cfb58ad4db2197 Mon Sep 17 00:00:00 2001 From: Marcos Dione Date: Wed, 28 Oct 2015 21:17:10 +0100 Subject: [PATCH 81/81] [*] release! --- ChangeLog.rst | 15 +++++++++++++++ README.md | 15 +++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 45fb021..287797f 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,18 @@ +ayrton (0.6) UNRELEASED; urgency=medium + + * Great improvements in `remote()`'s API and sematics: + * Made sure local varaibles go to and come back from the remote. + * Code block is executes syncronically. + * For the moment the streams are no longer returned. + * _python_only option is gone. + * Most tests actually connect to a listening netcat, only one test uses `ssh`. + * Fixed bugs in the new parser. + * Fixed globals/locals mix up. + * Scripts are no longer wrapped in a function. This means that you can't return values and that module semantics are restored. + * `ayrton` exits with status 1 when the script fails to run (SyntaxError, etc). + + -- Marcos Dione Wed, 28 Oct 2015 20:57:19 +0100 + ayrton (0.5) unstable; urgency=medium * Much better command detection. diff --git a/README.md b/README.md index 1c88d62..d159e6c 100644 --- a/README.md +++ b/README.md @@ -153,20 +153,15 @@ The cherry on top of the cake, or more like the melon of top of the cupcake, is (semi) transparent remote execution. This is achieved with the following construct: a= 42 - with remote ('localhost') as streams: - foo= input () - print (foo) + with remote ('localhost'): # we can also access variables already in the scope # even when we're actually running in another machine print (a) + # we can modify those variables + a= 27 - # streams returns 3 streams: stdin, stdout, stderr - (i, o, e)= streams - # notice that we must include the \n at the end so input() finishes - # and that you must transmit bytes only, no strings - i.write (b'bar\n') - print (o.readlines ()) - + # and those modifications are reflected locally + assert (a, 27) The body of the `with remote(): ...` statement is actually executed in a remote machine after connecting via `ssh`. The `remote()` context manager accepts the