From cd998143e5b710e409c187a9310babab96cbb0b6 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 15:09:32 -0600 Subject: [PATCH 1/7] add some typing data for '_:_ --- python-semantics-literals.k | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python-semantics-literals.k b/python-semantics-literals.k index 64a31bf..6b7e228 100644 --- a/python-semantics-literals.k +++ b/python-semantics-literals.k @@ -65,7 +65,7 @@ module PYTHON-SEMANTICS-LITERALS ... "__value__" |-> map(_ H |-> list(Values)) ... ... - rule bucketInsert(B, H, Key, K, ListItem(Key2 : Value2) Suffix:List, Prefix:List) => test(plbool(Key == Key2), setBucket(B, H, Prefix ListItem(K) Suffix), bucketInsert(B, H, Key, K, Suffix, Prefix ListItem(Key2 : Value2))) + rule bucketInsert(B, H, Key, K, ListItem(Key2:Expression : Value2:Expression) Suffix:List, Prefix:List) => test(plbool(Key == Key2), setBucket(B, H, Prefix ListItem(K) Suffix), bucketInsert(B, H, Key, K, Suffix, Prefix ListItem(Key2 : Value2))) rule bucketInsert(B, H, Key, K, ., Prefix) => . ... ... @@ -97,7 +97,7 @@ module PYTHON-SEMANTICS-LITERALS ... "__value__" |-> map(M) ... ... when notBool(H in keys(M)) - rule bucketLookup(Key, ListItem(Key2 : Value2) Suffix) => test(plbool(Key == Key2), Value2, bucketLookup(Key, Suffix)) + rule bucketLookup(Key, ListItem(Key2:Expression : Value2:Expression) Suffix) => test(plbool(Key == Key2), Value2, bucketLookup(Key, Suffix)) rule bucketLookup(Key, .) => raise ref("KeyError") (Key, .Arguments) rule tableDelete(B, H, Key) => bucketDelete(B, H, Key, Values, .) ... @@ -112,7 +112,7 @@ module PYTHON-SEMANTICS-LITERALS ... "__value__" |-> map(M) ... ... when notBool(H in keys(M)) - rule bucketDelete(B, H, Key, ListItem(Key2 : Value2) Suffix, Prefix) => test(plbool(Key == Key2), setBucket(B, H, Prefix Suffix), bucketDelete(B, H, Key, Suffix, Prefix ListItem(Key2 : Value2))) + rule bucketDelete(B, H, Key, ListItem(Key2:Expression : Value2:Expression) Suffix, Prefix) => test(plbool(Key == Key2), setBucket(B, H, Prefix Suffix), bucketDelete(B, H, Key, Suffix, Prefix ListItem(Key2 : Value2))) rule bucketDelete(_, _, Key, ., _) => raise ref("KeyError") (Key, .Arguments) rule mutable(V:Value,O) => mutable("__value__" |-> V "__revision__" |-> 0, O) ... From bb1f1d6863c690ccb68505484c3c4bbd670ea0ec Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 15:10:14 -0600 Subject: [PATCH 2/7] add some typing data for '_,_ and '*_ --- python-semantics-assignment.k | 2 +- python-semantics-common.k | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python-semantics-assignment.k b/python-semantics-assignment.k index ee555a5..4290571 100644 --- a/python-semantics-assignment.k +++ b/python-semantics-assignment.k @@ -30,7 +30,7 @@ module PYTHON-SEMANTICS-ASSIGNMENT rule (. => iterate(O, .)) ~> bindList(_, O, _) rule (. => T, .Expressions := O) ~> list((ListItem(O) => .) _) ~> bindList((ListItem(T:Target) => .) _, _, _) when getKLabel T =/=KLabel '*_ - rule (. => T, .Expressions := mutable(list(Refify(Idify(L))), ref("list"))) ~> list((L:List => .) L2:List) ~> bindList((ListItem(* T) => .) L3:List, _, (false => true)) when lengthList L2 ==Int lengthList L3 + rule (. => T, .Expressions := mutable(list(Refify(Idify(L))), ref("list"))) ~> list((L:List => .) L2:List) ~> bindList((ListItem(* T:Target) => .) L3:List, _, (false => true)) when lengthList L2 ==Int lengthList L3 rule list(.) ~> bindList(., _, _) => . rule list(ListItem(_) _) ~> bindList(., _, _) => raiseInternal("ValueError", "too many values to unpack") rule list(.) ~> bindList(ListItem(T) _, _, _) => raiseInternal("ValueError", "need more values to unpack") when getKLabel T =/=KLabel '*_ diff --git a/python-semantics-common.k b/python-semantics-common.k index f32590f..e11624d 100644 --- a/python-semantics-common.k +++ b/python-semantics-common.k @@ -113,11 +113,13 @@ module PYTHON-SEMANTICS-COMMON rule Ids2Strings(.) => . [function, anywhere] syntax List ::= "Listify" "(" K ")" [function] - rule Listify((P:Parameter , Ps:Parameters)) => ListItem(P) Listify(Ps) [function, anywhere] + rule Listify((P:DefParameter , Ps:Parameters)) => ListItem(P) Listify(Ps) [function, anywhere] + rule Listify((A:Argument , As:Arguments)) => ListItem(A) Listify(As) [function, anywhere] rule Listify(.Parameters) => . [function, anywhere] syntax Expressions ::= "DeListify" "(" List ")" [function] - rule DeListify(ListItem(P) L) => P , DeListify(L) [function, anywhere] + rule DeListify(ListItem(P:DefParameter) L) => P , DeListify(L) [function, anywhere] + rule DeListify(ListItem(A:Argument) L) => A , DeListify(L) [function, anywhere] rule DeListify(.) => .Expressions [function, anywhere] syntax List ::= "Refify" "(" List ")" [function] @@ -186,7 +188,7 @@ module PYTHON-SEMANTICS-COMMON rule listvalue(obj(_,... "__value__" |-> list(L:List) ...)) => L [function, anywhere] syntax Map ::= "mapvalue" "(" Object ")" [function] - rule mapvalue(obj(_,... "__value__" |-> map(_ _ |-> list(_ (ListItem(K : K2) => .) _)) ...)) (. => K |-> K2) [function, anywhere] + rule mapvalue(obj(_,... "__value__" |-> map(_ _ |-> list(_ (ListItem(K:Expression : K2:Expression) => .) _)) ...)) (. => K |-> K2) [function, anywhere] rule mapvalue(obj(_,... "__value__" |-> map(_ (_ |-> list(.) => .)) ...)) [function, anywhere] rule mapvalue(obj(_,... "__value__" |-> map(.) ...)) => . [function, anywhere] From 749027cd366a4ce4705cef506022b2a01f0b17e7 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 15:10:43 -0600 Subject: [PATCH 3/7] update syntax of annotations to avoid parsing ambiguities --- python-semantics-functions.k | 57 ++++++++++++++++++++---------------- python-syntax.k | 14 +++++---- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/python-semantics-functions.k b/python-semantics-functions.k index f77c7b7..36fc1a2 100644 --- a/python-semantics-functions.k +++ b/python-semantics-functions.k @@ -5,16 +5,17 @@ module PYTHON-SEMANTICS-FUNCTIONS imports PYTHON-VISITOR imports PYTHON-SEMANTICS-BOOLEAN-OPS - syntax K ::= "makeFunction" "(" Int "," List "," K "," Bool ")" - | "makeCodeObject" "(" Int "," List "," K "," Bool ")" - syntax String ::= "paramName" "(" Argument ")" + syntax K ::= makeFunction(Int, List, K, Bool) + | makeCodeObject(Int, List, K, Bool) + syntax String ::= paramName(DefParameter) rule paramName(X:Id) => Id2String(X) - rule paramName(X = _) => Id2String(X) - rule paramName(* X) => Id2String(X) - rule paramName(** X) => Id2String(X) + rule paramName(X:Id : _) => Id2String(X) + rule paramName(P:Parameter = _) => paramName(P) + rule paramName(* P:Parameter) => paramName(P) + rule paramName(** P:Parameter) => paramName(P) - syntax ObjRef ::= "function" "(" Parameters "," Expression "," K ")" [strict(2)] + syntax ObjRef ::= function(Parameters, Expression, K) [strict(2)] rule [evaluate-function-object]: function(Ps:K, O:Object, Return:K) => setref(N, "__doc__", (O . String2Id("co_consts") [0])) ~> @@ -47,12 +48,12 @@ module PYTHON-SEMANTICS-FUNCTIONS ...) [allocation] - context makeFunction(_, (ListItem(X = HOLE) _:List), _, _) - context makeFunction(_, (_:List ListItem(X : HOLE) _:List), _, _) - context makeFunction(_, (ListItem((X = HOLE) : _) _:List), _, _) - context makeFunction(_, (_:List ListItem((X = _) : HOLE) _:List), _, _) - context makeFunction(_, (_:List ListItem((* _) : HOLE) _:List), _, _) - context makeFunction(_, (_:List ListItem((** _) : HOLE) _:List), _, _) + context makeFunction(_, (ListItem(_ = HOLE) _:List), _, _) + context makeFunction(_, (_:List ListItem(_ : HOLE) _:List), _, _) + context makeFunction(_, (ListItem((_ : _) = HOLE) _:List), _, _) + context makeFunction(_, (_:List ListItem((_ : HOLE) = _) _:List), _, _) + context makeFunction(_, (_:List ListItem(* (_ : HOLE)) _:List), _, _) + context makeFunction(_, (_:List ListItem(** (_ : HOLE)) _:List), _, _) context makeFunction(_, _, HOLE, _) rule (. => addParam(X)) ~> makeCodeObject(N, (ListItem(X) L:List => L), _, false) ... @@ -67,10 +68,10 @@ module PYTHON-SEMANTICS-FUNCTIONS ... "co_nlocals" |-> (N2 => N2 +Int 1) "co_kwonlyargcount" |-> (N3 => N3 +Int 1) ... ... - rule makeCodeObject(_, ((ListItem(X = _) => ListItem(X)) _), _, _) + rule makeCodeObject(_, ((ListItem(P:Parameter = _) => ListItem(P)) _), _, _) - rule [makeFunction-defaults]: (. => setref(N, "__defaults__", test(plbool(getref2(ref(N), "__defaults__") is ref("None")), tuple(O), (getref2(ref(N), "__defaults__") + tuple(O))))) ~> makeFunction(N, ((ListItem(X:Id = O:Object) => ListItem(X)) _), _, false) - rule [makeFunction-kwdefaults]: (. => test(plbool(getref2(ref(N), "__kwdefaults__") is ref("None")), setref(N, "__kwdefaults__", {Id2String(X) : O}), (getref2(ref(N), "__kwdefaults__")[Id2String(X)] := O))) ~> makeFunction(N, ((ListItem(X:Id = O:Object) => ListItem(X)) _), _, true) + rule [makeFunction-defaults]: (. => setref(N, "__defaults__", test(plbool(getref2(ref(N), "__defaults__") is ref("None")), tuple(O), (getref2(ref(N), "__defaults__") + tuple(O))))) ~> makeFunction(N, ((ListItem(P:Parameter = O:Object) => ListItem(P)) _), _, false) + rule [makeFunction-kwdefaults]: (. => test(plbool(getref2(ref(N), "__kwdefaults__") is ref("None")), setref(N, "__kwdefaults__", {paramName(P) : O}), (getref2(ref(N), "__kwdefaults__")[paramName(P)] := O))) ~> makeFunction(N, ((ListItem(P:Parameter = O:Object) => ListItem(P)) _), _, true) rule [makeFunction-parameter]: makeFunction(_, ((ListItem(X:Id) => .) _), _, _) rule (. => setattr(N, "co_flags", (getattr(ref(N), "co_flags") |Int 4)) ~> addParam(* X)) ~> makeCodeObject(N, ((ListItem(* X:Id) => ListItem(*)) L), _, false) @@ -79,10 +80,14 @@ module PYTHON-SEMANTICS-FUNCTIONS rule makeFunction(_, ((ListItem(*) => .) L), _, (false => true)) rule makeCodeObject(_, ((ListItem(*) => .) L), _, (false => true)) rule (. => setattr(N, "co_flags", (getattr(ref(N), "co_flags") |Int 8)) ~> addParam(** X)) ~> makeCodeObject(N, ((ListItem(** X:Id) => .) L), _, _) - rule makeFunction(N, ((ListItem(** X) => .) L), _, _) - rule (. => getref2(ref(N), "__annotations__")[paramName(Arg)] := O) ~> makeFunction(N, ((ListItem(Arg:K : O:Object) => ListItem(Arg)) _), _, _) + rule makeFunction(N, ((ListItem(** X:Id) => .) L), _, _) + rule (. => getref2(ref(N), "__annotations__")[Id2String(X)] := O) ~> makeFunction(N, ((ListItem(X:Id : O:Object) => ListItem(X)) _), _, _) + rule (. => getref2(ref(N), "__annotations__")[Id2String(X)] := O) ~> makeFunction(N, ((ListItem(* (X:Id : O:Object)) => ListItem(* X)) _), _, _) + rule (. => getref2(ref(N), "__annotations__")[Id2String(X)] := O) ~> makeFunction(N, ((ListItem(** X:Id : O:Object) => ListItem(** X)) _), _, _) rule (. => getref2(ref(N), "__annotations__")["return"] := O) ~> makeFunction(N, _, (O:Object => .Obj), _) - rule makeCodeObject(N, ((ListItem(Arg : _) => ListItem(Arg)) _), _, _) + rule makeCodeObject(N, ((ListItem(X:Id : _) => ListItem(X)) _), _, _) + rule makeCodeObject(N, ((ListItem(* (X:Id : _)) => ListItem(* X)) _), _, _) + rule makeCodeObject(N, ((ListItem(** X:Id : _) => ListItem(** X)) _), _, _) syntax ObjRef ::= "makeClosure" "(" Expression ")" [strict] | "cellNum" "(" Expression "," Expression "," Expression ")" [strict] @@ -147,12 +152,12 @@ module PYTHON-SEMANTICS-FUNCTIONS rule BOUND(Bound:Set, L scope(_, Defs, _, Nonlocals, Globals2, _, Params, false, _)) => BOUND((Bound LOCAL(Defs, Globals2, Nonlocals, Params)) -Set Globals2, L) [function, anywhere] rule BOUND(Bound:Set, L scope(_, _:Set, _:Set, _:Set, _:Set, _:Set, _:List, true, _)) => BOUND(Bound, L) rule BOUND(Bound, .) => Bound [function, anywhere] - rule PARAMS(ListItem(X:Id) Params) => ListItem(Id2String(X)) PARAMS(Params) [function, anywhere] - rule PARAMS(ListItem(* X:Id) Params) => PARAMS(Params ListItem(* X)) when Params =/=List . [function, anywhere] - rule PARAMS(ListItem(** X:Id) ListItem(* Y:Id)) => ListItem(Id2String(Y)) ListItem(Id2String(X)) [function, anywhere] - rule PARAMS(ListItem(* X:Id)) => ListItem(Id2String(X)) [function, anywhere] - rule PARAMS(ListItem(** X:Id)) => ListItem(Id2String(X)) [function, anywhere] - rule PARAMS(ListItem(* X:Id) ListItem(** Y:Id)) => ListItem(Id2String(X)) ListItem(Id2String(Y)) [function, anywhere] + rule PARAMS(ListItem(P:Parameter) Params) => ListItem(paramName(P)) PARAMS(Params) [function, anywhere] + rule PARAMS(ListItem(* P:Parameter) Params) => PARAMS(Params ListItem(* P)) when Params =/=List . [function, anywhere] + rule PARAMS(ListItem(** P:Parameter) ListItem(* Q:Parameter)) => ListItem(paramName(Q)) ListItem(paramName(P)) [function, anywhere] + rule PARAMS(ListItem(* P:Parameter)) => ListItem(paramName(P)) [function, anywhere] + rule PARAMS(ListItem(** P:Parameter)) => ListItem(paramName(P)) [function, anywhere] + rule PARAMS(ListItem(* P:Parameter) ListItem(** Q:Parameter)) => ListItem(paramName(P)) ListItem(paramName(Q)) [function, anywhere] rule PARAMS(.) => . [function, anywhere] rule createFunction(X, Ps, K:K, O:Object) => visited((X, .Expressions) := function(Ps, O, K)) @@ -175,7 +180,7 @@ module PYTHON-SEMANTICS-FUNCTIONS | addUse(Id) | addNonlocals(NAMES) | addGlobals(NAMES) - | addParam(Parameter) + | addParam(DefParameter) | "setGenerator" rule addDef(X) => . ... diff --git a/python-syntax.k b/python-syntax.k index 8073429..bceb4f4 100755 --- a/python-syntax.k +++ b/python-syntax.k @@ -166,11 +166,15 @@ module PYTHON-SYNTAX imports PYTHON-TOKENS | FuncDef | "@" Expression "newline" Decorated [strict(1)] - syntax Parameters ::= List{Parameter,","} [seqstrict] - syntax Parameter ::= Argument - | KeyDatum - | Argument ":" Expression - | "*" + syntax Parameter ::= NAME + | NAME ":" Expression + + syntax DefParameter ::= Parameter + | Parameter "=" Expression + | "*" + | "*" Parameter + | "**" Parameter + syntax Parameters ::= List{DefParameter,","} [seqstrict] syntax FuncDef ::= "def" NAME "(" Parameters ")" "->" Expression ":" Statements | "def" NAME "(" Parameters ")" ":" Statements From e0194042d922b21682b0a3a6ca7ebd5af9de4687 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 17:13:56 -0600 Subject: [PATCH 4/7] minor fixes to annotations code --- python-semantics-functions.k | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python-semantics-functions.k b/python-semantics-functions.k index 36fc1a2..a16ba9c 100644 --- a/python-semantics-functions.k +++ b/python-semantics-functions.k @@ -56,13 +56,13 @@ module PYTHON-SEMANTICS-FUNCTIONS context makeFunction(_, (_:List ListItem(** (_ : HOLE)) _:List), _, _) context makeFunction(_, _, HOLE, _) - rule (. => addParam(X)) ~> makeCodeObject(N, (ListItem(X) L:List => L), _, false) ... + rule (. => addParam(X)) ~> makeCodeObject(N, (ListItem(X:Id) L:List => L), _, false) ... ... N ... "co_argcount" |-> (N2:Int => N2 +Int 1) "co_nlocals" |-> (N3:Int => N3 +Int 1) ... ... - rule (. => addParam(X)) ~> makeCodeObject(N, (ListItem(X) L:List => L), _, true) ... + rule (. => addParam(X)) ~> makeCodeObject(N, (ListItem(X:Id) L:List => L), _, true) ... ... N ... "co_nlocals" |-> (N2 => N2 +Int 1) "co_kwonlyargcount" |-> (N3 => N3 +Int 1) ... @@ -147,7 +147,7 @@ module PYTHON-SEMANTICS-FUNCTIONS syntax Set ::= "LOCAL" "(" Set "," Set "," Set "," List ")" [function] syntax Set ::= "BOUND" "(" Set "," List ")" [function] syntax List ::= "PARAMS" "(" List ")" [function] - rule LOCAL(Defs, Globals, Nonlocals, Params) => ((Defs Setify(PARAMS(Params))) -Set Globals) -Set Nonlocals when Setify(PARAMS(Params)) &Set Nonlocals ==Set . andBool Setify(PARAMS(Params)) &Set Globals ==Set . [function, anywhere] + rule [LOCAL]: LOCAL(Defs, Globals, Nonlocals, Params) => ((Defs Setify(PARAMS(Params))) -Set Globals) -Set Nonlocals when Setify(PARAMS(Params)) &Set Nonlocals ==Set . andBool Setify(PARAMS(Params)) &Set Globals ==Set . [function, anywhere] rule FREE(Defs, Uses, Globals, Nonlocals, ChildFree, L) => Nonlocals ChildFree (((Uses Defs) &Set BOUND(.,L)) -Set Globals) when (Nonlocals &Set BOUND(., L)) ==Set Nonlocals [function, anywhere] rule BOUND(Bound:Set, L scope(_, Defs, _, Nonlocals, Globals2, _, Params, false, _)) => BOUND((Bound LOCAL(Defs, Globals2, Nonlocals, Params)) -Set Globals2, L) [function, anywhere] rule BOUND(Bound:Set, L scope(_, _:Set, _:Set, _:Set, _:Set, _:Set, _:List, true, _)) => BOUND(Bound, L) From 340a29707470226aaf7a08c10b6e537cceb6a599 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 17:14:07 -0600 Subject: [PATCH 5/7] fix broken disambiguation --- python-semantics-assignment.k | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-semantics-assignment.k b/python-semantics-assignment.k index 4290571..dc20867 100644 --- a/python-semantics-assignment.k +++ b/python-semantics-assignment.k @@ -30,7 +30,7 @@ module PYTHON-SEMANTICS-ASSIGNMENT rule (. => iterate(O, .)) ~> bindList(_, O, _) rule (. => T, .Expressions := O) ~> list((ListItem(O) => .) _) ~> bindList((ListItem(T:Target) => .) _, _, _) when getKLabel T =/=KLabel '*_ - rule (. => T, .Expressions := mutable(list(Refify(Idify(L))), ref("list"))) ~> list((L:List => .) L2:List) ~> bindList((ListItem(* T:Target) => .) L3:List, _, (false => true)) when lengthList L2 ==Int lengthList L3 + rule (. => T, .Expressions := mutable(list(Refify(Idify(L))), ref("list"))) ~> list((L:List => .) L2:List) ~> bindList((ListItem(* T:Expression) => .) L3:List, _, (false => true)) when lengthList L2 ==Int lengthList L3 rule list(.) ~> bindList(., _, _) => . rule list(ListItem(_) _) ~> bindList(., _, _) => raiseInternal("ValueError", "too many values to unpack") rule list(.) ~> bindList(ListItem(T) _, _, _) => raiseInternal("ValueError", "need more values to unpack") when getKLabel T =/=KLabel '*_ From 414eeff004ec89e53b44ef5c4ee05ab7d9131211 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 17:14:18 -0600 Subject: [PATCH 6/7] fix follow restriction --- python-syntax.k | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python-syntax.k b/python-syntax.k index bceb4f4..d8d36ae 100755 --- a/python-syntax.k +++ b/python-syntax.k @@ -101,7 +101,9 @@ module PYTHON-SYNTAX imports PYTHON-TOKENS > Expression "if" Expression "else" Expression [strict(2), right] > "lambda" Parameters ":" Expression - | "*" Expression [strict, hybrid, non-assoc] + | "*" Expression [strict, hybrid] + + syntax "*" -/- [\*] syntax Expressions ::= List{Expression,","} From 28cdae0948f738b6fc124becea8af38607dfd61c Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Thu, 21 Feb 2013 17:42:09 -0600 Subject: [PATCH 7/7] closing this branch because I don't want to implement the parse time fix this way. Will deal with this in the bigass refactoring later --HG-- extra : close : 1