From a811544eeb2120cb91802ac623f87a63e824d631 Mon Sep 17 00:00:00 2001 From: Tim Vieira Date: Fri, 12 Jul 2013 14:53:07 -0400 Subject: [PATCH] `dump_rules` won't crash when rule failed to load dyna source. Show failure context when an update handler crashes (work in progress) when an initializer fails rules is still added to the program. Only some of the emits which succeeded get scheduled (this isn't quite right). added depth limit argument to trace (available via --limit= at the repl). --- examples/forward-backward.dyna | 26 ++++---- src/Dyna/Backend/Python/interpreter.py | 82 +++++++++++++++++++++++--- src/Dyna/Backend/Python/post/trace.py | 50 +++++++++++----- src/Dyna/Backend/Python/repl.py | 15 ++++- src/Dyna/Backend/Python/utils.py | 1 + 5 files changed, 135 insertions(+), 39 deletions(-) diff --git a/examples/forward-backward.dyna b/examples/forward-backward.dyna index facf3c5..0ad2318 100644 --- a/examples/forward-backward.dyna +++ b/examples/forward-backward.dyna @@ -25,23 +25,23 @@ word(last_time+1) := &eos. num_iterations := 10. is_iter(0). -is_iter(I+1) |= true for is_iter(I) & I < num_iterations. +is_iter(I+1) :- true for is_iter(I) & I < num_iterations. % --------------------------------------------------- % FORWARD-BACKWARD ALGORITHM (THE "E STEP" OF THE EM ALGORITHM) % --------------------------------------------------- -% At each iteration I = 0, 1, ... num_iterations, +% At each iteration I = 0, 1, ... num_iterations, % define a trellis graph on states of the form &state(Time,Tag), % which expresses all ways of explaining the word sequence. % The edge weights are defined from iteration I's probability model. -edge(I,&state(Time-1,PrevTag),&state(Time,Tag)) +edge(I,&state(Time-1,PrevTag),&state(Time,Tag)) = p_transition(I,PrevTag,Tag) * p_emission(I,Tag,word(Time)). start_state = &state(0,&start). % the initial tag at time 0 is called &start end_state = &state(last_time+1,&stop). % the final tag is called &stop -% Compute alphas and betas on that trellis graph, using +% Compute alphas and betas on that trellis graph, using % dynamic programming. alpha(I,start_state) += 1 for is_iter(I). @@ -59,27 +59,27 @@ z(I) = alpha(I,end_state). % true path is one of the paths in the trellis. prob(I,State) = alpha(I,State) * beta(I,State) / z(I). -prob(I,PrevState,State) +prob(I,PrevState,State) = alpha(I,PrevState) * edge(I,PrevState,State) * beta(I,State) / z(I). % --------------------------------------------------- % COUNTING TRANSITIONS AND EMISSIONS % --------------------------------------------------- -% From the above, we can get the expected counts of specific +% From the above, we can get the expected counts of specific % transitions and emissions. count_transition(I,PrevTag,Tag) += prob(I,&state(Time-1,PrevTag),&state(Time,Tag)). count_emission(I,Tag,word(Time)) += prob(I,&state(Time,Tag)). -% Smooth by adding lambda to all counts corresponding to parameters, +% Smooth by adding lambda to all counts corresponding to parameters, % even counts that are still 0 or null. lambda := 0.0. % can be changed count_transition(I,PrevTag,Tag) += lambda for (Param is p_transition(I,PrevTag,Tag)). count_emission(I,Tag,Word) += lambda for (Param is p_emission(I,Tag,Word)). -% Get the total smoothed count in each context. These will be the +% Get the total smoothed count in each context. These will be the % denominators of our smoothed probabiities. count_transition_denom(I,PrevTag) += count_transition(I,PrevTag,Tag). @@ -100,10 +100,10 @@ p_emission(0,Tag,Word) := p_e(Tag,Word). % Given the smoothed counts at time I, we define the new model at time I+1: -p_transition(I+1,PrevTag,Tag) - = count_transition(I,PrevTag,Tag) / count_transition_denom(I,PrevTag) +p_transition(I+1,PrevTag,Tag) + = count_transition(I,PrevTag,Tag) / count_transition_denom(I,PrevTag) for I < num_iterations. -p_emission(I+1,Tag,Word) +p_emission(I+1,Tag,Word) := count_emission(I,Tag,Word) / count_emission_denom(I,Tag) for I < num_iterations. @@ -117,7 +117,7 @@ p_emission(I,&stop,&eos) := 1 for is_iter(I). % DRAW CONCLUSIONS FROM THE ESTIMATED MODEL % --------------------------------------------------- -% After the model is estimated, look at the computations for the final +% After the model is estimated, look at the computations for the final % iteration to find the most likely tag at each time step, and its probability. besttagprob(Time) max= prob(num_iterations, &state(Time,Tag)) with_key Tag. @@ -130,5 +130,3 @@ bestpathprob(State) max= edge(num_iterations,State,NextState) * bestpathprob(Nex with_key [Tag | $key(bestpathprob(NextState))] for &state(_,Tag) = State. bestpathprob = bestpathprob(start_state). bestpath = $key(bestpathprob(start_state)). - -p \ No newline at end of file diff --git a/src/Dyna/Backend/Python/interpreter.py b/src/Dyna/Backend/Python/interpreter.py index 6788629..f90de61 100644 --- a/src/Dyna/Backend/Python/interpreter.py +++ b/src/Dyna/Backend/Python/interpreter.py @@ -91,18 +91,24 @@ from stdlib import todyna class Rule(object): + def __init__(self, index): self.index = index self.init = None self.updaters = [] self.query = None + @property def span(self): - span = parse_attrs(self.init or self.query)['Span'] - return hide_ugly_filename(span) + if self.init or self.query: + span = parse_attrs(self.init or self.query)['Span'] + return hide_ugly_filename(span) + @property def src(self): - return strip_comments(parse_attrs(self.init or self.query)['rule']) + if self.init or self.query: + return strip_comments(parse_attrs(self.init or self.query)['rule']) + def __repr__(self): return 'Rule(%s, %r)' % (self.index, self.src) @@ -248,7 +254,21 @@ class Interpreter(object): break print >> out, ' when `%s` = %s' % (item, _repr(value)) print >> out, ' %s' % (e) - print >> out + print >> out + + # TODO: include an undefined or unknown marker + # TODO: can highlight the expression which raise the error. + from post.trace import Crux + + c = Crux(head=None, + rule=r, + body=None, + vs = dict(e.exception_frame)) + + for line in c.format(): + print ' ', line + + print >> out print >> out @@ -411,8 +431,50 @@ class Interpreter(object): try: handler(item, val, emit=t_emit) except (TypeError, ZeroDivisionError, KeyboardInterrupt, OverflowError) as e: + + import traceback + + # Move to the frame where the exception occurred, which is often not the + # same frame where the exception was caught. + tb = sys.exc_info()[2] + if tb is not None: + while 1: + if not tb.tb_next: + break + tb = tb.tb_next + f = tb.tb_frame + else: # no exception occurred + f = sys._getframe() + + # get the stack frames + stack = [] + while f: + stack.append(f) + f = f.f_back + stack.reverse() + + if 0: + print 'Traceback:' + print '==========' + print traceback.format_exc() + + print 'Locals by frame:' + print '================' + for frame in stack: + print 'Frame %s in %s at line %s' % (frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_lineno) + for key, value in frame.f_locals.iteritems(): + print '%20s = %r' % (key, value) + + print + print + + + e.exception_frame = stack[-1].f_locals.items() error.append((e, handler)) + if error: self.error[item] = (val, error) return @@ -504,7 +566,6 @@ class Interpreter(object): def _emit(*args): emits.append(args) - # TODO: this should be a transaction. for k, v in env.agg_decl.items(): self.new_fn(k, v) @@ -527,7 +588,13 @@ class Interpreter(object): except (TypeError, ZeroDivisionError) as e: raise DynaInitializerException(e, init) - else: + + finally: + + # process emits + for e in emits: + self.emit(*e, delete=False) + for fn, r, h in env.updaters: self.new_updater(fn, r, h) for r, h in env.initializers: @@ -536,9 +603,6 @@ class Interpreter(object): # accept the new parser state self.parser_state = env.parser_state - # process emits - for e in emits: - self.emit(*e, delete=False) # ------ $rule for fun and profit ------- interp = self diff --git a/src/Dyna/Backend/Python/post/trace.py b/src/Dyna/Backend/Python/post/trace.py index 2a17161..c431959 100644 --- a/src/Dyna/Backend/Python/post/trace.py +++ b/src/Dyna/Backend/Python/post/trace.py @@ -41,11 +41,16 @@ class Tracer(object): groups[a] = groupby(lambda x: x[1], groups[a]) self.items = groups - def __call__(self, item): + def __call__(self, item, depth_limit=-1): if item not in self.items: print print 'no trace for', item - print '\n'.join(dig(item, set(), tuple(), self.items, self.interp)) + print '\n'.join(dig(head = item, + visited = set(), + tail = tuple(), + groups = self.items, + interp = self.interp, + depth_limit = depth_limit)) def groupby(key, data): @@ -55,7 +60,11 @@ def groupby(key, data): return dict(g) -def dig(head, visited, tail, groups, interp): +def dig(head, visited, tail, groups, interp, depth_limit=-1): + + print head, len(tail), depth_limit + if depth_limit >= 0 and len(tail) >= depth_limit: + return [yellow % '*max depth*'] if head in tail: return ['%s = %s' % (yellow % head, _repr(head.value))] \ @@ -80,12 +89,12 @@ def dig(head, visited, tail, groups, interp): for (_, _, body, vs) in groups[head][ruleix]: crux = Crux(head, interp.rules[ruleix], body, dict(vs)) - block = branch([dig(x, visited, tail + (head,), groups, interp) for x in body]) + block = branch([dig(x, visited, tail + (head,), groups, interp, depth_limit) for x in body]) if block: - contribs.append(crux.format() + ['|'] + block) + contribs.append(crux.fvalue() + [''] + crux.format() + ['|'] + block) else: - contribs.append(crux.format() + ['']) + contribs.append(crux.fvalue() + [''] + crux.format() + ['']) return ['%s = %s' % (yellow % head, cyan % _repr(head.value))] \ + ['|'] \ @@ -131,10 +140,13 @@ class Crux(object): except (SyntaxError, NameError): return x + def fvalue(self): + # show value and aggregator.. separate from format so we can use in + # error handling + return ['%s %s' % (green % self.rule.anf.agg, self.values(self.rule.anf.result))] + def format(self): rule = self.rule - #src = rule.src.replace('\n',' ').strip() - #user_vars = dict(user_vars(self.vs.items())) graph = self.graph side = [self.get_function(x) for x in graph.outputs if x != rule.anf.result and x != rule.anf.head] @@ -149,9 +161,7 @@ class Crux(object): else: explode += '.' - lines = ['%s %s' % (green % rule.anf.agg, self.values(rule.anf.result)), - '', - explode] + lines = [explode] if side: lines.append(side) @@ -165,8 +175,10 @@ class Crux(object): """ g = self.graph if isinstance(x, debug.Edge): - label = re.sub('@\d+$', '', x.label) - label = re.sub('^& ', '&', label) + + # clean up edge name + label = re.sub('@\d+$', '', x.label) # remove evaluation index + label = re.sub('^& ', '&', label) # normalize prefix quote operator if label == '=': [b] = x.body @@ -191,6 +203,7 @@ class Crux(object): return x[1:] + (cyan % '=%s' % self.values(x)) return self.values(x) + # handle multiple values (can happen at unification nodes) if len(g.incoming[x]) > 1: return ' = '.join('(%s%s)' % (self.get_function(e), (cyan % '=%s' % self.values(e.head))) for e in g.incoming[x]) @@ -200,7 +213,6 @@ class Crux(object): return self.get_function(e) # handle lists - if e.label == '& nil': return '[]' @@ -220,3 +232,13 @@ class Crux(object): return self.get_function(e) return self.get_function(e) + (cyan % '=%s' % self.values(x)) + + +class Expr(object): + pass +class Fn(Expr): + pass +class Infix(Expr): + pass +class Unif(Expr): + pass diff --git a/src/Dyna/Backend/Python/repl.py b/src/Dyna/Backend/Python/repl.py index bd661d8..3b2874e 100644 --- a/src/Dyna/Backend/Python/repl.py +++ b/src/Dyna/Backend/Python/repl.py @@ -7,7 +7,7 @@ TODO: help should print call signature of loads and post-processors in addition to help. """ -import os, cmd, readline +import re, os, cmd, readline from utils import ip, lexer, subst, drepr, _repr, get_module from stdlib import topython, todyna from errors import DynaCompilerError, DynaInitializerException @@ -550,6 +550,17 @@ class REPL(cmd.Cmd, object): print "Queries don't end with a dot." return + depth_limit = -1 + p = re.compile('--limit=(\d+)\\b') + x = p.findall(q) + if x: + q = p.sub('', q) + depth_limit = int(x[0]) + + self._trace(q, depth_limit=depth_limit) + + def _trace(self, q, depth_limit=-1): + self.interp.new_rules = set() try: @@ -572,7 +583,7 @@ class REPL(cmd.Cmd, object): for item in results: print - tracer(todyna(item)) + tracer(todyna(item), depth_limit=depth_limit) finally: # cleanup: diff --git a/src/Dyna/Backend/Python/utils.py b/src/Dyna/Backend/Python/utils.py index b2e8130..b84c1af 100644 --- a/src/Dyna/Backend/Python/utils.py +++ b/src/Dyna/Backend/Python/utils.py @@ -34,6 +34,7 @@ def _repr(x): # TODO: this assertion should eventually hold. # assert x is not True and x is not False, x + if x is True: return 'true' elif x is False: -- 2.50.1