From 6cd1ad9647c18fabb989f057c9e8d490efb3300d Mon Sep 17 00:00:00 2001 From: timv Date: Thu, 6 Jun 2013 18:49:43 -0400 Subject: [PATCH] Interpreter states encapulated in an `Interpreter` instance. - this was a big refactoring adds a few new files (chart.py, repl.py, config.py) Fix: Errors stop on first failed handler. Adding a new rule is safe. We check that the initializers run and there are no aggregator conflicts before added the new rules to `Interpreter` state. We stage propagation. I've document a few more errors (see `examples/string-quote.dyna`) Using the ~/.dyna directory - adds a dependency to the `path.py` module --- examples/errors.dyna | 4 +- examples/string-quote.dyna | 3 + src/Dyna/Backend/Python/Backend.hs | 8 +- src/Dyna/Backend/Python/chart.py | 117 ++++ src/Dyna/Backend/Python/config.py | 5 + src/Dyna/Backend/Python/debug.py | 52 +- src/Dyna/Backend/Python/interpreter.py | 826 ++++++++----------------- src/Dyna/Backend/Python/repl.py | 140 ++++- src/Dyna/Backend/Python/utils.py | 7 +- 9 files changed, 564 insertions(+), 598 deletions(-) create mode 100644 examples/string-quote.dyna create mode 100644 src/Dyna/Backend/Python/chart.py create mode 100644 src/Dyna/Backend/Python/config.py diff --git a/examples/errors.dyna b/examples/errors.dyna index e44fa3b..084a021 100644 --- a/examples/errors.dyna +++ b/examples/errors.dyna @@ -5,11 +5,13 @@ a += 1/b. c += "" + b. +e := 0. + % initializer exceptions. %p += 1+null. %q max= 3/0. - b := b/0. a += b/0. + b := e/0. a += e/0. d += null. d += 1. diff --git a/examples/string-quote.dyna b/examples/string-quote.dyna new file mode 100644 index 0000000..a28931d --- /dev/null +++ b/examples/string-quote.dyna @@ -0,0 +1,3 @@ +% example to test string quoting + +test("I'd like to \"quote\"") += 1. diff --git a/src/Dyna/Backend/Python/Backend.hs b/src/Dyna/Backend/Python/Backend.hs index 380919f..a771efe 100644 --- a/src/Dyna/Backend/Python/Backend.hs +++ b/src/Dyna/Backend/Python/Backend.hs @@ -268,7 +268,7 @@ printPlanHeader r c mn = do printInitializer :: Handle -> Rule -> Cost -> Actions PyDopeBS -> IO () printInitializer fh rule@(Rule _ h _ r _ _ ucruxes _) cost dope = do displayIO fh $ renderPretty 1.0 100 - $ "@initializer" <> (uncurry pfa $ MA.fromJust $ findHeadFA h ucruxes) + $ "@_initializers.append" -- <> (uncurry pfa $ MA.fromJust $ findHeadFA h ucruxes) `above` "def" <+> char '_' <> tupled ["emit"] <> colon `above` (indent 4 $ printPlanHeader rule cost Nothing) `above` pdope dope @@ -278,11 +278,15 @@ printInitializer fh rule@(Rule _ h _ r _ _ ucruxes _) cost dope = do printUpdate :: Handle -> Rule -> Cost -> Int -> Maybe DFunctAr -> (DVar, DVar) -> Actions PyDopeBS -> IO () printUpdate fh rule@(Rule _ h _ r _ _ _ _) cost evalix (Just (f,a)) (hv,v) dope = do displayIO fh $ renderPretty 1.0 100 - $ "@register" <> (pfa f a) + $ "#" <+> (pfa f a) `above` "def" <+> char '_' <> tupled (map pretty [hv,v,"emit"]) <> colon `above` (indent 4 $ printPlanHeader rule cost (Just evalix)) `above` pdope dope <> line + <> "_updaters.append((" <> (pfa f a) <> ", _))" + <> line + <> line + <> line ------------------------------------------------------------------------}}} -- Driver {{{ diff --git a/src/Dyna/Backend/Python/chart.py b/src/Dyna/Backend/Python/chart.py new file mode 100644 index 0000000..8c06640 --- /dev/null +++ b/src/Dyna/Backend/Python/chart.py @@ -0,0 +1,117 @@ +from collections import defaultdict +from utils import notimplemented + + +# TODO: codegen should output a derived Term instance for each functor +class Term(object): + + __slots__ = 'fn args value aggregator'.split() + + def __init__(self, fn, args): + self.fn = fn + self.args = args + self.value = None + self.aggregator = None + + def __cmp__(self, other): + if other is None: + return 1 + return cmp((self.fn, self.args), (other.fn, other.args)) + + # default hash and eq suffice because we intern + #def __hash__(self): + #def __eq__(self): + + def __repr__(self): + "Pretty print a term. Will retrieve the complete (ground) term." + fn = '/'.join(self.fn.split('/')[:-1]) # drop arity from name. + if not self.args: + return fn + return '%s(%s)' % (fn, ','.join(map(_repr, self.args))) + + __add__ = __sub__ = __mul__ = notimplemented + + +#def _repr(x): +# if isinstance(x, basestring): +# # dyna doesn't accept single-quoted strings +# return '"%s"' % x.replace('"', r'\"') +# else: +# return repr(x) +_repr = repr + + +class Chart(object): + + def __init__(self, name, arity, new_aggregator): + self.name = name + self.arity = arity + self.intern = {} # args -> term + self.ix = [defaultdict(set) for _ in xrange(arity)] + self.new_aggregator = new_aggregator + + def __repr__(self): + rows = [term for term in self.intern.values() if term.value is not None] + x = '\n'.join('%-30s := %r' % (term, term.value) for term in sorted(rows)) + return '%s\n=================\n%s' % (self.name, x) + + def __getitem__(self, s): + assert len(s) == self.arity + 1, \ + 'item width mismatch: arity %s, item %s' % (self.arity, len(s)) + + args, val = s[:-1], s[-1] + + assert val is not None + + # filter set of candidates by each bound argument + candidates = None + for (ix, x) in zip(self.ix, args): + if isinstance(x, slice): + continue + if candidates is None: + # initial candidates determined by first non-bound column + candidates = ix[x].copy() + else: + candidates &= ix[x] + if not len(candidates): + # no candidates left + break + + if candidates is None: + # This happens when all arguments are free. + candidates = self.intern.values() + + # handle the value column separately because we don't index it yet. + if isinstance(val, slice): + for term in candidates: + if term.value is not None: + yield term, term.args + (term.value,) + else: + for term in candidates: + if term.value == val: + yield term, term.args + (term.value,) # TODO: change codegen to avoid addition.. + + def lookup(self, args): + "find index for these args" + assert len(args) == self.arity + + try: + return self.intern[args] + except KeyError: + return None + + def insert(self, args, val): + + # debugging check: row is not already in chart. + assert self.lookup(args) is None, \ + '%r already in chart with value %r' % (args, val) + + self.intern[args] = term = Term(self.name, args) + term.value = val + term.aggregator = self.new_aggregator() + + # index new term + for i, x in enumerate(args): + self.ix[i][x].add(term) + + return term diff --git a/src/Dyna/Backend/Python/config.py b/src/Dyna/Backend/Python/config.py new file mode 100644 index 0000000..5f5993b --- /dev/null +++ b/src/Dyna/Backend/Python/config.py @@ -0,0 +1,5 @@ +from path import path + +dotdynadir = path('~/.dyna').expand() +if not dotdynadir.exists(): + dotdynadir.mkdir() diff --git a/src/Dyna/Backend/Python/debug.py b/src/Dyna/Backend/Python/debug.py index 6ebb1fb..5a89edc 100644 --- a/src/Dyna/Backend/Python/debug.py +++ b/src/Dyna/Backend/Python/debug.py @@ -4,8 +4,9 @@ Generates a visual representation of a Dyna program rules after the normalization process. """ -import re, os, shutil +import re, os, shutil, webbrowser from collections import defaultdict, namedtuple +from cStringIO import StringIO from utils import magenta, red, green, yellow, white, read_anf, dynahome from pygments import highlight @@ -398,10 +399,57 @@ Initializer: print >> html, '' if browser: - import webbrowser webbrowser.open(html.name) +def hypergraph(interpreter): + # collect edges + interpreter.collect_edges() + # create hypergraph object + g = Hypergraph() + for c in interpreter.chart.values(): + for x in c.intern.values(): + for e in interpreter.edges[x]: + label, body = e + g.edge(str(x), str(label), map(str, body)) + return g + + +def draw(interpreter): + g = hypergraph(interpreter) + with file('/tmp/state.html', 'wb') as f: + print >> f, """ + + + + + + """ + + x = StringIO() + interpreter.dump_charts(x) + + print >> f, '
%s
' \ + % '

Charts

%s' \ + % '
%s
' \ + % x.getvalue() + + print >> f, """ +
+

Hypergraph

+ %s +
+ """ % g.render('/tmp/hypergraph') + + print >> f, '' + + webbrowser.open(f.name) + if __name__ == '__main__': diff --git a/src/Dyna/Backend/Python/interpreter.py b/src/Dyna/Backend/Python/interpreter.py index b941ecf..4e8a24f 100644 --- a/src/Dyna/Backend/Python/interpreter.py +++ b/src/Dyna/Backend/Python/interpreter.py @@ -1,20 +1,14 @@ #!/usr/bin/env python """ +MISC +==== -This error message is unhelpful - - :-dispos_def dyna. - :-ruleix 27. - rewrite("VP", "V", "NP") -= 100. + - TODO: faster charts (dynamic argument types? jason's trie data structure) - FATAL: Encountered error in input program: - Parser error - /tmp/tmp.dyna:1:1: error: expected: end of input - :-dispos_def dyna. + - TODO: write files to ~/.dyna -MISC -==== + - TODO: (@nwf) String quoting (see example/stringquote.py) - TODO: filter and bulk loader @@ -22,11 +16,7 @@ MISC few things -- I think assertionerror is one them... we should probably do whatever this is doing with a custom exception. - - TODO: mode planning failures are slient. - - timv: I think this is a job for @nwf - - - TODO: create an Interpreter object to hold state. + - TODO: (@nwf) mode planning failures are slient - TODO: deleting a rule: (1) remove update handlers (2) run initializers in delete mode (3) remove initializers. @@ -49,14 +39,9 @@ MISC - TODO: doc tests for Dyna code! - - TODO: repl needs to pass parser a rule index pragma to start from. - - blocked: nwf will tell me what bits of parser state to send back to him. - - TODO: Numeric precision is an issue with BAggregators. - timv: Are we sure we have this bug? or possible that we want to handle - it in an adhoc fashion? + timv: Are we sure we have this bug? a[0.1] += 1 a[0.1 + eps] -= 1 @@ -97,38 +82,19 @@ Warnings/lint checking - Catch typos! Warn the user if they write a predicate that is not defined on the LHS of a rule and it's not quoted (i.e. not some new piece of structure). - -REPL -==== - - - TODO: (Aggregator conflicts) - - We throw and AggregatorConflict exception if newly loaded code tries to - overwrite an aggregator in `_agg_decl`. - - However, we need to make sure that we don't load subsequent code pertaining - to this rule that we should reject altogether. - - timv: At the moment I believe we're safe because `_agg_decl` is set before - any of the registers (i.e. it's at the top of the generated code). This - obviously isn't the best way to do this, but we're going to have to overhaul - this entire infrastructure soon to handle rule-retraction.. So we can fix - this later. - """ from __future__ import division -import re, os, sys +import os, sys from collections import defaultdict -from functools import partial from argparse import ArgumentParser -from StringIO import StringIO -import webbrowser - -from utils import ip, red, green, blue, magenta, yellow, dynahome, \ - notimplemented, prioritydict +import debug +from chart import Chart, Term from defn import aggregator +from utils import ip, red, green, blue, magenta, yellow, dynahome, \ + notimplemented, prioritydict, parse_attrs +from config import dotdynadir class AggregatorConflict(Exception): @@ -147,288 +113,280 @@ class DynaInitializerException(Exception): super(DynaInitializerException, self).__init__(msg) -# TODO: as soon as we have safe names for these things we can get rid of this. -class chart_indirect(dict): - def __missing__(self, key): - arity = int(key.split('/')[-1]) - c = self[key] = Chart(name = key, arity = arity) - return c +# TODO: +class DynaCompilerError(Exception): + pass -# when a new rule comes along it puts a string in the following dictionary -class aggregator_declaration(object): - def __init__(self): - self.map = {} - def __setitem__(self, key, val): - if key in self.map and self.map[key] != val: - raise AggregatorConflict(key, self.map[key], val) - self.map[key] = val - def __getitem__(self, key): - try: - return self.map[key] - except KeyError: - return None +class Interpreter(object): + def __init__(self): + # declarations + self.agg_name = {} + self.edges = defaultdict(set) + self.updaters = defaultdict(list) + self.initializers = [] + # data structures + self.agenda = prioritydict() + self.parser_state = '' + + agg = self.agg_name + class dd(dict): + def __missing__(self, fn): + arity = int(fn.split('/')[-1]) + self[fn] = c = Chart(fn, arity, lambda: aggregator(agg[fn])) + return c + self.chart = dd() + + self.errors = {} + # misc + self.trace = file(dotdynadir / 'trace', 'wb') + + def new_fn(self, fn, agg): + if fn in self.agg_name: + # check for aggregator conflict. + if self.agg_name[fn] != agg: + raise AggregatorConflict(fn, self.agg_name[fn], agg) + return + self.agg_name[fn] = agg -# options -error_suppression = True -trace = None - -agenda = prioritydict() -chart = chart_indirect() -errors = {} -changed = {} - -# declarations -_agg_decl = aggregator_declaration() -_edges = defaultdict(set) -_updaters = defaultdict(list) -_initializers = [] -_rules = {} - - -def dump_charts(out=sys.stdout): - print >> out - print >> out, 'Charts' - print >> out, '============' - - fns = chart.keys() - fns.sort() + def collect_edges(self): + """ + Use rule initializers to find all active hyperedges in the current + Chart. + """ + edges = self.edges + def _emit(item, _, ruleix, variables): + b = variables['nodes'] + b.sort() + b = tuple(b) + edges[item].add((ruleix, b)) + for init in self.initializers: + init(emit=_emit) - for x in fns: - print >> out, chart[x] + def dump_charts(self, out=sys.stdout): + print >> out + print >> out, 'Charts' + print >> out, '============' + fns = self.chart.keys() + fns.sort() + for x in fns: + print >> out, self.chart[x] + print >> out + self.dump_errors(out) + + def dump_errors(self, out=sys.stdout): + # We only dump the error chart if it's non empty. + if not self.errors: + return + print >> out + print >> out, 'Errors' + print >> out, '============' + for item, (val, es) in self.errors.items(): + print >> out, 'because %r is %r:' % (item, val) + for e in es: + print >> out, ' ', e print >> out - dump_errors(out) - - -def dump_errors(out=sys.stdout): - if not errors: - return - # only print errors if we 'em. - print >> out - print >> out, 'Errors' - print >> out, '============' - for item, (val, es) in errors.items(): - print >> out, 'because %r is %r:' % (item, val) - for e in es: - print >> out, ' ', e - print >> out - + def build(self, fn, *args): + # TODO: codegen should handle true/0 is True and false/0 is False + if fn == "true/0": + return True + if fn == "false/0": + return False + + # FIXME: + if fn not in self.agg_name: + # TODO: if the item has no aggregator (e.g purely structural stuff) + self.new_fn(fn, None) + + term = self.chart[fn].lookup(args) + if term is None: + term = self.chart[fn].insert(args, None) # don't know val yet. + return term -# TODO: codegen should output a derived Term instance for each functor -class Term(object): + def go(self): + "the main loop" - __slots__ = 'fn args value aggregator'.split() + changed = {} + agenda = self.agenda + errors = self.errors + trace = self.trace - def __init__(self, fn, args): - self.fn = fn - self.args = args - self.value = None - self.aggregator = None + while agenda: + item = agenda.pop_smallest() - def __cmp__(self, other): - if other is None: - return 1 - return cmp((self.fn, self.args), (other.fn, other.args)) + print >> trace + print >> trace, magenta % 'pop ', item, - # default hash and eq suffice because we intern - #def __hash__(self): - #def __eq__(self): + was = item.value + print >> trace, '(was: %r,' % (was,), - def __repr__(self): - "Pretty print a term. Will retrieve the complete (ground) term." - fn = '/'.join(self.fn.split('/')[:-1]) # drop arity from name. - if not self.args: - return fn - return '%s(%s)' % (fn, ','.join(map(repr, self.args))) + try: + now = item.aggregator.fold() + except (ZeroDivisionError, TypeError) as e: + errors[item] = ('failed to aggregate %r' % item.aggregator, [e]) + # TODO: Are we sure there is never a reason to requeue this item. + continue - __add__ = __sub__ = __mul__ = notimplemented + print >> trace, 'now: %r)' % (now,) + if was == now: + print >> trace, yellow % 'unchanged' + continue -def edges(): - def _emit(item, val, ruleix, variables): - b = variables['nodes'] - b.sort() - b = tuple(b) - _edges[item].add((ruleix, b)) - for init in _initializers: - init(emit=_emit) + was_error = False + if item in errors: # clear the error + was_error = True + del errors[item] + # TODO: handle `was` and `now` at the same time to avoid the two passes. + # TODO: will need to propagate was=None when we have question mark + if was is not None and not was_error: + # if `was` is marked as an error we know it didn't propagate. + # Thus, we can skip the delete-updates. + self.update_dispatcher(item, was, delete=True) -class Chart(object): + item.value = now - def __init__(self, name, arity): - self.name = name - self.arity = arity - self.intern = {} # args -> term - self.ix = [defaultdict(set) for _ in xrange(arity)] + if now is not None: + self.update_dispatcher(item, now, delete=False) - def __repr__(self): - rows = [term for term in self.intern.values() if term.value is not None] - x = '\n'.join('%-30s := %r' % (term, term.value) for term in sorted(rows)) - return '%s\n=================\n%s' % (self.name, x) + changed[item] = now - def __getitem__(self, s): - assert len(s) == self.arity + 1, \ - 'item width mismatch: arity %s, item %s' % (self.arity, len(s)) + return changed - args, val = s[:-1], s[-1] + def update_dispatcher(self, item, val, delete): + """ + Passes update to relevant handlers. + """ + # XXX: fix for `?` prefix operator assert val is not None - # filter set of candidates by each bound argument - candidates = None - for (ix, x) in zip(self.ix, args): - if isinstance(x, slice): - continue - if candidates is None: - # initial candidates determined by first non-bound column (if any) - candidates = ix[x].copy() - else: - candidates &= ix[x] - if not len(candidates): - # no candidates left - break - - if candidates is None: - # This happens when all arguments are free. - candidates = self.intern.values() - - # handle the value column separately because we don't index it yet. - if isinstance(val, slice): - for term in candidates: - if term.value is not None: - yield term, term.args + (term.value,) - else: - for term in candidates: - if term.value == val: - yield term, term.args + (term.value,) # TODO: change codegen to avoid addition.. - - def lookup(self, args): - "find index for these args" - assert len(args) == self.arity + # store emissions, make sure all of them succeed before propagating + # changes to aggregators. + emittiers = [] + t_emit = lambda item, val, ruleix, variables: \ + emittiers.append((item, val, ruleix, variables, delete)) try: - return self.intern[args] - except KeyError: - return None - - def insert(self, args, val): - # debugging check: row is not already in chart. - assert self.lookup(args) is None, \ - '%r already in chart with value %r' % (args, val) + # TODO: do we want to collect all handlers with errors? + for handler in self.updaters[item.fn]: + handler(item, val, emit=t_emit) - self.intern[args] = term = Term(self.name, args) - term.value = val - term.aggregator = aggregator(_agg_decl[self.name]) + except (TypeError, ZeroDivisionError) as e: - # indexes new term - for i, x in enumerate(args): - self.ix[i][x].add(term) + if item not in self.errors: + self.errors[item] = (val, []) - return term + # TODO: don't eagerly format the message. + self.errors[item][1].append('%s\n in rule %s\n %s' % \ + (e, + handler.dyna_attrs['Span'], + handler.dyna_attrs['rule'])) + else: + # no exception, accept emissions. + for e in emittiers: + # an error could happen here, but we assume (by contract) that + # this is not possible. + self.emit(*e) -def build(fn, *args): - # TODO: codegen should handle true/0 is True and false/0 is False - if fn == "true/0": - return True - if fn == "false/0": - return False - term = chart[fn].lookup(args) - if term is None: - term = chart[fn].insert(args, None) # don't know val yet. - return term + def new_updater(self, fn, handler): + self.updaters[fn].append(handler) + def new_initializer(self, init): + self.initializers.append(init) -# Update handler indirection -- a temporary hack. Allow us to have many handlers -# on the same functor/arity. Eventually, we'll fuse handlers into one handler. + def emit(self, item, val, ruleix, variables, delete): + print >> self.trace, (red % 'delete' if delete else green % 'update'), \ + '%s (val %s; curr: %s)' % (item, val, item.value) -def register(fn): - """ - Decorator for registering update handlers. Used by update dispatcher. + if delete: + item.aggregator.dec(val, ruleix, variables) + else: + item.aggregator.inc(val, ruleix, variables) - Note: registration is with a global/mutable table. - """ + self.agenda[item] = 0 # everything is high priority - def wrap(handler): - handler.dyna_attrs = parse_attrs(handler) - _updaters[fn].append(handler) - # you can't call these guys directly. Must go thru handler - # indirection table - return None + def repl(self, hist): + import repl + repl.REPL(self, hist).cmdloop() - return wrap + def do(self, filename): + "Compile, load, and execute dyna code." + assert os.path.exists(filename) -# - "initializers" aren't just initializers -- They are fully-naive bottom-up -# inference routines. At the moment we only use them to initialize the chart. + # for debuggging + with file(filename) as h: + print >> self.trace, magenta % 'Loading new code' + print >> self.trace, yellow % h.read() -from utils import parse_attrs + # TODO: loading new code should be atomic. if we fail for some reason we + # need to revert. -def initializer(_): - "Implementation idea is very similar to register." + env = {'_initializers': [], '_updaters': [], '_agg_decl': {}, + 'chart': self.chart, 'build': self.build, 'peel': peel, + 'parser_state': None} - def wrap(handler): - handler.dyna_attrs = parse_attrs(handler) - _initializers.append(handler) - return None + # load generated code. + execfile(filename, env) - return wrap + emits = [] + def _emit(*args): + emits.append(args) + for k, v in env['_agg_decl'].items(): + self.new_fn(k, v) + for fn, h in env['_updaters']: + h.dyna_attrs = parse_attrs(h) + for h in env['_initializers']: + h.dyna_attrs = parse_attrs(h) -def update_dispatcher(item, val, delete): - """ - Passes update to relevant handlers. - """ - - if val is None: - return + try: + # only run new initializers + for init in env['_initializers']: + init(emit=_emit) - for handler in _updaters[item.fn]: + except (TypeError, ZeroDivisionError) as e: + raise DynaInitializerException(e, init) - emittiers = [] - _emit = lambda item, val, ruleix, variables: \ - emittiers.append((item, val, ruleix, variables, delete)) + else: + # add new updaters + for fn, h in env['_updaters']: + self.new_updater(fn, h) - try: - handler(item, val, emit=_emit) - except (TypeError, ZeroDivisionError) as e: - if error_suppression: - #print >> trace, - print '%s on update %s = %s' % (e, item, val) + # add new initializers + for h in env['_initializers']: + self.new_initializer(h) - if item not in errors: - errors[item] = (val, []) + # process emits + for e in emits: + self.emit(*e, delete=False) - errors[item][1].append('%s\n in rule %s\n %s' % \ - (e, - handler.dyna_attrs['Span'], - handler.dyna_attrs['rule'])) + self.parser_state = env['parser_state'] - else: - raise e - else: - # no exception, accept emissions. - for e in emittiers: - # an error could happen here, but we assume (by contract) that - # this is not possible. - emit(*e) + return self.go() + def draw(self): + debug.draw(self) -def emit(item, val, ruleix, variables, delete): + def dynac_code(self, code): + dyna = dotdynadir / 'tmp.dyna' + out = '%s.plan.py' % dyna - print >> trace, (red % 'delete' if delete else green % 'update'), \ - '%s (val %s; curr: %s)' % (item, val, item.value) + with file(dyna, 'wb') as f: + f.write(self.parser_state) # include parser state if any. + f.write(code) - if delete: - item.aggregator.dec(val, ruleix, variables) - else: - item.aggregator.inc(val, ruleix, variables) + # TODO: grab stderr store in DynaCompilerError + if dynac(dyna, out): # stop if the compiler failed. + raise DynaCompilerError("Failed to compile %r." % dyna) - agenda[item] = 0 # everything is high priority + return out def peel(fn, item): @@ -449,100 +407,10 @@ def peel(fn, item): return item.args -def _go(): - "the main loop" - - changed.clear() - - while agenda: - item = agenda.pop_smallest() - - print >> trace - print >> trace, magenta % 'pop ', item, - - was = item.value - print >> trace, '(was: %s,' % (was,), - - try: - now = item.aggregator.fold() - except (ZeroDivisionError, TypeError) as e: - errors[item] = ('failed to aggregate %r' % item.aggregator, [e]) - # TODO: Are we sure there is never a reason to requeue this item. - continue - - print >> trace, 'now: %s)' % (now,) - - if was == now: - print >> trace, yellow % 'unchanged' - continue - - was_error = False - if item in errors: # clear the error - was_error = True - del errors[item] - - # TODO: handle `was` and `now` at the same time to avoid the two passes. - if was is not None and not was_error: - - # if `was` resulted in an error we know it didn't propagate so we - # can skip running the update dispatcher in delete mode. - - update_dispatcher(item, was, delete=True) - - item.value = now - - if now is not None: - update_dispatcher(item, now, delete=False) - - changed[item] = now - - -def go(): - try: - _go() - except KeyboardInterrupt: - pass - - def dynac(f, out): return os.system('%s/dist/build/dyna/dyna -B python -o "%s" "%s"' % (dynahome, out, f)) -def dynac_code(code, debug=False, run=True): - "skip the file." - - dyna = '/tmp/tmp.dyna' - - out = '%s.plan.py' % dyna - - with file(dyna, 'wb') as f: - f.write(globals().get('parser_state', '')) # include parser state if any. - f.write(code) - - if dynac(dyna, out): # stop if the compiler failed. - return True - - if debug: - import debug - debug.main(dyna) - - if run: - do(out) - - -def load(f): - - with file(f) as h: - print >> trace, magenta % 'Loading new code' - print >> trace, yellow % h.read() - - # TODO: loading new code should be atomic. if we fail for some reason we - # need to revert. - - # load generated code. - execfile(f, globals()) # if we want to isolate side-effects of new code - # we can pass in something insead of globals() - def dump(code, filename='/tmp/tmp.dyna'): "Write code to file." @@ -551,218 +419,7 @@ def dump(code, filename='/tmp/tmp.dyna'): return filename -def do(filename): - "Compile, load, and execute dyna code." - - assert os.path.exists(filename) - - global _initializers - _initializers = [] # XXX: do we really want to clear? - - load(filename) - - for init in _initializers: # assumes we have cleared - - def _emit(head, val, *args, **kw): - return emit(head, val, *args, delete=False, **kw) - - try: - init(emit=_emit) - except (TypeError, ZeroDivisionError) as e: - raise DynaInitializerException(e, init) - - go() - - -import cmd, readline - -class REPL(cmd.Cmd, object): - - def __init__(self, hist): - cmd.Cmd.__init__(self) - self.hist = hist - if not os.path.exists(hist): - readline.clear_history() - with file(hist, 'wb') as f: - f.write('') - readline.read_history_file(hist) - self.do_trace('off') - self.lineno = 0 - - @property - def prompt(self): - return ':- ' #% self.lineno - - def do_exit(self, _): - readline.write_history_file(self.hist) - return -1 - - def do_EOF(self, args): - "Exit on end of file character ^D." - print 'exit' - return self.do_exit(args) - - def precmd(self, line): - """ - This method is called after the line has been input but before it has - been interpreted. If you want to modify the input line before execution - (for example, variable substitution) do it here. - """ - return line - - def postcmd(self, stop, line): - self.lineno += 1 - return stop - - def do_changed(self, _): - if not changed: - return - print '=============' - for x, v in sorted(changed.items()): - print '%s := %r' % (x, v) - print - dump_errors() - - def do_chart(self, args): - if not args: - dump_charts() - else: - unrecognized = set(args.split()) - set(chart.keys()) - for f in unrecognized: - print 'unrecognized predicate', f - if unrecognized: - print 'available:\n\t' + '\t'.join(chart.keys()) - return - for f in args.split(): - print chart[f] - print - dump_errors() - - def emptyline(self): - """Do nothing on empty input line""" - pass - - def do_ip(self, _): - ip() - - def do_go(self, _): - go() - - def do_trace(self, args): - global trace - if args == 'on': - trace = sys.stdout - elif args == 'off': - trace = file(os.devnull, 'w') - else: - print 'Did not understand argument %r please use (on or off).' % args - - def do_debug(self, line): - dynac_code(line, debug=True, run=False) - - def do_query(self, line): - - if line.endswith('.'): - print "Queries don't end with a dot." - return - - query = 'out(%s) dict= _VALUE is (%s), _VALUE.' % (self.lineno, line) - - print blue % query - - self.default(query) - - for (_, results) in chart['out/1'][self.lineno,:]: - for result in results: - print result - print - - def default(self, line): - """ - Called on an input line when the command prefix is not recognized. In - that case we execute the line as Python code. - """ - line = line.strip() - if not line.endswith('.'): - print "ERROR: Line doesn't end with period." - return - try: - if dynac_code(line): # failure. - return - except AggregatorConflict as e: - print 'AggregatorConflict:', e - else: - self.do_changed('') - - def do_draw(self, _): - draw() - - def cmdloop(self, _=None): - try: - super(REPL, self).cmdloop() - except KeyboardInterrupt: - print '^C' - self.cmdloop() - except Exception as e: - readline.write_history_file(self.hist) - raise e - - -def repl(hist): - REPL(hist).cmdloop() - - -def hypergraph(): - from debug import Hypergraph - # collect edges - edges() - # create hypergraph object - g = Hypergraph() - for c in chart.values(): - for x in c.intern.values(): - for e in _edges[x]: - label, body = e - g.edge(str(x), str(label), map(str, body)) - return g - - -def draw(): - g = hypergraph() - with file('/tmp/state.html', 'wb') as f: - print >> f, """ - - - - - - """ - - x = StringIO() - dump_charts(x) - - print >> f, '
%s
' \ - % '

Charts

%s' \ - % '
%s
' \ - % x.getvalue() - - print >> f, """ -
-

Hypergraph

-%s -
-""" % g.render('/tmp/hypergraph') - - print >> f, '' - - webbrowser.open(f.name) - def main(): -# from repl import repl parser = ArgumentParser(description="The dyna interpreter!") parser.add_argument('source', help='Path to Dyna source file (or plan if --plan=true).', nargs='?') @@ -776,13 +433,15 @@ def main(): argv = parser.parse_args() - global trace + interp = Interpreter() + if argv.trace == 'stderr': - trace = sys.stderr + interp.trace = sys.stderr elif argv.trace == 'stdout': - trace = sys.stdout + interp.trace = sys.stdout else: - trace = file(argv.trace, 'wb') + interp.trace = file(argv.trace, 'wb') + if argv.source: @@ -796,26 +455,25 @@ def main(): plan = "%s.plan.py" % argv.source dynac(argv.source, plan) - do(plan) + interp.do(plan) if argv.output: if argv.output == "-": - dump_charts(sys.stdout) + interp.dump_charts(sys.stdout) else: with file(argv.output, 'wb') as f: - dump_charts(f) + interp.dump_charts(f) else: - dump_charts() + interp.dump_charts() if argv.interactive: - repl(hist = argv.source + '.hist') + interp.repl(hist = argv.source + '.hist') else: - repl(hist = '/tmp/dyna.hist') + interp.repl(hist = '/tmp/dyna.hist') if argv.draw: - draw() - + interp.draw() if __name__ == '__main__': diff --git a/src/Dyna/Backend/Python/repl.py b/src/Dyna/Backend/Python/repl.py index a022e75..460bd5e 100644 --- a/src/Dyna/Backend/Python/repl.py +++ b/src/Dyna/Backend/Python/repl.py @@ -1,6 +1,134 @@ -# -# - TODO: Put REPL in it's own module. -# -# - timv: I noticed some quirky behavior with global variables when REPL was -# in a different module from interpreter so I switched it back temporarily. -# +import os, sys +import cmd, readline +from utils import blue, yellow, green, magenta, ip + +from interpreter import AggregatorConflict +import debug + +class REPL(cmd.Cmd, object): + + def __init__(self, interp, hist): + self.interp = interp + cmd.Cmd.__init__(self) + self.hist = hist + if not os.path.exists(hist): + readline.clear_history() + with file(hist, 'wb') as f: + f.write('') + readline.read_history_file(hist) + self.do_trace('off') + self.lineno = 0 + + @property + def prompt(self): + return ':- ' #% self.lineno + + def do_exit(self, _): + readline.write_history_file(self.hist) + return -1 + + def do_EOF(self, args): + "Exit on end of file character ^D." + print 'exit' + return self.do_exit(args) + + def precmd(self, line): + """ + This method is called after the line has been input but before it has + been interpreted. If you want to modify the input line before execution + (for example, variable substitution) do it here. + """ + return line + + def postcmd(self, stop, line): + self.lineno += 1 + return stop + + def do_chart(self, _): +# if not args: + self.interp.dump_charts() +# else: +# unrecognized = set(args.split()) - set(interp.chart.keys()) +# for f in unrecognized: +# print 'unrecognized predicate', f +# if unrecognized: +# print 'available:\n\t' + '\t'.join(chart.keys()) +# return +# for f in args.split(): +# print chart[f] +# print +# interp.dump_errors() + + def emptyline(self): + """Do nothing on empty input line""" + pass + + def do_ip(self, _): + ip() + + def do_trace(self, args): + if args == 'on': + self.interp.trace = sys.stdout + elif args == 'off': + self.interp.trace = file(os.devnull, 'w') + else: + print 'Did not understand argument %r please use (on or off).' % args + +# def do_debug(self, line): +# dynac_code(line, debug=True, run=False) + + def do_query(self, line): + + if line.endswith('.'): + print "Queries don't end with a dot." + return + + query = 'out(%s) dict= _VALUE is (%s), _VALUE.' % (self.lineno, line) + + print blue % query + + self.default(query) + + for (_, results) in self.interp.chart['out/1'][self.lineno,:]: + for result in results: + print result + print + + def default(self, line): + """ + Called on an input line when the command prefix is not recognized. In + that case we execute the line as Python code. + """ + line = line.strip() + if not line.endswith('.'): + print "ERROR: Line doesn't end with period." + return + try: + src = self.interp.dynac_code(line) # failure. + + changed = self.interp.do(src) + + except AggregatorConflict as e: + print 'AggregatorConflict:', e + + else: + if not changed: + return + print '=============' + for x, v in sorted(changed.items()): + print '%s := %r' % (x, v) + print + self.interp.dump_errors() + + def do_draw(self, _): + self.interp.draw() + + def cmdloop(self, _=None): + try: + super(REPL, self).cmdloop() + except KeyboardInterrupt: + print '^C' + self.cmdloop() + except Exception as e: + readline.write_history_file(self.hist) + raise e diff --git a/src/Dyna/Backend/Python/utils.py b/src/Dyna/Backend/Python/utils.py index 5f2922f..aa6d2fc 100644 --- a/src/Dyna/Backend/Python/utils.py +++ b/src/Dyna/Backend/Python/utils.py @@ -152,15 +152,16 @@ def parse_attrs(fn): return attrs -def rule_source(span): +def rule_source(span, src=None): """ Utility for retrieving source code for Parsec error message. """ [(filename, bl, bc, el, ec)] = re.findall(r'(.*):(\d+):(\d+)-\1:(\d+):(\d+)', span) (bl, bc, el, ec) = map(int, [bl, bc, el, ec]) - with file(filename) as f: - src = f.read() + if not src: + with file(filename) as f: + src = f.read() lines = [l + '\n' for l in src.split('\n')] -- 2.50.1