From 3f5af0c959bff717e7d69fb35e3b1dad839a613a Mon Sep 17 00:00:00 2001 From: Tim Vieira Date: Sun, 4 Aug 2013 10:48:43 -0400 Subject: [PATCH] Refactor - put compiler interface in separate module. Unified duplicate parser state Cleaned up imports path.py is back as a dep since IPython's version is too old. --- src/Dyna/Backend/Python/aggregator.py | 4 +- src/Dyna/Backend/Python/chart.py | 2 +- src/Dyna/Backend/Python/config.py | 2 +- src/Dyna/Backend/Python/debug.py | 4 +- src/Dyna/Backend/Python/dyna_doctest.py | 10 +- src/Dyna/Backend/Python/dynac.py | 120 +++++++++++++++++++++++ src/Dyna/Backend/Python/interpreter.py | 124 ++++++++---------------- src/Dyna/Backend/Python/load/matrix.py | 2 +- src/Dyna/Backend/Python/load/pickled.py | 2 +- src/Dyna/Backend/Python/load/sexpr.py | 3 +- src/Dyna/Backend/Python/load/tsv.py | 2 +- src/Dyna/Backend/Python/main.py | 2 +- src/Dyna/Backend/Python/term.py | 5 +- src/Dyna/Backend/Python/utils.py | 59 +---------- 14 files changed, 179 insertions(+), 162 deletions(-) create mode 100644 src/Dyna/Backend/Python/dynac.py diff --git a/src/Dyna/Backend/Python/aggregator.py b/src/Dyna/Backend/Python/aggregator.py index e27a08e..9794891 100644 --- a/src/Dyna/Backend/Python/aggregator.py +++ b/src/Dyna/Backend/Python/aggregator.py @@ -11,6 +11,7 @@ import operator from collections import Counter from utils import drepr, _repr, user_vars, isbool, true, false from errors import AggregatorError +from stdlib import todyna class NoAggregatorError(Exception): @@ -94,7 +95,6 @@ class DictEquals(BAggregator): def fold(self): if not self.empty(): - from stdlib import todyna return todyna([b + (('$val', v),) for (v, b), cnt in self.iteritems() if cnt > 0]) @@ -188,7 +188,6 @@ class and_equals(BAggregator): class set_equals(BAggregator): def fold(self): - from stdlib import todyna s = {x for x, m in self.iteritems() if m > 0} if len(s): return todyna(s) @@ -197,7 +196,6 @@ class set_equals(BAggregator): class bag_equals(BAggregator): def fold(self): if any(m > 0 for m in self.itervalues()): - from stdlib import todyna x = list(Counter(self).elements()) x.sort() return todyna(x) diff --git a/src/Dyna/Backend/Python/chart.py b/src/Dyna/Backend/Python/chart.py index 1c09ea1..505ff2d 100644 --- a/src/Dyna/Backend/Python/chart.py +++ b/src/Dyna/Backend/Python/chart.py @@ -19,7 +19,7 @@ class Chart(object): def set_aggregator(self, agg): self.agg_name = agg for item in self.intern.values(): - assert item.value is None # shouldn't change aggregator when non-null. + assert item.value is None, [item, item.value, item.aggregator] # shouldn't change aggregator when non-null. item.aggregator = self.new_aggregator(item) def __repr__(self): diff --git a/src/Dyna/Backend/Python/config.py b/src/Dyna/Backend/Python/config.py index 4695c0a..c78da72 100644 --- a/src/Dyna/Backend/Python/config.py +++ b/src/Dyna/Backend/Python/config.py @@ -1,5 +1,5 @@ import os -from IPython.external.path import path +from utils import path dotdynadir = path('~/.dyna').expand() if not dotdynadir.exists(): diff --git a/src/Dyna/Backend/Python/debug.py b/src/Dyna/Backend/Python/debug.py index 8075c0c..7ab3b4e 100644 --- a/src/Dyna/Backend/Python/debug.py +++ b/src/Dyna/Backend/Python/debug.py @@ -6,9 +6,8 @@ normalization process. import re, os, shutil, webbrowser from collections import defaultdict -from utils import dynac, read_anf +from utils import read_anf, path from config import dynahome -from IPython.external.path import path from warnings import warn try: @@ -267,6 +266,7 @@ def main(dynafile, browser=True): print >> html, '
' print >> html, '
' + from dynac import dynac dynac(dynafile, out = d / 'plan', anf = d / 'anf', diff --git a/src/Dyna/Backend/Python/dyna_doctest.py b/src/Dyna/Backend/Python/dyna_doctest.py index a32ebc7..d280c1d 100644 --- a/src/Dyna/Backend/Python/dyna_doctest.py +++ b/src/Dyna/Backend/Python/dyna_doctest.py @@ -3,7 +3,7 @@ import re, sys, traceback from interpreter import Interpreter from repl import REPL from cStringIO import StringIO -from utils import bold, red, green, yellow, strip_comments +from utils import bold, red, green, yellow def diff(expect, got): @@ -32,10 +32,16 @@ def extract(code): yield '\n'.join(cmd).strip(), '\n'.join(expect).strip() +def strip_comments(src): + return re.compile('%.*$', re.MULTILINE).sub('', src) + +def remove_color(x): + return re.sub('\033\[[0-9;]+m', '', x) + def clean(x): # remove whitespace at end of line # remove ansi color codes - return re.compile('(\s*)$', re.MULTILINE).sub('', re.sub('\033\[\d+m', '', strip_comments(x)).strip()) + return re.compile('(\s*)$', re.MULTILINE).sub('', remove_color(strip_comments(x))).strip() def run(code, out=None): diff --git a/src/Dyna/Backend/Python/dynac.py b/src/Dyna/Backend/Python/dynac.py new file mode 100644 index 0000000..4816ee5 --- /dev/null +++ b/src/Dyna/Backend/Python/dynac.py @@ -0,0 +1,120 @@ +""" +Interface to compiler. + +TODO: + - read parser state + - manage temp dir/files + - anf + - recompile rule should give same index + +""" + +import re, os +from subprocess import Popen, PIPE + +from utils import path +from config import dynahome, dotdynadir +from hashlib import sha1 +from errors import DynaCompilerError +from utils import hide_ugly_filename, span_to_src + + +def dynac(f, out, anf=None, compiler_args=()): + """ + Run compiler on file, ``f``, write results to ``out``. Raises + ``DynaCompilerError`` on failure. + """ + + f = path(f) + if not f.exists(): + raise DynaCompilerError("File '%s' does not exist." % f) + + cmd = ['%s/dist/build/dyna/dyna' % dynahome, + '-B', 'python', '-o', out, f] + + if anf is None: + cmd += ['--dump-anf=' + out + '.anf'] + else: + cmd += ['--dump-anf=' + anf] + + cmd += compiler_args + + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + + stdout, stderr = p.communicate() + if p.returncode: + assert not stdout.strip(), [stdout, stderr] + stderr = hide_ugly_filename(stderr, lambda m: '\n %s\n' % span_to_src(m.group(0))) + raise DynaCompilerError(stderr, f) + + +class Compiler(object): + + def __init__(self): + self.files = [] + self.tmp = tmp = (dotdynadir / 'tmp' / str(os.getpid())) + if tmp.exists(): + tmp.rmtree() + tmp.makedirs_p() + + def dynac(self, filename): + """ + Compile a file full of dyna code. Note: this routine does not pass along + parser_state. + """ + filename = path(filename) + self.files.append(filename) + out = self.tmp / filename.read_hexhash('sha1') + '.plan.py' + #out = filename + '.plan.py' + self.files.append(out) + dynac(filename, out) + return out + + def dynac_code(self, code, pstate): + "Compile a string of dyna code." + x = sha1() + x.update(pstate) + x.update(code) + dyna = self.tmp / ('%s.dyna' % x.hexdigest()) + with file(dyna, 'wb') as f: + f.write(pstate) # include parser state if any. + f.write(code) + return self.dynac(dyna) + + @staticmethod + def parser_state(bc, rix, agg, other): + # TODO: this is pretty hacky. XREF:parser-state + lines = [':-ruleix %d.' % rix] + for fn in bc: + [(fn, arity)] = re.findall('(.*)/(\d+)', fn) + lines.append(":-backchain '%s'/%s." % (fn, arity)) + for fn, agg in agg.items(): + [(fn, arity)] = re.findall('(.*)/(\d+)', fn) + if agg is not None: + lines.append(":-iaggr '%s'/%s %s." % (fn, arity, agg)) + lines.extend(':-%s %s.' % (k,v) for k,v in other) + lines.append('\n') + return '\n'.join(lines) + + @staticmethod + def read_parser_state(parser_state): + """ + TODO: This is pretty hacky we should have the codegen produce something + easier to serialize/modify/unserialize. XREF:parser-state. + """ + backchain = set() + ruleix = None + iaggr = {} + other = [] + for k, v in re.findall('^:-\s*(\S+) (.*?)\s*\.$', parser_state, re.MULTILINE): + if k == 'backchain': + [(fn, arity)] = re.findall("'(.*?)'/(\d+)", v) + backchain.add('%s/%s' % (fn, arity)) + elif k == 'iaggr': + [(fn, arity, agg)] = re.findall("'(.*?)'/(\d+)\s*(.*)", v) + iaggr['%s/%s' % (fn, arity)] = agg + elif k == 'ruleix': + ruleix = int(v) + else: + other.append((k,v)) + return backchain, ruleix, iaggr, other diff --git a/src/Dyna/Backend/Python/interpreter.py b/src/Dyna/Backend/Python/interpreter.py index 4f2fdda..d6a3eb8 100644 --- a/src/Dyna/Backend/Python/interpreter.py +++ b/src/Dyna/Backend/Python/interpreter.py @@ -2,19 +2,17 @@ import re, os, sys, imp, traceback from collections import defaultdict -from hashlib import sha1 -from IPython.external.path import path from term import Term, Cons, Nil, MapsTo, Error from chart import Chart -from utils import red, parse_attrs, dynac, read_anf, strip_comments, _repr, \ - hide_ugly_filename, true, false, parse_parser_state, magenta, indent +from utils import red, parse_attrs, read_anf, _repr, hide_ugly_filename, \ + true, false, magenta, indent, path from prioritydict import prioritydict from config import dotdynadir from errors import rule_error_context, AggregatorError, DynaCompilerError from stdlib import todyna - +from dynac import Compiler #sys.setrecursionlimit(10000) @@ -78,40 +76,13 @@ def none(): class Interpreter(object): - def parser_state(self, ruleix=None): - # TODO: this is pretty hacky. XREF:parser-state - bc, _rix, agg, other = self.pstate - if ruleix is None: - if not self.rules: - rix = 0 - else: - rix = max(self.rules) + 1 # next available - rix = max(rix, _rix) - else: - rix = ruleix # override rule index from pstate - lines = [':-ruleix %d.' % rix] - for fn in bc: - [(fn, arity)] = re.findall('(.*)/(\d+)', fn) - lines.append(":-backchain '%s'/%s." % (fn, arity)) - for fn, agg in agg.items(): - [(fn, arity)] = re.findall('(.*)/(\d+)', fn) - lines.append(":-iaggr '%s'/%s %s." % (fn, arity, agg)) - lines.extend(':-%s %s.' % (k,v) for k,v in other) - lines.append('\n') - return '\n'.join(lines) - - def set_parser_state(self, x): - self.pstate = (bc, _rix, _agg, _other) = x - for fn in bc: - if fn not in self._gbc: # new backchain declaration - for r in self.rule_by_head[fn]: - self.needs_recompile(r) - def __init__(self): - # declarations + self.compiler = Compiler() + # parser state self.agg_name = defaultdict(none) - self.pstate = (set(), 0, {}, []) - self.files = [] + self.bc = set() + self.other = [] + self.ruleix = 0 # rules self.rules = {} self.updaters = defaultdict(list) @@ -120,13 +91,9 @@ class Interpreter(object): self.agenda = prioritydict() self.chart = foo(self.agg_name) self.error = {} + self.changed = {} # misc self.time_step = 0 - # interpretor needs a place for it's temporary files. - self.tmp = tmp = (dotdynadir / 'tmp' / str(os.getpid())) - if tmp.exists(): - tmp.rmtree() - tmp.makedirs_p() # coarsening of the program shows which rules might depend on each other self.coarse_deps = defaultdict(set) self.rule_by_head = defaultdict(set) @@ -304,7 +271,6 @@ class Interpreter(object): emits.append((item, val, ruleix, variables, False)) errors = [] - for handler in self._gbc[item.fn]: try: handler(*item.args, emit=t_emit) @@ -348,7 +314,7 @@ class Interpreter(object): anf[x.ruleix] = x # update parser state - self.set_parser_state(parse_parser_state(env.parser_state)) + self.set_parser_state(env.parser_state) for k, v in env.agg_decl.items(): self.new_fn(k, v) @@ -367,18 +333,29 @@ class Interpreter(object): return new_rules + def set_parser_state(self, x): + (bc, _rix, _agg, _other) = self.compiler.read_parser_state(x) + # update misc parser state + self.other = _other + # update backchain predicates + for fn in bc: + self.bc.add(fn) + if fn not in self._gbc: # new backchain declaration + for r in self.rule_by_head[fn]: + self.needs_recompile(r) + # update ruleix + if not self.rules: + self.ruleix = _rix + else: + self.ruleix = max(max(self.rules) + 1, _rix) # next available + # update fn->aggregator map + for fn, agg in _agg.items(): + self.new_fn(fn, agg) + def recompile_rule(self, r): "returns a plan, it's up to you to retract the old rule and load the plan" - pstate = self.parser_state(ruleix=r.index) # override ruleix - code = r.src - x = sha1() - x.update(pstate) - x.update(code) - dyna = self.tmp / ('%s.dyna' % x.hexdigest()) - with file(dyna, 'wb') as f: - f.write(pstate) - f.write(code) - return self.dynac(dyna) + pstate = self.compiler.parser_state(self.bc, r.index, self.agg_name, self.other) # override ruleix + return self.compiler.dynac_code(r.src, pstate) def needs_recompile(self, r): self.retract_rule(r.index) # clears errors @@ -388,17 +365,17 @@ class Interpreter(object): # run to fixed point. while self.recompile: success = set() - failed = set() + failure = set() for r in list(self.recompile): try: r.plan = self.recompile_rule(r) except DynaCompilerError as e: - failed.add(r) + failure.add(r) self.set_error(r, e) else: success.add(r) self.clear_error(r) - self.recompile = failed + self.recompile = failure if not success: break for r in success: @@ -456,7 +433,7 @@ class Interpreter(object): self.rules[index] = rule = Rule(index) rule.span = hide_ugly_filename(parse_attrs(init or query)['Span']) - rule.src = strip_comments(parse_attrs(init or query)['rule']) + rule.src = parse_attrs(init or query)['rule'] rule.anf = anf rule.head_fn = head_fn @@ -473,13 +450,6 @@ class Interpreter(object): # fix dependents if head_fn not in self._gbc: - - # quick monkey patch assertion. - def monkey(_, _args): - assert False, '__getitem__ should never be called because' \ - ' `%s` should be backchained' % head_fn - self.chart.__getitem__ = monkey - # retract and replan rules dependent on this predicate which is # now backchained. for d in self.rule_dep[head_fn]: @@ -583,8 +553,6 @@ class Interpreter(object): self.chart[rule.head_fn].set_aggregator(None) if rule.head_fn in self.agg_name: del self.agg_name[rule.head_fn] - if rule.head_fn in self.pstate[2]: - del self.pstate[2][rule.head_fn] # remove fn aggr def from parser state return self.changed @@ -621,28 +589,12 @@ class Interpreter(object): Compile a file full of dyna code. Note: this routine does not pass along parser_state. """ - filename = path(filename) - self.files.append(filename) - - # TODO: crashlogs/amareshj:2013-07-01:22:26:54:13412.log -- file doesn't exist. - - out = self.tmp / filename.read_hexhash('sha1') + '.plan.py' - #out = filename + '.plan.py' - self.files.append(out) - dynac(filename, out) - return out + return self.compiler.dynac(filename) def dynac_code(self, code): "Compile a string of dyna code." - pstate = self.parser_state() - x = sha1() - x.update(pstate) - x.update(code) - dyna = self.tmp / ('%s.dyna' % x.hexdigest()) - with file(dyna, 'wb') as f: - f.write(pstate) # include parser state if any. - f.write(code) - return self.dynac(dyna) + pstate = self.compiler.parser_state(self.bc, self.ruleix, self.agg_name, self.other) + return self.compiler.dynac_code(code, pstate) #___________________________________________________________________________ # Routines for showing things to the user. diff --git a/src/Dyna/Backend/Python/load/matrix.py b/src/Dyna/Backend/Python/load/matrix.py index 4b28443..dad044d 100644 --- a/src/Dyna/Backend/Python/load/matrix.py +++ b/src/Dyna/Backend/Python/load/matrix.py @@ -1,5 +1,5 @@ import re -from IPython.external.path import path +from utils import path class matrix(object): diff --git a/src/Dyna/Backend/Python/load/pickled.py b/src/Dyna/Backend/Python/load/pickled.py index d7f8f28..a1d58f6 100644 --- a/src/Dyna/Backend/Python/load/pickled.py +++ b/src/Dyna/Backend/Python/load/pickled.py @@ -7,7 +7,7 @@ TODO: Can we merge a pickled interpreter into an existing one? """ import cPickle -from IPython.external.path import path +from utils import path class pickled(object): diff --git a/src/Dyna/Backend/Python/load/sexpr.py b/src/Dyna/Backend/Python/load/sexpr.py index f701ffa..fbfcefd 100644 --- a/src/Dyna/Backend/Python/load/sexpr.py +++ b/src/Dyna/Backend/Python/load/sexpr.py @@ -1,7 +1,6 @@ from cStringIO import StringIO -from utils import parse_sexpr +from utils import parse_sexpr, path from stdlib import todyna -from IPython.external.path import path class sexpr(object): diff --git a/src/Dyna/Backend/Python/load/tsv.py b/src/Dyna/Backend/Python/load/tsv.py index 761385a..78211f6 100644 --- a/src/Dyna/Backend/Python/load/tsv.py +++ b/src/Dyna/Backend/Python/load/tsv.py @@ -5,7 +5,7 @@ TODO: option for strict number of columns. import re from utils import true -from IPython.external.path import path +from path import path class tsv(object): diff --git a/src/Dyna/Backend/Python/main.py b/src/Dyna/Backend/Python/main.py index e2949db..fc8aba3 100644 --- a/src/Dyna/Backend/Python/main.py +++ b/src/Dyna/Backend/Python/main.py @@ -1,5 +1,5 @@ import argparse -from IPython.external.path import path +from utils import path from errors import DynaCompilerError from errors import crash_handler from interpreter import Interpreter diff --git a/src/Dyna/Backend/Python/term.py b/src/Dyna/Backend/Python/term.py index 404a0ce..f46f625 100644 --- a/src/Dyna/Backend/Python/term.py +++ b/src/Dyna/Backend/Python/term.py @@ -1,5 +1,4 @@ from utils import _repr -from aggregator import NoAggregator # TODO: codegen should output a derived Term instance for each functor @@ -54,7 +53,7 @@ class Cons(NoIntern, Term): self.head = head self.tail = tail Term.__init__(self, 'cons/2', (head, tail)) - self.aggregator = NoAggregator + self.aggregator = None self.aslist = [self.head] + self.tail.aslist def __cmp__(self, other): @@ -89,7 +88,7 @@ class _Nil(Term): def __init__(self): Term.__init__(self, 'nil/0', ()) - self.aggregator = NoAggregator + self.aggregator = None self.aslist = [] def __repr__(self): diff --git a/src/Dyna/Backend/Python/utils.py b/src/Dyna/Backend/Python/utils.py index 47fa5a5..aa2f70a 100644 --- a/src/Dyna/Backend/Python/utils.py +++ b/src/Dyna/Backend/Python/utils.py @@ -1,7 +1,6 @@ import re +from path import path # used by other modules from IPython.frontend.terminal.embed import InteractiveShellEmbed -from IPython.external.path import path -from subprocess import Popen, PIPE from config import dynahome, dotdynadir from collections import namedtuple, defaultdict from cStringIO import StringIO @@ -14,27 +13,6 @@ def groupby(key, data): return dict(g) -# TODO: This is pretty hacking we should have the codegen produce something -# easier to serialize/modify/unserialize. XREF:parser-state -def parse_parser_state(parser_state): - backchain = set() - ruleix = None - iaggr = {} - other = [] - for k, v in re.findall('^:-\s*(\S+) (.*?)\s*\.$', parser_state, re.MULTILINE): - if k == 'backchain': - [(fn, arity)] = re.findall("'(.*?)'/(\d+)", v) - backchain.add('%s/%s' % (fn, arity)) - elif k == 'iaggr': - [(fn, arity, agg)] = re.findall("'(.*?)'/(\d+)\s*(.*)", v) - iaggr['%s/%s' % (fn, arity)] = agg - elif k == 'ruleix': - ruleix = int(v) - else: - other.append((k,v)) - return backchain, ruleix, iaggr, other - - def indent(x, indent=''): if isinstance(x, basestring): return re.compile('^(.*)$', flags=re.MULTILINE).sub(indent + r'\1', x) @@ -119,41 +97,6 @@ bold = '\033[1m%s\033[0m' #from fabulous.color import red, green, yellow, blue, magenta, cyan, white, bold, underline -_comments = re.compile('%.*$', re.MULTILINE) -def strip_comments(src): - return _comments.sub('', src).strip() - - -def dynac(f, out, anf=None, compiler_args=()): - """ - Run compiler on file, ``f``, write results to ``out``. Raises - ``DynaCompilerError`` on failure. - """ - from errors import DynaCompilerError - - f = path(f) - if not f.exists(): - raise DynaCompilerError("File '%s' does not exist." % f) - - cmd = ['%s/dist/build/dyna/dyna' % dynahome, - '-B', 'python', '-o', out, f] - - if anf is None: - cmd += ['--dump-anf=' + out + '.anf'] - else: - cmd += ['--dump-anf=' + anf] - - cmd += compiler_args - - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - - stdout, stderr = p.communicate() - if p.returncode: - assert not stdout.strip(), [stdout, stderr] - stderr = hide_ugly_filename(stderr, lambda m: '\n %s\n' % span_to_src(m.group(0))) - raise DynaCompilerError(stderr, f) - - def hide_ugly_filename(x, replacement=''): p = dotdynadir + '[a-z0-9/.]+\.dyna\S*' return re.sub(p, replacement, x) -- 2.50.1