From 0dba0124d1fc078cc291d45e1c40c8a6b56bb088 Mon Sep 17 00:00:00 2001 From: Nathaniel Wesley Filardo Date: Wed, 11 Feb 2015 00:02:56 -0500 Subject: [PATCH] Initial checkin --- .gitignore | 1 + COPYING | 26 ++++++ README.rst | 233 +++++++++++++++++++++++++++++++++++++++++++++++ grade.pl | 192 ++++++++++++++++++++++++++++++++++++++ make-skeleton.pl | 39 ++++++++ 5 files changed, 491 insertions(+) create mode 100644 .gitignore create mode 100644 COPYING create mode 100644 README.rst create mode 100644 grade.pl create mode 100644 make-skeleton.pl diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a01ee28 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.*.swp diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..b9d2113 --- /dev/null +++ b/COPYING @@ -0,0 +1,26 @@ +Copyright (c) 2015, Nathaniel Wesley Filardo +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of any entity other than the author. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..4ebd89f --- /dev/null +++ b/README.rst @@ -0,0 +1,233 @@ +########################################## +GRADE Reporting And Definition Environment +########################################## + +Inspired by the grading infrastructure developed by David Eckhardt at +Carnegie Mellon University for http://www.cs.cmu.edu/~410/ , GRADE is a +lightweight, file-based, *directive*-based approach to managing the work of +grading students in a course. It uses a minimal markup format for both its +machine-readable scoring rubric and for grader data files. + +Student View +############ + +Students will get back tastefully formatted grade files. A very short +example looks something like this:: + + Packaging: [10/10] + + Grader comments: + + Everything looks great here. Thanks for the very informative README! + + Functionality Tests: [30/40] + + (-6) + The buffer-passing test seems to mangle bytes on occasion. + + (-4) + There is a minor problem with the buffer-passing test output + when given an unusually long input string. + + Grader comments: + + Both of these test failures occur because ... + + Overall: [40/50] (80.0%) + +Basic Grader Use +################ + +The instructor will have prepared a ``skeleton`` file which will look +something like this:: + + # Basic features of the handin + @packaging + #:ftbfs_all + #:missing_readme + #:readme_no_commentary + #:readme_no_instructions + #:tarball_directory + #:missing_make + + $BEGIN_COMMENTS + + $END_COMMENTS + + # Automated test result section + @tests + # Un-comment the appropriate directive for each test failed. + #:ftbfs_all + #:simple_test + #:more_interesting_test + #:more_interesting_test_minor + #:test_everything + + $BEGIN_COMMENTS + + $END_COMMENTS + +etc. The graders should copy this skeleton and mutate it appropriately for +each assignment they grade. The example grade output above might have come +from a grading data file like this:: + + @packaging + $BEGIN_COMMENTS + + Everything looks great here. Thanks for the very informative README! + + $END_COMMENTS + + @tests + :simple_test + :more_interesting_test_minor + + $BEGIN_COMMENTS + + Both of these test failures occur because ... + + $END_COMMENTS + +In more detail: + +* Sections are separated by lines beginning with ``@``. + +* Blocks surrounded by ``$BEGIN_COMMENTS``/``$END_COMMENTS`` will be copied + into the appropriate section of the student report under the heading of + ``Grader comments``. + +* **outside** the comment blocks, all characters on a line after a ``#`` + mark will be ignored. Inside comment blocks, all bytes will be copied + over verbatim (modulo an indentation prefix for pretty-printing). + +* Text and scores associated with each un-commented ``:``-line (which are + defined by the instructor) will also be emitted to the correct section. + +Advanced Handling +----------------- + +Usually with permission of the instructor (just so there are no surprises), +there are two additional directives that can be used within a section's +directives (not in a comment block): + +* ``!pointsadj N`` will adjust the final score by ``N`` (though not lower + than a total of 0 or more than the section maximum) and should be used + only sparingly; it is, in general, much better to ask the instructor to + define a new flag for your case than to make many ad-hoc adjustments. + +* ``!pointsset N`` will override the automated score computation within a + section (to the value ``N``, though the score cannot be set less than 0 or + more than the section maximum) and should be used very rarely. + +These commands may not be mixed in a section. Multiple ``!pointsadj`` +commands will be understood. + +Instructor Use +############## + +Constructing a Rubric +--------------------- + +A rubric file, typically called ``defines.conf``, consists of a number of +sections. Each section has a name as well as some other parameters, and +contains the definition of the flags seen in the grader data. + +Sections are introduced with ``@``-lines, like in the grading data, except +that here, they take arguments:: + + @section-name type max extra friendly-name + +where + +* ``section-name`` is the short name as used in the grade data. It may + not contain whitespace. + +* ``type`` indicates to the ``grade.pl`` script how to interpret this + section. + + * Type ``0`` defines a section of define flags whose invoked scores are + simply summed. ``extra`` is ignored for this type. This is the only + type defined at the moment and it must be used in rubric files. + +* ``max`` denotes the maximum (and initial) number of points in this section + +* ``friendly-name`` is the section heading as presented to students. It may + contain spaces, and is in fact the remainder of the @ line. + +Within sections, each flag definition takes the form :: + + :flag-name score-modifier + Commentary paragraph 1 + paragraph 1 line 2 + + paragraph 2 + . + +``flag-name`` is the name of the flag used in the grading data files. +``score-modifier`` is a number (as understood by Perl) and is almost always +negative (i.e., beginning with a ``-``), but positive numbers are understood +for some form of extra credit. (Note that the script will refuse to set a +score higher than the section maximum.) + +Text between the line beginning with ``:`` and the dot on a line by itself +will be copied into student grade reports whenever the flag is given in a +grade data file. In many cases, there are many conditions that may merit +the use of the same flag, and students will benefit from additional feedback +about exactly what offense has been committed; historically, rather than +introduce many flags, a simple "see the note below" in the prose has +sufficed. Of course, this is up to judgement and taste. + +Lines that begin with ``#`` and not ``#!`` will be copied into the skeleton. +Lines beginning with ``#!`` will be ignored entirely. + +Continuing the example above, the corresponding ``defines.conf`` contains, +among other defines :: + + #! This line will be ignored + # Basic features of the handin + @packaging 0 10 - Packaging + :ftbfs_all -10 + The submission failed to compile. + . + + #! ... + + # Automated test result section + @tests 0 40 - Functionality Tests + # Un-comment the appropriate directive for each test failed. + + #! ... + + :simple_test -6 + The buffer-passing test seems to mangle bytes on occasion. + . + + #! ... + + :more_interesting_test_minor -4 + There is a minor problem with the buffer-passing test output + when given an unusually long input string. + . + + #! ... + +Generating a Skeleton +--------------------- + +Given a rubric, typically called ``defines.conf``, one can produce a +skeletal grading file by :: + + ./make-skeleton.pl < defines.conf > skeleton + +Producing grade results +----------------------- + +Given a rubric and a grader data file, ``student.data``, one runs :: + + ./grade.pl defines.conf < student.data + +to obtain the pretty-printed report and numeric score result. + +It is easy to adjust the weights of different flags and sections by simply +altering the values in ``defines.conf`` file and re-running the ``grade.pl`` +script. diff --git a/grade.pl b/grade.pl new file mode 100644 index 0000000..db8c7a3 --- /dev/null +++ b/grade.pl @@ -0,0 +1,192 @@ +#!/usr/bin/env perl + +# Copyright (c) 2015, Nathaniel Wesley Filardo +# All rights reserved. +# See COPYING for details. + +# A grading script! Compatible in spirit with the CMU Operating System +# grading infrastructure, if not in actual thought or deed quite yet. +# +# See README.rst for syntax and usage. + +use strict; +use warnings; + +sub max($$) { return @_[$_[0] < $_[1]]; } +sub min($$) { return @_[$_[0] > $_[1]]; } + +open (DEFINES,($ARGV[0] or die "No defines file?")) + or die "Could not open DEFINES file $ARGV[0]: $!"; + +my $ptssum = 0; +my $ptsmax = 0; +# Section state {{{ +my $sectionoffset; +my $secname; +my $secty; +my $secmax; +my $secextra; +my $pointsadj; +my $pointsset; +my $dings; +my $dingsum; +my $dingtext; +my $commenttext; + +sub initSection() { + $sectionoffset = undef; + $secname = undef; + $secty = undef; + $secmax = undef; + $secextra = undef; + $pointsadj = undef; + $pointsset = undef; + $dings = 0; + $dingsum = 0; + $dingtext = [ ]; + $commenttext = [ ]; +} + +initSection(); +# }}} +sub untilDefinesDot($) { # {{{ + my ($cb) = @_; + while(my $line = ) { + chomp $line; + last if $line =~ /^\.$/; + &$cb($line) ; + } +} # }}} +sub findDefinesSection($) { # {{{ + my ($newsecname) = @_; + seek DEFINES, 0, 0; + my $dline; + while($dline = ) { + if ($dline =~ /^\@$newsecname\s+(.*)$/) { + $sectionoffset = tell(DEFINES); + my $secargs = $1; + if ($secargs =~ /^(\S+)\s+(\S+)\s+(\S+)\s+(.*)$/) { + ($secty, $secmax, $secextra, $secname) = ($1,$2,$3,$4); + } else { die "Malformed section line"; } + return; + } + if ($dline =~ /^:/) { + untilDefinesDot(sub ($) { return; }); + } + } +} # }}} +sub doDing($) { # {{{ + my ($ding) = @_; + + $dings += 1; + + seek DEFINES, $sectionoffset, 0; + while(my $line = ) { + last if ($line =~ /^@/); + if ($line =~ /^:$ding\s+(.*)$/) { + $dingsum += $1; + push @$dingtext, "($1)"; + untilDefinesDot(sub ($) { push @$dingtext, @_; }); + push @$dingtext, ""; + return; + } + } + + die "$ding not defined?\n"; +} # }}} +sub finishSection() { # {{{ + + # Compute score + my $score; + + if ($secty == 0) { + $score = $secmax + ($dingsum or 0) + ($pointsadj or 0); + } else { die "Unknown section type $secty for $secname\n"; } + + if (defined $pointsset) { + $score = $pointsset; + } + + die if not defined $score; + + $score = max(0, min($secmax, $score)); + + print "$secname: [$score/$secmax]\n\n"; + print "", (join "\n", map { " $_" } @$dingtext), "\n"; + + if (scalar @$commenttext != 0) { + print " Grader comments: \n\n"; + print "", (join "\n", map { " $_" } @$commenttext), "\n"; + } + + print "\n\n"; + + $ptssum += $score; + $ptsmax += $secmax; + + initSection(); +} # }}} +sub finishOverall() { # {{{ + + print "Overall: [$ptssum/$ptsmax] ", + (sprintf ("(%.1f%%)", (100*$ptssum/$ptsmax))), + "\n"; + +} # }}} +# Main dispatch {{{ +while(my $line = ) { + $line =~ s/#.*$//; # trim line comments + next if ($line =~ /^\s*$/) ; + chomp $line; + + # @section directive? + if ($line =~ /^\@(\S+)\s*$/) { + my $newsecname = $1; + finishSection() if defined $secname; + + # Find this section in the defines file + my $dline = findDefinesSection($newsecname); + + die "Unknown section $newsecname" if not defined $secname; + } + + # :define directive? + elsif ($line =~ /^:(\S+)\s*$/) { + die "No active section" if not defined $secname; + doDing($1); + } + + # !points-like directive? + elsif ($line =~ /^!points(adj|set)\s+(([+-])(\.|\d)+)$/) { + my ($mode, $arg) = ($1, $2); + if ($mode eq "adj") { + die "Both adjusting and setting points" if defined $pointsset; + $pointsadj = ($pointsadj or 0) + scalar $arg; + } elsif ($mode eq "set") { + die "Both adjusting and setting points" if defined $pointsadj; + die "Setting points twice" if defined $pointsset; + $pointsset = $arg; + } else { die "Unknown points-like directive: $line\n"; }; + } + + # $BEGIN_COMMENTS directive? + elsif ($line =~ /^\$BEGIN_COMMENTS/) { + my $seen_text = 0; + while (my $cline = ) { + chomp $cline; + last if $cline =~ /^\$END_COMMENTS/; + next if (not $seen_text and $cline =~ /^$/); + $seen_text = 1; + push @$commenttext, $cline; + } + } + + else { die "Unknown directive in: $line\n"; } +} + +finishSection() if defined $secname; +finishOverall(); + +# }}} + +# vim:foldmethod=marker diff --git a/make-skeleton.pl b/make-skeleton.pl new file mode 100644 index 0000000..cd1362e --- /dev/null +++ b/make-skeleton.pl @@ -0,0 +1,39 @@ +#!/usr/bin/env perl + +# Copyright (c) 2015, Nathaniel Wesley Filardo +# All rights reserved. +# See COPYING for details. + +use strict; +use warnings; + +my $section = undef; + +sub comments() { + print "\n\$BEGIN_COMMENTS\n\n\$END_COMMENTS\n\n"; +} + +while(my $line = ) { + # @section directive? + if ($line =~ /^@(\S+)\s/) { + comments() if defined $section; + $section = $1; + print "\@$section\n"; + } + + # :define directive? + elsif ($line =~ /^(:\S+)\s+/) { + die "Directive not within section" if not defined $section; + print "#$1\n"; + } + + # "#..." and not "#!..." get passed to template + elsif ($line =~ /^#(?!#)/) { + print $line; + } + + else { die "Unknown line in definition file"; } +} + +die "No sections encountered" if not defined $section; +comments(); -- 2.50.1