Support Student-written Executable Examples (WiP)

app/helpers/test_case_helper.rb

@@ -0,0 +1,99 @@
+module TestCaseHelper
+
+    # if called from coding_prompt.rb, cp_answer is nil
+    def generate_CSV_tests(file_name, coding_prompt, cp_answer = nil)
+        lang = coding_prompt.language
+        tests = ''
+        if cp_answer.nil?
+            loc = coding_prompt.test_cases.only_dynamic
+        else
+            loc = cp_answer.student_test_cases
+        end
+        loc.each do |test_case|
+            tests << self.to_code(lang, test_case)
+        end
+        body = File.read('usr/resources/' + lang + '/' + lang +
+          'BaseTestFile.' + Exercise.extension_of(lang))
+        File.write(file_name, body % {
+          tests: tests,
+          method_name: coding_prompt.method_name,
+          class_name: coding_prompt.class_name
+          })
+      end
+
+    # moved from test_case.rb and student_test_case.rb
+    def to_code(language, test_case)
+        inp = test_case.input
+        if !inp.blank?
+          # TODO: need to fix this to handle nested parens appropriately
+          inp.gsub!(/map\([^()]*\)/) do |map_expr|
+            map_expr.split(/"/).map.with_index{ |x, i|
+              if i % 2 == 0
+                x.gsub(/\s*=(>?)\s*/, ', ')
+              else
+                x
+              end
+            }.join('"')
+          end
+        end
+        if test_case.is_a?(StudentTestCase)
+            coding_prompt = test_case.coding_prompt_answer.actable.prompt.specific
+            TEST_METHOD_TEMPLATES[language] % {
+                id: test_case.id.to_s,
+                method_name: coding_prompt.method_name,
+                class_name: coding_prompt.class_name,
+                input: inp,
+                expected_output: test_case.expected_output,
+                negative_feedback: 'check your understanding',
+              }
+        else
+            TEST_METHOD_TEMPLATES[language] % {
+                id: test_case.id.to_s,
+                method_name: test_case.coding_prompt.method_name,
+                class_name: test_case.coding_prompt.class_name,
+                input: inp,
+                expected_output: test_case.expected_output,
+                negative_feedback: test_case.negative_feedback,
+                array: ((test_case.expected_output.start_with?('new ') &&
+                  test_case.expected_output.include?('[]')) ||
+                  test_case.expected_output.start_with?('array(')) ? 'Array' : ''
+              }
+        end
+    end
+
+
+      # heredoc
+      # moved from test_case.rb and student_test_case.rb
+      TEST_METHOD_TEMPLATES = {
+        'Ruby' => <<RUBY_TEST,
+    def test%{id}
+      assert_equal(%{expected_output}, @@subject.%{method_name}(%{input}), "%{negative_feedback}")
+    end
+
+RUBY_TEST
+        'Python' => <<PYTHON_TEST,
+      def test%{id}(self):
+          self.assertEqual(%{expected_output}, self.__%{method_name}(%{input}))
+
+PYTHON_TEST
+        'Java' => <<JAVA_TEST,
+      @Test
+      public void test_%{id}()
+      {
+          assertEquals(
+            "%{negative_feedback}",
+            %{expected_output},
+            subject.%{method_name}(%{input}));
+      }
+JAVA_TEST
+        'C++' => <<CPP_TEST
+      void test%{id}()
+      {
+          TSM_ASSERT_EQUALS(
+            "%{negative_feedback}",
+            %{expected_output},
+            subject.%{method_name}(%{input}));
+      }
+CPP_TEST
+      }
+end

app/jobs/code_worker.rb

@@ -8,18 +8,57 @@
 
 class CodeWorker
   include SuckerPunch::Job
-
+  include TestCaseHelper
   # Reducing to 2 workers, since it seems that each puma process will
   # have its own job queue and its own set of sucker punch worker threads.
   # We'll get parallelism through puma processes instead of sucker punch
   # workers.
   workers 2 # 10
 
+  # grabs student-written tests from answer text
+  # String -> List[[]]
+  def self.get_tests_from_javadoc(answer_text)
+    test_list = []
+    flag = false
+    source_list = (answer_text.split("/"))[1].split("*")
+    source_list.each do |elem|
+      if elem.include? "@"
+        if elem.include? "@test"
+          flag = true
+        else
+          flag = false
+        end
+      elsif flag && elem.include?("->")
+
+        temp = elem.split("->")
+        #check elements here
+        test_list.append([temp[0][/\(([^()]*)\)/, 1], temp[1].strip, elem])
+      end
+    end
+    return test_list
+  end
+
+  # -------------------------------------------------------------
+  # directs parsing to appropriate method
+  # String, String -> List[[]]
+  def self.get_tests_from_answer_text(answer_text, language)
+    case language
+    when 'Java'
+      return get_tests_from_javadoc(answer_text)
+    when 'Ruby'
+      return nil
+    when 'Python'
+      return nil
+    when 'C++'
+      return nil
+    end
+  end
+
+
   # -------------------------------------------------------------
   def perform(attempt_id)
     ActiveRecord::Base.connection_pool.with_connection do
       start_time = Time.now
-
       attempt = Attempt.find(attempt_id)
       exv = attempt.exercise_version
       prompt = exv.prompts.first.specific
@@ -29,7 +68,7 @@ def perform(attempt_id)
       answer_text = answer.answer
       answer_lines = answer_text ? answer_text.count("\n") : 0
       if !prompt.wrapper_code.blank?
-        code_body = prompt.wrapper_code.sub(/\b___\b/, answer_text)
+        code_body = prompt.wrapper_code.sub(/\b___\b/, answer_text) # student's answer
         if $`
           # Want pre_lines to be a count of the number of lines preceding
           # the one the match is on, so use count() instead of lines() here
@@ -51,7 +90,8 @@ def perform(attempt_id)
       term_name = term ? term.slug : 'no-term'
 
       # compile and evaluate the attempt in a temporary location
-      attempt_dir = "usr/attempts/active/#{current_attempt}"
+      working_dir = "usr/attempts/active/#{current_attempt}" ######## replicate this code but with inst soln
+      attempt_dir = "#{working_dir}/attempt"
       # puts "DIRECTORY",attempt_dir,"DIRECTORY"
       FileUtils.mkdir_p(attempt_dir)
       if !Dir[attempt_dir].empty?
@@ -65,7 +105,38 @@ def perform(attempt_id)
         prompt.regenerate_tests
       end
       FileUtils.cp(prompt.test_file_name, attempt_dir)
-      File.write(attempt_dir + '/' + prompt.class_name + '.' + lang, code_body)
+      File.write(attempt_dir + '/' + prompt.class_name + '.' + lang, code_body) ###
+
+      # compile and load student tests into DB
+      answer.parse_student_tests!(answer_text, language, current_attempt) #rename
+
+      # run against inst soln
+      # creating reference directory
+      ref_dir = "#{working_dir}/reference"
+      ref_body = prompt.wrapper_code.sub(/\b___\b/, prompt.reference_solution)
+
+      FileUtils.mkdir_p(ref_dir)
+      if !Dir[ref_dir].empty?
+        puts 'WARNING, OVERWRITING EXISTING DIRECTORY = ' + ref_dir
+        FileUtils.remove_dir(ref_dir, true)
+        FileUtils.mkdir_p(ref_dir)
+      end
+
+      generate_CSV_tests(ref_dir + '/' + prompt.class_name + 'Test.' + Exercise.extension_of(prompt.language), prompt, answer)
+
+
+      File.write(ref_dir + '/' + prompt.class_name + '.' + lang, ref_body) ###
+
+      ref_lines = ref_body.count("\n")
+      execute_javatest(
+        prompt.class_name, ref_dir, pre_lines, ref_lines)
+
+      CSV.foreach(ref_dir + '/results.csv') do |line|
+        # find test id
+        test_id = line[2][/\d+$/].to_i
+        test_case = answer.student_test_cases.where(id: test_id).first
+        tc_score = test_case.record_result(answer, line)
+      end
 
       # Run static checks
       result = nil
@@ -138,8 +209,9 @@ def perform(attempt_id)
       attempt.feedback_ready = true
 
       # clean up log and class files that were generated during testing
+
       cleanup_files = Dir.glob([
-        "#{attempt_dir}/*.class",
+        "#{working_dir}/**/*.class",
         "#{attempt_dir}/*.log",
         "#{attempt_dir}/reports/TEST-*.csv",
         "#{attempt_dir}/__pycache__/*.pyc",
@@ -160,7 +232,7 @@ def perform(attempt_id)
       # move the attempt to permanent storage
       term_dir = "usr/attempts/#{term_name}/"
       FileUtils.mkdir_p(term_dir) # create the term_dir if it doesn't exist
-      FileUtils.mv(attempt_dir, term_dir)
+      FileUtils.mv(working_dir, term_dir)
 
       # calculate various time values. all times are in ms
       time_taken = (Time.now - attempt.submit_time) * 1000

app/models/coding_prompt.rb

@@ -23,6 +23,7 @@
 # gem).
 #
 class CodingPrompt < ActiveRecord::Base
+  include TestCaseHelper
 
   #~ Relationships ............................................................
 
@@ -124,13 +125,13 @@ def regenerate_tests
         end
       end
       # Default, if none of above cases return
-      generate_CSV_tests(test_file_name)
+      generate_CSV_tests(test_file_name, self)
     end
   end
 
 
   #~ Private instance methods .................................................
-  private
+  private 
 
   # -------------------------------------------------------------
   def set_defaults
@@ -181,7 +182,7 @@ def parse_tests
       end
       # Default, if none of above cases return
       parse_CSV_tests(self.test_script)
-      generate_CSV_tests(test_file_name)
+      generate_CSV_tests(test_file_name, self)
     end
   end
 
@@ -227,24 +228,7 @@ def parse_CSV_tests(csv_text)
       end
     end
   end
-
-
-  # -------------------------------------------------------------
-  def generate_CSV_tests(file_name)
-    lang = self.language
-    tests = ''
-    self.test_cases.only_dynamic.each do |test_case|
-      tests << test_case.to_code(lang)
-    end
-    body = File.read('usr/resources/' + lang + '/' + lang +
-      'BaseTestFile.' + Exercise.extension_of(lang))
-    File.write(file_name, body % {
-      tests: tests,
-      method_name: self.method_name,
-      class_name: self.class_name
-      })
-  end
-
+
 
   # -------------------------------------------------------------
   def parse_JUnit_tests
@@ -318,6 +302,7 @@ def parse_JUnit_tests
           end
         end
         tc.parse_description_specifier(desc)
+        puts(tc.description)
 
         # look for "example" tag in comments or attribute
         if comment =~ /example\s*:\s*true\s*(?:\*\/\s*)?$/i || attrs =~ /@Example\b/

app/models/coding_prompt_answer.rb

@@ -19,6 +19,8 @@ class CodingPromptAnswer < ActiveRecord::Base
   #~ Relationships ............................................................
 
   acts_as :prompt_answer
+  has_many :student_test_cases
+  has_many :student_test_case_results
   has_many :test_case_results,
     #-> { includes :test_case },
     -> { order('test_case_id ASC').includes(:test_case) },
@@ -32,9 +34,32 @@ class CodingPromptAnswer < ActiveRecord::Base
   # answer all prompts, and that would constitute an empty answer. We
   # want to allow that, so do not add validations preventing it.
 
+  def prompt_dir
+    'usr/resources/' + self.language + '/tests/' + self.id.to_s
+  end
 
+  def test_file_name
+    prompt_dir + '/' + self.class_name + 'Test.' +
+      Exercise.extension_of(self.language)
+  end
   #~ Instance methods .........................................................
 
+  # -------------------------------------------------------------
+  def parse_student_tests!(answer_text, language, id)
+    testList = CodeWorker.get_tests_from_answer_text(answer_text, language)
+    testList.each do |test|
+      tc = StudentTestCase.new(
+        input: test[0],
+        expected_output: test[1],
+        description: test[2],
+        coding_prompt_answer_id: self.id
+      )
+      unless tc.save
+        puts "error saving test case: #{tc.errors.full_messages.to_s}"
+      end
+    end
+  end
+
   # -------------------------------------------------------------
   def execute_static_tests
     result = nil

app/models/student_test_case.rb

@@ -0,0 +1,34 @@
+class StudentTestCase < ActiveRecord::Base
+
+    # Relationships
+    belongs_to :coding_prompt_answer
+    has_one :student_test_case_result
+
+    def record_result(answer, test_results_array)
+        tcr = StudentTestCaseResult.new(
+          test_case_id: self.id,
+          coding_prompt_answer: answer,
+          pass: (test_results_array.length == 8 && test_results_array[7].to_i == 1)
+          )
+
+        if !test_results_array[5].blank?
+            exception_name = test_results_array[6] #.sub(/^.*\./, '')
+            if !(['AssertionFailedError',
+              'AssertionError',
+              'ComparisonFailure',
+              'ReflectionSupportError'].include?(exception_name) ||
+              (exception_name == 'Exception' &&
+              test_results_array[6].start_with?('test timed out'))) ||
+              test_results_array[6].blank? ||
+              "null" == test_results_array[6]
+              tcr.feedback = exception_name
+            end
+        end
+        tcr.save!    
+    end
+
+    def display_description()
+        self.description
+    end
+
+end

app/models/student_test_case_result.rb

@@ -0,0 +1,12 @@
+class StudentTestCaseResult < ActiveRecord::Base
+
+    # Relationships
+    belongs_to :coding_prompt_answer, inverse_of: :student_test_case_results
+    belongs_to :student_test_case, class_name: "StudentTestCase", 
+        foreign_key: :test_case_id, inverse_of: :student_test_case_result
+
+    def display_description
+        student_test_case.display_description()
+    end
+
+end