Code Coverage

docs/getting-started/db-insert-statements/insert-rubric-database.md

@@ -10,7 +10,7 @@ The primary things that needs to be updated on an ongoing basis are:
 
 ## Statements
 > [!NOTE]
-> Updated as of **WINTER 2026**
+> Updated as of **SUMMER 2026**
 
 ```mysql
 INSERT INTO rubric_config (phase, type, category, criteria, points, rubric_id) VALUES
@@ -23,16 +23,16 @@ INSERT INTO rubric_config (phase, type, category, criteria, points, rubric_id) V
 ('Phase3', 'GIT_COMMITS', 'Git Commits', 'Necessary commit amount', 0, '90344_2520'),
 ('Phase3', 'PASSOFF_TESTS', 'Web API Works', 'All pass off test cases in StandardAPITests.java succeed', 125, '_5202'),
 ('Phase3', 'QUALITY', 'Code Quality', 'Chess Code Quality Rubric (see GitHub)', 30, '_3003'),
-('Phase3', 'UNIT_TESTS', 'Unit Tests', 'All test cases pass\nEach public method on your Service classes has two test cases, one positive test and one negative test\nEvery test case includes an Assert statement of some type', 25, '90344_776'),
+('Phase3', 'UNIT_TESTS', 'Unit Tests', 'All test cases pass\nCode coverage on service package must be at least 80% line coverage\nEvery test case includes an Assert statement of some type', 25, '90344_776'),
 ('Phase4', 'GIT_COMMITS', 'Git Commits', 'Necessary commit amount', 0, '90346_6245'),
 ('Phase4', 'PASSOFF_TESTS', 'Functionality', 'All pass off test cases succeed', 100, '_2614'),
 ('Phase4', 'QUALITY', 'Code Quality', 'Chess Code Quality Rubric (see GitHub)', 30, '90346_8398'),
-('Phase4', 'UNIT_TESTS', 'Unit Tests', 'All test cases pass\nEach public method on DAO classes has two test cases, one positive test and one negative test\nEvery test case includes an Assert statement of some type', 25, '90346_5755'),
+('Phase4', 'UNIT_TESTS', 'Unit Tests', 'All test cases pass\nCode coverage on dataaccess package must be at least 80% line coverage\nEvery test case includes an Assert statement of some type', 25, '90346_5755'),
 ('Phase5', 'GIT_COMMITS', 'Git Commits', 'Necessary commit amount', 0, '90347_8497'),
 ('Phase5', 'QUALITY', 'Code Quality', 'Chess Code Quality Rubric (see GitHub)', 30, '90347_9378'),
-('Phase5', 'UNIT_TESTS', 'Unit Tests', 'All test cases pass\nEach public method on the Server Facade class has two test cases, one positive test and one negative test\nEvery test case includes an Assert statement of some type', 25, '90347_2215'),
+('Phase5', 'UNIT_TESTS', 'Unit Tests', 'All test cases pass\nCode coverage on facade package must be at least 80% line coverage\nEvery test case includes an Assert statement of some type', 25, '90347_2215'),
 ('Phase6', 'GIT_COMMITS', 'Git Commits', 'Necessary commit amount', 0, '90348_9048'),
-('Phase6', 'PASSOFF_TESTS', 'Automated Pass Off Test Cases', 'Each provided test case passed is worth a proportional number of points ((passed / total) * 50).', 50, '90348_899'),
+('Phase6', 'PASSOFF_TESTS', 'Automated Pass Off Test Cases', 'All pass off test cases succeed', 50, '90348_899'),
 ('Phase6', 'QUALITY', 'Code Quality', 'Chess Code Quality Rubric (see GitHub)', 30, '90348_3792'),
 ('Quality', 'QUALITY', 'Code Quality', 'Chess Code Quality Rubric (see GitHub)', 30, NULL);
 ```

src/main/java/edu/byu/cs/autograder/compile/CompileHelper.java

@@ -26,7 +26,8 @@ public CompileHelper(GradingContext gradingContext) {
 
     private final Collection<StudentCodeVerifier> currentVerifiers =
             List.of(new ProjectStructureVerifier(), new ModuleIndependenceVerifier(), new ModifiedTestFilesVerifier(),
-                    new TestLocationVerifier(), new TestClassNameVerifier(), new ServerFacadeTestPortVerifier(), new ModifiedWebResourcesVerifier());
+                    new TestLocationVerifier(), new TestClassNameVerifier(), new ServerFacadeTestPortVerifier(),
+                    new ModifiedWebResourcesVerifier(), new CodeCoverageVerifier());
 
 
     private final Collection<StudentCodeModifier> currentModifiers =

src/main/java/edu/byu/cs/autograder/compile/verifers/CodeCoverageVerifier.java

@@ -0,0 +1,67 @@
+package edu.byu.cs.autograder.compile.verifers;
+
+import edu.byu.cs.autograder.GradingContext;
+import edu.byu.cs.autograder.GradingException;
+import edu.byu.cs.autograder.compile.StudentCodeReader;
+import edu.byu.cs.autograder.compile.StudentCodeVerifier;
+import edu.byu.cs.model.CoverageRequirement;
+import edu.byu.cs.model.Phase;
+import edu.byu.cs.util.PhaseUtils;
+
+import java.io.File;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Verifies that the necessary packages and classes exists to collect code coverage on unit tests
+ */
+public class CodeCoverageVerifier implements StudentCodeVerifier {
+    private final Set<String> missingPackages = new TreeSet<>();
+    private final Set<String> missingFiles = new TreeSet<>();
+
+    @Override
+    public void verify(GradingContext context, StudentCodeReader reader) throws GradingException {
+        Phase currPhase = context.phase();
+        CoverageRequirement coverage = PhaseUtils.unitTestCoverageRequirements(currPhase);
+        if (coverage.type() == CoverageRequirement.CoverageType.PACKAGE){
+            File packageDirectory = new File(context.stageRepo(), coverage.name());
+            if (!packageDirectory.isDirectory()){
+                missingPackages.add(coverage.name());
+            }
+        }
+        else{
+
+            if (reader.filesMatching(buildFileRegex(coverage.name())).findAny().isEmpty()){
+                missingFiles.add(coverage.name());
+            }
+        }
+
+        String message = buildMessage();
+        if (!message.isBlank()){
+            context.observer().notifyWarning(message);
+        }
+    }
+
+    private String buildMessage(){
+        StringBuilder stringBuilder = new StringBuilder();
+        if (!missingPackages.isEmpty()){
+            stringBuilder.append("Missing package(s) for Code Coverage: ")
+                    .append(String.join(", ", missingPackages))
+                    .append(".\n");
+        }
+        if (!missingFiles.isEmpty()){
+            stringBuilder.append("Missing file(s) for Code Coverage: ")
+                    .append(String.join(", ", missingFiles))
+                    .append(".\n");
+        }
+        if (!stringBuilder.isEmpty()){
+            stringBuilder.append("Code coverage cannot be calculated without the proper project structure.\n")
+                    .append("Please double check the phase's specification.");
+        }
+        return stringBuilder.toString();
+    }
+
+    private String buildFileRegex(String fileName){
+        return "^.*[\\/]"+ fileName + "\\.java$|^"+ fileName + "\\.java$";
+    }
+}

src/main/java/edu/byu/cs/autograder/test/CoverageAnalyzer.java

@@ -1,29 +1,47 @@
 package edu.byu.cs.autograder.test;
 
 import edu.byu.cs.autograder.GradingException;
+import edu.byu.cs.dataAccess.DaoService;
+import edu.byu.cs.dataAccess.DataAccessException;
+import edu.byu.cs.dataAccess.daoInterface.ConfigurationDao;
 import edu.byu.cs.model.ClassCoverageAnalysis;
 import edu.byu.cs.model.CoverageAnalysis;
 import edu.byu.cs.util.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.util.Collection;
 import java.util.HashSet;
 
 /**
  * Parses the code coverage output stored in a JaCoCo CSV file into a
- * {@link CoverageAnalysis} containing the appropriate coverage results
+ * {@link CoverageAnalysis} containing the appropriate coverage results.
+ * <br>
+ * Supports Branch and Line coverage. Default coverage is Line Coverage.
  */
 public class CoverageAnalyzer {
-    /**
-     * The type of coverage to be tested for. This is the only value that should be
-     * updated if coverage requirements change.
-     */
-    private static final String COVERAGE_TESTED = "BRANCH";
-    private static final String COVERAGE_MISSED_HEADER = COVERAGE_TESTED + "_MISSED";
-    private static final String COVERAGE_COVERED_HEADER = COVERAGE_TESTED + "_COVERED";
+    private final String coverageMissedHeader;
+    private final String coverageCoveredHeader;
     private static final String PACKAGE_HEADER = "PACKAGE";
     private static final String CLASS_HEADER = "CLASS";
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(CoverageAnalyzer.class);
+
+    public CoverageAnalyzer(){
+        String coverageTested = "LINE";
+        try{
+            String config = DaoService.getConfigurationDao().getConfiguration(ConfigurationDao.Configuration.COVERAGE_TYPE, String.class);
+            if (config.equals("BRANCH")){ //ensure the value is valid
+                coverageTested = config;
+            }
+        } catch (DataAccessException e) {
+            LOGGER.error("Could not get coverage type, using line coverage", e);
+        }
+        coverageMissedHeader = coverageTested + "_MISSED";
+        coverageCoveredHeader = coverageTested + "_COVERED";
+    }
+
     /**
      * Parses the output of a JaCoCo CSV file, if it exists,
      * and returns the coverage results as a {@link CoverageAnalysis}
@@ -52,8 +70,14 @@ public CoverageAnalysis parse(File jacocoCsvOutput) throws GradingException {
             switch (s) {
                 case CLASS_HEADER -> classHeader = i;
                 case PACKAGE_HEADER -> packageHeader = i;
-                case COVERAGE_MISSED_HEADER -> coverageMissedHeader = i;
-                case COVERAGE_COVERED_HEADER -> coverageCoveredHeader = i;
+                default -> {
+                    if (s.equals(this.coverageMissedHeader)){
+                        coverageMissedHeader = i;
+                    }
+                    else if (s.equals(this.coverageCoveredHeader)) {
+                        coverageCoveredHeader = i;
+                    }
+                }
             }
         }
 

src/main/java/edu/byu/cs/autograder/test/TestGrader.java

@@ -71,7 +71,8 @@ public Rubric.Results runTests() throws GradingException, DataAccessException {
         } else {
             results = new TestHelper().runJUnitTests(new File(gradingContext.stageRepo(),
                             "/" + module + "/target/" + module + "-test-dependencies.jar"), stageTestsPath,
-                    packagesToTest(), ignoredTests(), modulesToCheckCoverage());
+                    packagesToTest(), ignoredTests(), modulesToCheckCoverage(),
+                    PhaseUtils.unitTestCoverageRequirements(gradingContext.phase()));
         }
 
         if (results.root() == null) {

src/main/java/edu/byu/cs/autograder/test/TestHelper.java

@@ -3,7 +3,9 @@
 import edu.byu.cs.autograder.GradingException;
 import edu.byu.cs.dataAccess.DaoService;
 import edu.byu.cs.dataAccess.daoInterface.ConfigurationDao;
+import edu.byu.cs.model.ClassCoverageAnalysis;
 import edu.byu.cs.model.CoverageAnalysis;
+import edu.byu.cs.model.CoverageRequirement;
 import edu.byu.cs.model.Rubric;
 import edu.byu.cs.model.TestNode;
 import edu.byu.cs.model.TestOutput;
@@ -15,10 +17,8 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
+import java.util.regex.Pattern;
 
 /**
  * A helper class for running common test operations
@@ -161,7 +161,7 @@ private static List<String> getCompileCommands(String stagePath, String chessJar
      * @return A TestNode object containing the results of the tests.
      */
     TestOutput runJUnitTests(File uberJar, File compiledTests, Set<String> packagesToTest,
-                             Set<String> ignoredTests, Set<String> coverageModules) throws GradingException {
+                             Set<String> ignoredTests, Set<String> coverageModules, CoverageRequirement coverageRequirement) throws GradingException {
         // Process cannot handle relative paths or wildcards,
         // so we need to only use absolute paths and find
         // to get the files
@@ -199,6 +199,7 @@ TestOutput runJUnitTests(File uberJar, File compiledTests, Set<String> packagesT
             File coverageOutput = new File(testOutputDirectory, "coverage.csv");
 
             CoverageAnalysis coverage = coverageOutput.exists() ? new CoverageAnalyzer().parse(coverageOutput) : null;
+            coverage = removeUnmatchedPackages(coverageRequirement, coverage);
             TestNode testAnalysis = testAnalyzer.parse(junitXmlOutput, ignoredTests);
 
             return new TestOutput(testAnalysis, coverage, trimErrorOutput(error));
@@ -208,6 +209,29 @@ TestOutput runJUnitTests(File uberJar, File compiledTests, Set<String> packagesT
         }
     }
 
+    private CoverageAnalysis removeUnmatchedPackages(CoverageRequirement requirement, CoverageAnalysis coverage) {
+        if (coverage == null || coverage.classAnalyses() == null || requirement == null) {
+            return coverage;
+        }
+        Pattern pattern = Pattern.compile("^" + requirement.name());
+        if (pattern.pattern().isEmpty()){
+            return coverage;
+        }
+        Collection<ClassCoverageAnalysis> matchedList = new ArrayList<>();
+        for (ClassCoverageAnalysis classCoverageAnalysis : coverage.classAnalyses()) {
+            if (requirement.type() == CoverageRequirement.CoverageType.CLASS) {
+                if (pattern.matcher(classCoverageAnalysis.className()).find()) {
+                    matchedList.add(classCoverageAnalysis);
+                }
+            } else {
+                if (pattern.matcher(classCoverageAnalysis.packageName()).find()) {
+                    matchedList.add(classCoverageAnalysis);
+                }
+            }
+        }
+        return new CoverageAnalysis(matchedList);
+    }
+
     private static List<String> getRunCommands(Set<String> packagesToTest, String uberJarPath) {
         List<String> commands = new ArrayList<>();
         commands.add("java");