improve evaluation setup

Author: noahvanes

code/jvm/src/main/scala/maf/cli/experiments/SchemeAnalyses.scala

@@ -159,6 +159,10 @@ object SchemeAnalyses:
           with FIFOWorklistAlgorithm[SchemeExp]
           with SchemeModFLocalFSAnalysisResults
 
+    def modfADIAnalysis(prg: SchemeExp, k: Int) = 
+      new SchemeModFADIAnalysis(prg, k)
+          with SchemeModFADIAnalysisResults
+
     // Flow sensitive analysis
     def modFFlowSensitive(prg: SchemeExp) =
         new SimpleFlowSensitiveAnalysis(prg)

code/jvm/src/main/scala/maf/cli/experiments/precision/AnalysisComparisonAlt.scala

@@ -23,10 +23,10 @@ abstract class AnalysisComparisonAlt[Num: IntLattice, Rea: RealLattice, Bln: Boo
     // - the number of runs for the concrete interpreter
     def analyses: List[(SchemeExp => Analysis, String)]
     def benchmarks: List[Benchmark]
-    def runs = 3 // number of runs for the concrete interpreter
+    def runs = 1 // number of runs for the concrete interpreter
 
     // and can, optionally, be configured in its timeouts (default: 10min.) and the number of concrete runs
-    def timeout() = Timeout.start(Duration(10, MINUTES)) // timeout for the analyses
+    def timeout() = Timeout.start(Duration(5, MINUTES)) // timeout for the analyses
 
     // keep the results of the benchmarks in a table
     enum Result:
@@ -80,10 +80,13 @@ abstract class AnalysisComparisonAlt[Num: IntLattice, Rea: RealLattice, Bln: Boo
 
     private def runBenchmarks(benchmarks: List[Benchmark], outputFolder: Option[String] = None) =
       assert(benchmarks.size == benchmarks.toSet.size) // sanity check to make sure we don't have duplicates
-      benchmarks.foreach(runBenchmark)
-      showResults(outputFolder)
+      benchmarks.foreach { benchmark =>
+        runBenchmark(benchmark)
+        writeResultsToFile(outputFolder)
+      }
+      showResults()
 
-    protected def showResults(outputFolder: Option[String]) =
+    protected def showResults() =
       println()
       println("PRECISION RESULTS")
       println("=================")
@@ -94,16 +97,18 @@ abstract class AnalysisComparisonAlt[Num: IntLattice, Rea: RealLattice, Bln: Boo
       println("===================")
       println()
       println(performanceResults.prettyString())
-      outputFolder.foreach { dir => 
-        writeToFile(precisionResults,   s"$dir/precision-benchmarks.csv")
-        writeToFile(performanceResults, s"$dir/performance-benchmarks.csv")      
-      }
 
     protected def writeToFile[X](results: Table[X], path: String) =
       val writer = Writer.open(path)
       Writer.write(writer, results.toCSVString(rowName = "benchmark"))
       Writer.close(writer)
 
+    protected def writeResultsToFile(outputFolder: Option[String]) =
+      outputFolder.foreach { dir => 
+        writeToFile(precisionResults,   s"$dir/precision-benchmarks.csv")
+        writeToFile(performanceResults, s"$dir/performance-benchmarks.csv")      
+      }
+
     def main(args: Array[String]) = runBenchmarks(benchmarks, args.headOption)
 
 
@@ -123,6 +128,9 @@ abstract class SASBenchmarks
     lazy val dss0: (SchemeExp => Analysis, String)        = (SchemeAnalyses.modflocalAnalysis(_, 0),          "0-CFA DSS")
     lazy val dss1: (SchemeExp => Analysis, String)        = (SchemeAnalyses.modflocalAnalysis(_, 1),          "1-CFA DSS")
     lazy val dss2: (SchemeExp => Analysis, String)        = (SchemeAnalyses.modflocalAnalysis(_, 2),          "2-CFA DSS")
+    lazy val adi0: (SchemeExp => Analysis, String)        = (SchemeAnalyses.modfADIAnalysis(_, 0),            "0-CFA ADI")
+    lazy val adi1: (SchemeExp => Analysis, String)        = (SchemeAnalyses.modfADIAnalysis(_, 1),            "1-CFA ADI")
+    lazy val adi2: (SchemeExp => Analysis, String)        = (SchemeAnalyses.modfADIAnalysis(_, 2),            "2-CFA ADI")
     lazy val dssFS0: (SchemeExp => Analysis, String)      = (SchemeAnalyses.modflocalFSAnalysis(_, 0, true),  "0-CFA DSS-FS")
     lazy val dssFS1: (SchemeExp => Analysis, String)      = (SchemeAnalyses.modflocalFSAnalysis(_, 1, true),  "1-CFA DSS-FS")
     lazy val dssFS2: (SchemeExp => Analysis, String)      = (SchemeAnalyses.modflocalFSAnalysis(_, 2, true),  "2-CFA DSS-FS")
@@ -144,6 +152,18 @@ abstract class SASBenchmarks
         //"triangl" <- times out in concrete interpreter
       ).map(name => s"test/R5RS/gabriel/$name.scm")
 
+    def slowBenchmarks = 
+      List(
+        "boyer",
+        "browse",
+        "destruc"
+      ).map(name => s"test/R5RS/gabriel/$name.scm")
+      ++
+      List(
+          "test/R5RS/gambit/matrix.scm",
+          "test/R5RS/various/mceval.scm",
+      )
+
     def extraBenchmarks = 
       List(
           "test/R5RS/various/grid.scm",
@@ -188,26 +208,44 @@ object Table4Benchmarks extends SASBenchmarks:
     dssFS0NoGC, dssFS1NoGC, dssFS2NoGC
   )
 
+object ADIBenchmarks extends SASBenchmarks:
+  def analyses = List(
+    dss0, dss1, dss2,
+    adi0, adi1, adi2
+  )
+
+object DSS2Benchmark extends SASBenchmarks:
+  def analyses = List(dssFS2NoGC)
+  override def benchmarks = List("test/R5RS/gabriel/boyer.scm")
+
 object CountIterationBenchmark extends SASBenchmarks:
 
   def analyses = List(
     dssFS0, dssFS0NoGC
   )
 
-  var results: Table[Int] = Table.empty 
+  var results: Table[Option[Long]] = Table.empty 
 
   override protected def forBenchmark(path: Benchmark, program: SchemeExp): Unit =
     println(s"ANALYZING $path")
     analyses.foreach { case (analysis, name) => 
       val anl = analysis(program)
       anl.analyzeWithTimeout(Timeout.start(Duration(15, MINUTES)))
-      results = results.add(path, name, anl.asInstanceOf[SchemeModFLocalFS].iterations)
+      val res = anl.asInstanceOf[SchemeModFLocalFS].computeOneCountsPercentage.map(_ * 100)
+      results = results.add(path, name, res.map(_.round))
     }
 
   override def timeout() = Timeout.none
 
-  override protected def showResults(outputFolder: Option[String]) =
-    println(results.prettyString())
+  private def showResult(res: Option[Long]): String = 
+    res match
+      case None => "N/A"
+      case Some(value) => s"$value%"
+  
+  override protected def showResults() =
+    println(results.prettyString(format=showResult))
+
+  override protected def writeResultsToFile(outputFolder: Option[String]) = 
     outputFolder.foreach { dir => 
       writeToFile(results, s"$dir/iteration-benchmarks.csv") 
     }

code/shared/src/main/scala/maf/modular/scheme/modactor/ASchemeMirrorsSemantics.scala

@@ -355,13 +355,13 @@ trait ASchemeMirrorsSemantics extends ASchemeSemantics:
         case ASchemeCreate(beh, ags, idn) =>
             // we intercept actor creation and forward it to the meta layer if necessary
             for
-                evaluatedBeh <- nontail(eval(beh))
+                evaluatedBeh <- nontail(???)(eval(beh))
                 evaluatedAgs <- evalAll(ags)
                 result <- interceptCreate(evaluatedBeh, evaluatedAgs, lattice.bool(false), ags, idn)
             yield result
         case ASchemeSend(actorRef, Identifier(tag, _), ags, idn) =>
             for
-                evaluatedActor <- nontail(eval(actorRef))
+                evaluatedActor <- nontail(???)(eval(actorRef))
                 evaluatedAgs <- evalAll(ags)
                 self <- selfActor
                 message = Message(tag, evaluatedAgs, ags)
@@ -426,9 +426,9 @@ trait ASchemeMirrorsSemantics extends ASchemeSemantics:
             // TODO: in the concrete actor implementation we have that the create-with-mirror call
             // is also intercepted. We should do that here as well.
             for
-                evaluatedBehavior <- nontail(eval(behavior))
+                evaluatedBehavior <- nontail(???)(eval(behavior))
                 _ = log(s"+++ create-with-mirror (1) $behavior")
-                evaluatedAgs <- nontail(evalAll(ags))
+                evaluatedAgs <- nontail(???)(evalAll(ags))
                 _ = log(s"+++ create-with-mirror (2) $evaluatedAgs")
                 // defer spawning the actor since the mirror will be installed later
                 actorRef <- create(evaluatedBehavior, evaluatedAgs, idn, defer = true)
@@ -444,7 +444,7 @@ trait ASchemeMirrorsSemantics extends ASchemeSemantics:
 
         case SchemeFuncall(SchemeVar(Identifier("base/send-envelope" | "send-envelope", _)), List(envelopeExpression), _) =>
             for
-                evaluatedEnvelope <- nontail(eval(envelopeExpression))
+                evaluatedEnvelope <- nontail(???)(eval(envelopeExpression))
                 _ = log(s"+++ base/send-envelope $evaluatedEnvelope")
                 // get the envelopes from the abstract domain
                 envelope <- nondets(lattice.getEnvelopes(evaluatedEnvelope).map(unit))

code/shared/src/main/scala/maf/modular/scheme/modactor/ASchemeSemantics.scala

@@ -136,15 +136,15 @@ trait ASchemeSemantics extends SchemeSemantics, SchemeModFLocalSensitivity, Sche
             case ASchemeCreate(beh, ags, idn) =>
                 for
                     env <- getEnv
-                    evaluatedBeh <- nontail(eval(beh))
+                    evaluatedBeh <- nontail(???)(eval(beh))
                     evaluatedAgs <- evalAll(ags)
                     actorRef <- create(evaluatedBeh, evaluatedAgs, idn)
                 yield lattice.actor(actorRef)
 
             // Sending a message is atomic (asynchronous) and results in nil
             case ASchemeSend(actorRef, messageTpy, ags, idn) =>
                 for
-                    evaluatedActorRef <- nontail(eval(actorRef))
+                    evaluatedActorRef <- nontail(???)(eval(actorRef))
                     evaluatedAgs <- evalAll(ags)
                     _ = { log(s"++ intra - actor/send $evaluatedActorRef $messageTpy $evaluatedAgs") }
                     // TODO: the context should be inherited from the current context.
@@ -157,7 +157,7 @@ trait ASchemeSemantics extends SchemeSemantics, SchemeModFLocalSensitivity, Sche
             // Change the behavior of the current actor, and return nil
             case ASchemeBecome(beh, ags, idn) =>
                 for
-                    evaluatedBeh <- nontail(eval(beh))
+                    evaluatedBeh <- nontail(???)(eval(beh))
                     evaluatedAgs <- evalAll(ags)
                     _ <- become(evaluatedBeh, evaluatedAgs, idn)
                     // we stop the analysis here because the actor is in a suspended state,