Add benchmark for ResolveKinds with hot JIT

3 files changed, 85 insertions, 0 deletions

diff --git a/benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala b/benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala
new file mode 100644
index 00000000..61e9c1a6
--- /dev/null
+++ b/benchmark/src/main/scala/firrtl/benchmark/hot/ResolveKindsBenchmark.scala
@@ -0,0 +1,23 @@
+
+package firrtl
+package benchmark
+package hot
+
+import passes.ResolveKinds
+import stage.TransformManager
+
+import firrtl.benchmark.util._
+
+object ResolveKindsBenchmark extends App {
+  val inputFile = args(0)
+  val warmup = args(1).toInt
+  val runs = args(2).toInt
+
+  val input = filenameToCircuit(inputFile)
+  val state = CircuitState(input, ChirrtlForm)
+  val prereqs = ResolveKinds.prerequisites
+  val manager = new TransformManager(prereqs)
+  val preState = manager.execute(state)
+
+  hot.util.benchmark(warmup, runs)(ResolveKinds.run(preState.circuit))
+}
diff --git a/benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala b/benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala
new file mode 100644
index 00000000..c05b8359
--- /dev/null
+++ b/benchmark/src/main/scala/firrtl/benchmark/hot/util/package.scala
@@ -0,0 +1,28 @@
+
+package firrtl.benchmark.hot
+
+import firrtl.Utils.time
+import firrtl.benchmark.util._
+
+package object util {
+  def benchmark(nWarmup: Int, nRun: Int)(f: => Unit): Unit = {
+    // Warmup
+    for (i <- 0 until nWarmup) {
+      val (t, res) = time(f)
+      println(f"Warmup run $i took $t%.1f ms")
+    }
+
+    // Benchmark
+    val times: Array[Double] = Array.fill(nRun)(0.0)
+    for (i <- 0 until nRun) {
+      val (t, res) = time(f)
+      times(i) = t
+      println(f"Benchmark run $i took $t%.1f ms")
+    }
+
+    println(f"Mean:   ${mean(times)}%.1f ms")
+    println(f"Median: ${median(times)}%.1f ms")
+    println(f"Stddev: ${stdDev(times)}%.1f ms")
+  }
+
+}
diff --git a/benchmark/src/main/scala/firrtl/benchmark/util/package.scala b/benchmark/src/main/scala/firrtl/benchmark/util/package.scala
new file mode 100644
index 00000000..2923d8b5
--- /dev/null
+++ b/benchmark/src/main/scala/firrtl/benchmark/util/package.scala
@@ -0,0 +1,34 @@
+
+package firrtl
+package benchmark
+
+import firrtl.ir.Circuit
+import scala.util.control.NonFatal
+
+package object util {
+  def filenameToCircuit(filename: String): Circuit = try {
+    proto.FromProto.fromFile(filename)
+  } catch {
+    case NonFatal(_) => Parser.parseFile(filename, Parser.IgnoreInfo)
+  }
+
+  def mean(xs: Iterable[Double]): Double = xs.sum / xs.size
+
+  def median(xs: Iterable[Double]): Double = {
+    val size = xs.size
+    val sorted = xs.toSeq.sorted
+    if (size % 2 == 1) sorted(size / 2)
+    else {
+      val a = sorted(size / 2)
+      val b = sorted((size / 2) - 1)
+      (a + b) / 2
+    }
+  }
+
+  def variance(xs: Iterable[Double]): Double = {
+    val avg = mean(xs)
+    xs.map(a => math.pow(a - avg, 2)).sum / xs.size
+  }
+
+  def stdDev(xs: Iterable[Double]): Double = math.sqrt(variance(xs))
+}