Blackboxhelper (#418)

* First pass at implementing a annotation based mechanism to move black box verilator files into the target directory

* A little bit of style cleanup

* A little bit of style cleanup

* Fix the driver, wasn't appending targetDir properly
Add some docs

* test had wrong value now that targetdir is added to annnos

* Now saves a list of all black box verilog files moved into target directory.
Then creates a file black_box_verilog_files.f that contains this list with -v prepended to each line

* Made black box source helper be low to low form
Added it to the verilog compiler transforms
Added a test to make sure it got there

* targetDir annotation is targeted to a CircuitName("All")

5 files changed, 264 insertions, 4 deletions

diff --git a/src/main/scala/firrtl/Driver.scala b/src/main/scala/firrtl/Driver.scala
index 75a87789..238906c8 100644
--- a/src/main/scala/firrtl/Driver.scala
+++ b/src/main/scala/firrtl/Driver.scala
@@ -5,11 +5,13 @@ package firrtl
 import scala.collection._
 import scala.io.Source
 import java.io.{File, FileNotFoundException}
+
 import net.jcazevedo.moultingyaml._
 import logger.Logger
-import Parser.{InfoMode, IgnoreInfo}
+import Parser.{IgnoreInfo, InfoMode}
 import annotations._
 import firrtl.annotations.AnnotationYamlProtocol._
+import firrtl.transforms.{BlackBoxSourceHelper, BlackBoxTargetDir}
 
 
 /**
@@ -90,7 +92,20 @@ object Driver {
       val annotationFile = new File(annotationFileName)
       if (annotationFile.exists) {
         val annotationsYaml = io.Source.fromFile(annotationFile).getLines().mkString("\n").parseYaml
-        val annotationArray = annotationsYaml.convertTo[Array[Annotation]]
+        val annotationArray = {
+          val annos = annotationsYaml.convertTo[Array[Annotation]]
+
+          if (annos.nonEmpty) {
+            annos ++ List(Annotation(
+              CircuitName("All"),
+              classOf[BlackBoxSourceHelper],
+              BlackBoxTargetDir(optionsManager.targetDirName).serialize
+            ))
+          }
+          else {
+            annos
+          }
+        }
         optionsManager.firrtlOptions = firrtlConfig.copy(annotations = firrtlConfig.annotations ++ annotationArray)
       }
     }
diff --git a/src/main/scala/firrtl/LoweringCompilers.scala b/src/main/scala/firrtl/LoweringCompilers.scala
index 44d3a757..5201942a 100644
--- a/src/main/scala/firrtl/LoweringCompilers.scala
+++ b/src/main/scala/firrtl/LoweringCompilers.scala
@@ -110,6 +110,7 @@ class LowFirrtlOptimization extends CoreTransform {
 
 
 import CompilerUtils.getLoweringTransforms
+import firrtl.transforms.BlackBoxSourceHelper
 
 /** Emits input circuit
   * Will replace Chirrtl constructs with Firrtl
@@ -135,5 +136,5 @@ class LowFirrtlCompiler extends Compiler {
 class VerilogCompiler extends Compiler {
   def emitter = new VerilogEmitter
   def transforms: Seq[Transform] =
-    getLoweringTransforms(ChirrtlForm, LowForm) :+ (new LowFirrtlOptimization)
+    getLoweringTransforms(ChirrtlForm, LowForm) ++ Seq(new LowFirrtlOptimization, new BlackBoxSourceHelper)
 }
diff --git a/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala b/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala
new file mode 100644
index 00000000..914ca859
--- /dev/null
+++ b/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala
@@ -0,0 +1,144 @@
+// See LICENSE for license details.
+
+package firrtl.transforms
+
+import java.io.{File, FileNotFoundException, FileOutputStream, PrintWriter}
+
+import firrtl._
+import firrtl.annotations.{Annotation, ModuleName}
+
+import scala.collection.mutable.ArrayBuffer
+
+
+trait BlackBoxSource {
+  def serialize: String
+  def name: String
+}
+
+object BlackBoxSource {
+  val MaxFields = 3
+
+  def parse(s: String): Option[BlackBoxSource] = {
+    s.split("\n", MaxFields).toList  match {
+      case "resource" :: id ::  _ => Some(BlackBoxResource(id))
+      case "inline" :: name :: text :: _ => Some(BlackBoxInline(name, text))
+      case "targetDir" :: targetDir :: _ => Some(BlackBoxTargetDir(targetDir))
+      case _ => throw new FIRRTLException(s"Error: Bad BlackBox annotations $s")
+    }
+  }
+}
+
+case class BlackBoxTargetDir(targetDir: String) extends BlackBoxSource {
+  def serialize: String = s"targetDir\n$targetDir"
+  def name: String = targetDir
+}
+
+case class BlackBoxResource(resourceId: String) extends BlackBoxSource {
+  def serialize: String = s"resource\n$resourceId"
+  def name: String = resourceId.split("/").last
+}
+
+case class BlackBoxInline(name: String, text: String) extends BlackBoxSource {
+  def serialize: String = s"inline\n$name\n$text"
+}
+
+object BlackBoxSourceAnnotation {
+  def apply(targetDir: ModuleName, value: String): Annotation = {
+    assert(BlackBoxSource.parse(value).isDefined)
+    Annotation(targetDir, classOf[DedupModules], value)
+  }
+
+  def unapply(a: Annotation): Option[(ModuleName, BlackBoxSource)] = a match {
+    case Annotation(ModuleName(n, c), _, text) => Some((ModuleName(n, c), BlackBoxSource.parse(text).get))
+    case _ => None
+  }
+}
+
+/**
+  * This transform handles the moving of verilator source for black boxes into the
+  * target directory so that it can be accessed by verilator or other backend compilers
+  * While parsing it's annotations it looks for a BlackBoxTargetDir annotation that
+  * will set the directory where the verilog will be written.  This annotation is typically be
+  * set by the execution harness, or directly in the tests
+  */
+class BlackBoxSourceHelper extends firrtl.Transform {
+  private var targetDir: File = new File(".")
+  private val fileList = new ArrayBuffer[String]
+
+  override def inputForm: CircuitForm = LowForm
+  override def outputForm: CircuitForm = LowForm
+
+  /**
+    * parse the annotations and convert the generic annotations to specific information
+    * required to find the verilog
+    * @note Side effect is that while converting a magic target dir annotation is found and sets the target
+    * @param annos a list of generic annotations for this transform
+    * @return
+    */
+  def getSources(annos: Seq[Annotation]): Seq[BlackBoxSource] = {
+    annos.flatMap { anno => BlackBoxSource.parse(anno.value) }
+      .flatMap {
+        case BlackBoxTargetDir(dest) =>
+          targetDir = new File(dest)
+          if(! targetDir.exists()) { FileUtils.makeDirectory(targetDir.getAbsolutePath) }
+          None
+        case b: BlackBoxSource => Some(b)
+        case _ => None
+      }
+      .sortBy(a => a.name)
+      .distinct
+  }
+
+  /**
+    * write the verilog source for each annotation to the target directory
+    * @note the state is not changed by this transform
+    * @param state Input Firrtl AST
+    * @return A transformed Firrtl AST
+    */
+  override def execute(state: CircuitState): CircuitState = {
+    val resultState = getMyAnnotations(state) match {
+      case Nil => state
+      case myAnnotations =>
+        val sources = getSources(myAnnotations)
+        sources.foreach {
+          case BlackBoxResource(resourceId) =>
+            val name = resourceId.split("/").last
+            val outFile = new File(targetDir, name)
+            BlackBoxSourceHelper.copyResourceToFile(resourceId,outFile)
+            fileList += outFile.getAbsolutePath
+          case BlackBoxInline(name, text) =>
+            val outFile = new File(targetDir, name)
+            val writer = new PrintWriter(outFile)
+            writer.write(text)
+            writer.close()
+            fileList += outFile.getAbsolutePath
+          case _ =>
+        }
+        state
+    }
+    val writer = new PrintWriter(new File(targetDir, BlackBoxSourceHelper.FileListName))
+    writer.write(fileList.map { fileName => s"-v $fileName" }.mkString("\n"))
+    writer.close()
+
+    resultState
+  }
+}
+
+object BlackBoxSourceHelper {
+  val FileListName = "black_box_verilog_files.f"
+  /**
+    * finds the named resource and writes into the directory
+    * @param name the name of the resource
+    * @param file the file to write it into
+    */
+  def copyResourceToFile(name: String, file: File) {
+    val in = getClass.getResourceAsStream(name)
+    if (in == null) {
+      throw new FileNotFoundException(s"Resource '$name'")
+    }
+    val out = new FileOutputStream(file)
+    Iterator.continually(in.read).takeWhile(-1 != _).foreach(out.write)
+    out.close()
+  }
+
+}
diff --git a/src/test/scala/firrtlTests/DriverSpec.scala b/src/test/scala/firrtlTests/DriverSpec.scala
index ff888e2e..5bed2a2b 100644
--- a/src/test/scala/firrtlTests/DriverSpec.scala
+++ b/src/test/scala/firrtlTests/DriverSpec.scala
@@ -136,7 +136,7 @@ class DriverSpec extends FreeSpec with Matchers with BackendCompilationUtilities
     copyResourceToFile("/annotations/SampleAnnotations.anno", annotationsTestFile)
     optionsManager.firrtlOptions.annotations.length should be (0)
     Driver.loadAnnotations(optionsManager)
-    optionsManager.firrtlOptions.annotations.length should be (9)
+    optionsManager.firrtlOptions.annotations.length should be (10) // 9 from circuit plus 1 for targetDir
 
     optionsManager.firrtlOptions.annotations.head.transformClass should be ("firrtl.passes.InlineInstances")
     annotationsTestFile.delete()
diff --git a/src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala b/src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala
new file mode 100644
index 00000000..b037accf
--- /dev/null
+++ b/src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala
@@ -0,0 +1,100 @@
+// See LICENSE for license details.
+
+package firrtlTests.transforms
+
+import java.io.StringWriter
+
+import firrtl.annotations.{Annotation, CircuitName, ModuleName}
+import firrtl.transforms._
+import firrtl.{AnnotationMap, FIRRTLException, Transform, VerilogCompiler}
+import firrtlTests.{HighTransformSpec, LowTransformSpec}
+import org.scalacheck.Test.Failed
+import org.scalatest.{FreeSpec, Matchers, Succeeded}
+
+
+/**
+ * Tests inline instances transformation
+ */
+class BlacklBoxSourceHelperSpec extends FreeSpec with Matchers {
+  "BlackBoxSourceAnnotations" - {
+    val modName = ModuleName("dog", CircuitName("fox"))
+    val resource = "file://somefile.v"
+
+    "should parse and unparse" in {
+
+      val serialized = BlackBoxResource(resource).serialize
+      BlackBoxSource.parse(serialized) match {
+        case Some(BlackBoxResource(id)) =>
+          id should be (resource)
+          Succeeded
+        case _ => Failed
+      }
+    }
+    "should fail on unsupported kinds" in {
+      intercept[FIRRTLException] {
+        BlackBoxSourceAnnotation(modName, "bad value")
+      }
+      BlackBoxSourceAnnotation(modName, BlackBoxResource(resource).serialize).isInstanceOf[Annotation] should be(true)
+    }
+  }
+}
+
+class BlacklBoxSourceHelperTransformSpec extends LowTransformSpec {
+   def transform: Transform = new BlackBoxSourceHelper
+
+  private val moduleName = ModuleName("Top", CircuitName("Top"))
+  private val input = """
+                        |circuit Top :
+                        |
+                        |  extmodule AdderExtModule :
+                        |    input foo : UInt<16>
+                        |    output bar : UInt<16>
+                        |
+                        |    defname = BBFAdd
+                        |
+                        |  module Top :
+                        |   input x : UInt<16>
+                        |   output y : UInt<16>
+                        |
+                        |   inst a1 of AdderExtModule
+                        |   a1.foo <= x
+                        |   y <= a1.bar
+                      """.stripMargin
+  private val output = """
+                        |circuit Top :
+                        |
+                        |  extmodule AdderExtModule :
+                        |    input foo : UInt<16>
+                        |    output bar : UInt<16>
+                        |
+                        |    defname = BBFAdd
+                        |
+                        |  module Top :
+                        |   input x : UInt<16>
+                        |   output y : UInt<16>
+                        |
+                        |   inst a1 of AdderExtModule
+                        |   y <= a1.bar
+                        |   a1.foo <= x
+                      """.stripMargin
+
+  "annotated external modules" should "appear in output directory" in {
+
+    val writer = new StringWriter()
+    val aMap = AnnotationMap(Seq(
+      Annotation(moduleName, classOf[BlackBoxSourceHelper], BlackBoxTargetDir("test_run_dir").serialize),
+      Annotation(moduleName, classOf[BlackBoxSourceHelper], BlackBoxResource("/blackboxes/AdderExtModule.v").serialize)
+    ))
+
+    execute(writer, aMap, input, output)
+
+    new java.io.File("test_run_dir/AdderExtModule.v").exists should be (true)
+    new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.FileListName}").exists should be (true)
+  }
+
+  "verilog compiler" should "have BlackBoxSourceHelper transform" in {
+    val verilogCompiler = new VerilogCompiler
+    verilogCompiler.transforms.map { x => x.getClass } should contain (classOf[BlackBoxSourceHelper])
+  }
+}
+