Merge pull request #945 from seldridge/add-phase

- Change firrtl.options API, add Phase

12 files changed, 307 insertions, 44 deletions

diff --git a/src/main/scala/firrtl/options/OptionsView.scala b/src/main/scala/firrtl/options/OptionsView.scala
index dade5c56..aacd997e 100644
--- a/src/main/scala/firrtl/options/OptionsView.scala
+++ b/src/main/scala/firrtl/options/OptionsView.scala
@@ -12,7 +12,8 @@ trait OptionsView[T] {
   /** Convert an [[AnnotationSeq]] to some other type
     * @param options some annotations
     */
-  def view(options: AnnotationSeq): Option[T]
+  def view(options: AnnotationSeq): T
+
 }
 
 /** A shim to manage multiple "views" of an [[AnnotationSeq]] */
@@ -23,5 +24,6 @@ object Viewer {
     * @param optionsView a converter of options to the requested type
     * @tparam T the type to which the input [[AnnotationSeq]] should be viewed as
     */
-  def view[T](options: AnnotationSeq)(implicit optionsView: OptionsView[T]): Option[T] = optionsView.view(options)
+  def view[T](options: AnnotationSeq)(implicit optionsView: OptionsView[T]): T = optionsView.view(options)
+
 }
diff --git a/src/main/scala/firrtl/options/Phase.scala b/src/main/scala/firrtl/options/Phase.scala
new file mode 100644
index 00000000..22715bc2
--- /dev/null
+++ b/src/main/scala/firrtl/options/Phase.scala
@@ -0,0 +1,20 @@
+// See LICENSE for license details.
+
+package firrtl.options
+
+import firrtl.AnnotationSeq
+
+/** A transformation of an [[AnnotationSeq]]
+  *
+  * A [[Phase]] forms one block in the Chisel/FIRRTL Hardware Compiler Framework (HCF). Note that a [[Phase]] may
+  * consist of multiple phases internally.
+  */
+abstract class Phase {
+
+  /** A transformation of an [[AnnotationSeq]]
+    * @param annotations some annotations
+    * @return transformed annotations
+    */
+  def transform(annotations: AnnotationSeq): AnnotationSeq
+
+}
diff --git a/src/main/scala/firrtl/options/Shell.scala b/src/main/scala/firrtl/options/Shell.scala
index b9278f30..4fb89450 100644
--- a/src/main/scala/firrtl/options/Shell.scala
+++ b/src/main/scala/firrtl/options/Shell.scala
@@ -19,7 +19,7 @@ case class OptionsException(msg: String, cause: Throwable = null) extends Except
 class Shell(val applicationName: String) {
 
   /** Command line argument parser (OptionParser) with modifications */
-  final val parser = new OptionParser[AnnotationSeq](applicationName) with DoNotTerminateOnExit with DuplicateHandling
+  final val parser = new OptionParser[AnnotationSeq](applicationName) with DuplicateHandling
 
   /** Contains all discovered [[RegisteredLibrary]] */
   lazy val registeredLibraries: Seq[RegisteredLibrary] = {
@@ -60,4 +60,8 @@ class Shell(val applicationName: String) {
       .getOrElse(throw new OptionsException("Failed to parse command line options", new IllegalArgumentException))
   }
 
+  parser.note("Shell Options")
+  Seq( InputAnnotationFileAnnotation(),
+       TargetDirAnnotation() )
+    .map(_.addOptions(parser))
 }
diff --git a/src/main/scala/firrtl/options/Stage.scala b/src/main/scala/firrtl/options/Stage.scala
index cc651943..4e74652f 100644
--- a/src/main/scala/firrtl/options/Stage.scala
+++ b/src/main/scala/firrtl/options/Stage.scala
@@ -4,38 +4,17 @@ package firrtl.options
 
 import firrtl.AnnotationSeq
 
-/** Utilities mixed into something that looks like a [[Stage]] */
-object StageUtils {
-  /** Print a warning message (in yellow)
-    * @param message error message
-    */
-  //scalastyle:off regex
-  def dramaticWarning(message: String): Unit = {
-    println(Console.YELLOW + "-"*78)
-    println(s"Warning: $message")
-    println("-"*78 + Console.RESET)
-  }
+case class StageException(val str: String, cause: Throwable = null) extends RuntimeException(str, cause)
 
-  /** Print an error message (in red)
-    * @param message error message
-    * @note This does not stop the Driver.
-    */
-  //scalastyle:off regex
-  def dramaticError(message: String): Unit = {
-    println(Console.RED + "-"*78)
-    println(s"Error: $message")
-    println("-"*78 + Console.RESET)
-  }
-}
-
-/** A [[Stage]] represents one stage in the FIRRTL hardware compiler framework
+/** A [[Stage]] represents one stage in the FIRRTL hardware compiler framework. A [[Stage]] is, conceptually, a
+  * [[Phase]] that includes a command line interface.
   *
   * The FIRRTL compiler is a stage as well as any frontend or backend that runs before/after FIRRTL. Concretely, Chisel
   * is a [[Stage]] as is FIRRTL's Verilog emitter. Each stage performs a mathematical transformation on an
   * [[AnnotationSeq]] where some input annotations are processed to produce different annotations. Command line options
   * may be pulled in if available.
   */
-abstract class Stage {
+abstract class Stage extends Phase {
 
   /** A utility that helps convert command line options to annotations */
   val shell: Shell
@@ -44,18 +23,33 @@ abstract class Stage {
     * @param annotations input annotations
     * @return output annotations
     */
-  def execute(annotations: AnnotationSeq): AnnotationSeq
+  def run(annotations: AnnotationSeq): AnnotationSeq
+
+  /** Execute this [[Stage]] on some input annotations. Annotations will be read from any input annotation files.
+    * @param annotations input annotations
+    * @return output annotations
+    */
+  final def transform(annotations: AnnotationSeq): AnnotationSeq = {
+    val preprocessing: Seq[Phase] = Seq(
+      phases.GetIncludes,
+      phases.ConvertLegacyAnnotations,
+      phases.AddDefaults )
+
+    val a = preprocessing.foldLeft(annotations)((a, p) => p.transform(a))
+
+    run(a)
+  }
 
   /** Run this [[Stage]] on on a mix of arguments and annotations
     * @param args command line arguments
     * @param initialAnnotations annotation
     * @return output annotations
     */
-  def execute(args: Array[String], annotations: AnnotationSeq): AnnotationSeq =
-    execute(shell.parse(args, annotations))
+  final def execute(args: Array[String], annotations: AnnotationSeq): AnnotationSeq =
+    transform(shell.parse(args, annotations))
 
   /** The main function that serves as this [[Stage]]'s command line interface
     * @param args command line arguments
     */
-  def main(args: Array[String]): Unit = execute(args, Seq.empty)
+  final def main(args: Array[String]): Unit = execute(args, Seq.empty)
 }
diff --git a/src/main/scala/firrtl/options/StageAnnotations.scala b/src/main/scala/firrtl/options/StageAnnotations.scala
new file mode 100644
index 00000000..fdad07c3
--- /dev/null
+++ b/src/main/scala/firrtl/options/StageAnnotations.scala
@@ -0,0 +1,42 @@
+// See LICENSE for license details
+
+package firrtl.options
+
+import firrtl.AnnotationSeq
+import firrtl.annotations.NoTargetAnnotation
+
+import scopt.OptionParser
+
+sealed trait StageOption extends HasScoptOptions
+
+/** Holds a filename containing one or more [[annotations.Annotation]] to be read
+  *  - this is not stored in [[FirrtlExecutionOptions]]
+  *  - set with `-faf/--annotation-file`
+  * @param value input annotation filename
+  */
+case class InputAnnotationFileAnnotation(value: String) extends NoTargetAnnotation with StageOption {
+  def addOptions(p: OptionParser[AnnotationSeq]): Unit = p.opt[String]("annotation-file")
+    .abbr("faf")
+    .unbounded()
+    .valueName("<input-anno-file>")
+    .action( (x, c) => c :+ InputAnnotationFileAnnotation(x) )
+    .text("Used to specify annotation file")
+}
+
+object InputAnnotationFileAnnotation {
+  private [firrtl] def apply(): InputAnnotationFileAnnotation = InputAnnotationFileAnnotation("")
+}
+
+/** Holds the name of the target directory
+  *  - set with `-td/--target-dir`
+  *  - if unset, a [[TargetDirAnnotation]] will be generated with the
+  * @param value target directory name
+  */
+case class TargetDirAnnotation(dir: String = ".") extends NoTargetAnnotation with StageOption {
+  def addOptions(p: OptionParser[AnnotationSeq]): Unit = p.opt[String]("target-dir")
+    .abbr("td")
+    .valueName("<target-directory>")
+    .action( (x, c) => c ++ Seq(TargetDirAnnotation(x)) )
+    .unbounded() // See [Note 1]
+    .text(s"Work directory for intermediate files/blackboxes, default is '.' (current directory)")
+}
diff --git a/src/main/scala/firrtl/options/StageOptions.scala b/src/main/scala/firrtl/options/StageOptions.scala
new file mode 100644
index 00000000..af5ea17b
--- /dev/null
+++ b/src/main/scala/firrtl/options/StageOptions.scala
@@ -0,0 +1,51 @@
+// See LICENSE for license details.
+
+package firrtl.options
+
+import java.io.File
+
+/** Options that every stage shares
+  * @param targetDirName a target (build) directory
+  * @param an input annotation file
+  */
+final case class StageOptions(
+  targetDir:       String      = TargetDirAnnotation().dir,
+  annotationFiles: Seq[String] = Seq.empty ) {
+
+  /** Generate a filename (with an optional suffix) and create any parent directories. Suffix is only added if it is not
+    * already there.
+    * @param filename the name of the file
+    * @param suffix an optional suffix that the file must end in
+    * @return the name of the file
+    * @note the filename may include a path
+    * @throws IllegalArgumentException if the filename is empty or if the suffix doesn't start with a '.'
+    */
+  def getBuildFileName(filename: String, suffix: Option[String] = None): String = {
+    require(filename.nonEmpty, "requested filename must not be empty")
+    require(suffix.isEmpty || suffix.get.startsWith("."), s"suffix must start with '.', but got ${suffix.get}")
+
+    /* Mangle the file in the following ways:
+     *   1. Ensure that the file ends in the requested suffix
+     *   2. Prepend the target directory if this is not an absolute path
+     */
+    val file = {
+      val f = if (suffix.nonEmpty && !filename.endsWith(suffix.get)) {
+        new File(filename + suffix.get)
+      } else {
+        new File(filename)
+      }
+      if (f.isAbsolute) {
+        f
+      } else {
+        new File(targetDir + "/" + f)
+      }
+    }.getCanonicalFile
+
+    val parent = file.getParentFile
+
+    if (!parent.exists) { parent.mkdirs() }
+
+    file.toString
+  }
+
+}
diff --git a/src/main/scala/firrtl/options/StageUtils.scala b/src/main/scala/firrtl/options/StageUtils.scala
new file mode 100644
index 00000000..ebbbdd03
--- /dev/null
+++ b/src/main/scala/firrtl/options/StageUtils.scala
@@ -0,0 +1,33 @@
+// See LICENSE for license details.
+
+package firrtl.options
+
+import java.io.File
+
+/** Utilities related to working with a [[Stage]] */
+object StageUtils {
+  /** Print a warning message (in yellow)
+    * @param message error message
+    */
+  //scalastyle:off regex
+  def dramaticWarning(message: String): Unit = {
+    println(Console.YELLOW + "-"*78)
+    println(s"Warning: $message")
+    println("-"*78 + Console.RESET)
+  }
+
+  /** Print an error message (in red)
+    * @param message error message
+    * @note This does not stop the Driver.
+    */
+  //scalastyle:off regex
+  def dramaticError(message: String): Unit = {
+    println(Console.RED + "-"*78)
+    println(s"Error: $message")
+    println("-"*78 + Console.RESET)
+  }
+
+  // def canonicalFileName(suffix: String, directory: String = TargetDirAnnotation().targetDirName) {
+  // }
+
+}
diff --git a/src/main/scala/firrtl/options/package.scala b/src/main/scala/firrtl/options/package.scala
new file mode 100644
index 00000000..a0dcc194
--- /dev/null
+++ b/src/main/scala/firrtl/options/package.scala
@@ -0,0 +1,18 @@
+// See LICENSE for license details.
+
+package firrtl
+
+package object options {
+
+  implicit object StageOptionsView extends OptionsView[StageOptions] {
+    def view(options: AnnotationSeq): StageOptions = options
+      .collect { case a: StageOption => a }
+      .foldLeft(StageOptions())((c, x) =>
+        x match {
+          case TargetDirAnnotation(a) => c.copy(targetDir = a)
+          case InputAnnotationFileAnnotation(a) => c.copy(annotationFiles = a +: c.annotationFiles)
+        }
+      )
+  }
+
+}
diff --git a/src/main/scala/firrtl/options/phases/AddDefaults.scala b/src/main/scala/firrtl/options/phases/AddDefaults.scala
new file mode 100644
index 00000000..f0749b22
--- /dev/null
+++ b/src/main/scala/firrtl/options/phases/AddDefaults.scala
@@ -0,0 +1,26 @@
+// See LICENSE for license details.
+
+package firrtl.options.phases
+
+import firrtl.AnnotationSeq
+import firrtl.options.{Phase, StageOption, TargetDirAnnotation}
+
+/** Add default annotations for a [[Stage]]
+  *
+  * This currently only adds a [[TargetDirAnnotation]]. This isn't necessary for a [[StageOptionsView]], but downstream
+  * tools may expect a [[TargetDirAnnotation]] to exist.
+  */
+object AddDefaults extends Phase {
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = {
+    var td = true
+    annotations.collect { case a: StageOption => a }.map {
+      case _: TargetDirAnnotation => td = false
+      case _ =>
+    }
+
+    (if (td) Seq(TargetDirAnnotation()) else Seq()) ++
+      annotations
+  }
+
+}
diff --git a/src/main/scala/firrtl/options/phases/ConvertLegacyAnnotations.scala b/src/main/scala/firrtl/options/phases/ConvertLegacyAnnotations.scala
new file mode 100644
index 00000000..37667160
--- /dev/null
+++ b/src/main/scala/firrtl/options/phases/ConvertLegacyAnnotations.scala
@@ -0,0 +1,14 @@
+// See LICENSE for license details.
+
+package firrtl.options.phases
+
+import firrtl.AnnotationSeq
+import firrtl.annotations.LegacyAnnotation
+import firrtl.options.Phase
+
+/** Convert any [[LegacyAnnotation]]s to non-legacy variants */
+object ConvertLegacyAnnotations extends Phase {
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = LegacyAnnotation.convertLegacyAnnos(annotations)
+
+}
diff --git a/src/main/scala/firrtl/options/phases/GetIncludes.scala b/src/main/scala/firrtl/options/phases/GetIncludes.scala
new file mode 100644
index 00000000..8156dbbf
--- /dev/null
+++ b/src/main/scala/firrtl/options/phases/GetIncludes.scala
@@ -0,0 +1,65 @@
+// See LICENSE for license details.
+
+package firrtl.options.phases
+
+import net.jcazevedo.moultingyaml._
+
+import firrtl.AnnotationSeq
+import firrtl.annotations.{AnnotationFileNotFoundException, DeletedAnnotation, JsonProtocol, LegacyAnnotation}
+import firrtl.annotations.AnnotationYamlProtocol._
+import firrtl.options.{InputAnnotationFileAnnotation, Phase, StageUtils}
+
+import java.io.File
+
+import scala.collection.mutable
+import scala.util.{Try, Failure}
+
+/** Recursively expand all [[InputAnnotationFileAnnotation]]s in an [[AnnotationSeq]] */
+object GetIncludes extends Phase {
+
+  /** Read all [[annotations.Annotation]] from a file in JSON or YAML format
+    * @param filename a JSON or YAML file of [[annotations.Annotation]]
+    * @throws annotations.AnnotationFileNotFoundException if the file does not exist
+    */
+  private def readAnnotationsFromFile(filename: String): AnnotationSeq = {
+    val file = new File(filename).getCanonicalFile
+    if (!file.exists) { throw new AnnotationFileNotFoundException(file) }
+    JsonProtocol.deserializeTry(file).recoverWith { case jsonException =>
+      // Try old protocol if new one fails
+      Try {
+        val yaml = io.Source.fromFile(file).getLines().mkString("\n").parseYaml
+        val result = yaml.convertTo[List[LegacyAnnotation]]
+        val msg = s"$file is a YAML file!\n" + (" "*9) + "YAML Annotation files are deprecated! Use JSON"
+        StageUtils.dramaticWarning(msg)
+        result
+      }.orElse { // Propagate original JsonProtocol exception if YAML also fails
+        Failure(jsonException)
+      }
+    }.get
+  }
+
+  /** Recursively read all [[Annotation]]s from any [[InputAnnotationFileAnnotation]]s while making sure that each file is
+    * only read once
+    * @param includeGuard filenames that have already been read
+    * @param annos a sequence of annotations
+    * @return the original annotation sequence with any discovered annotations added
+    */
+  private def getIncludes(includeGuard: mutable.Set[String] = mutable.Set())
+                         (annos: AnnotationSeq): AnnotationSeq = {
+    val phaseName = this.getClass.getName
+    annos.flatMap {
+      case a @ InputAnnotationFileAnnotation(value) =>
+        if (includeGuard.contains(value)) {
+          StageUtils.dramaticWarning("Tried to import the same annotation file twice! (Did you include it twice?)")
+          Seq(DeletedAnnotation(phaseName, a))
+        } else {
+          includeGuard += value
+          DeletedAnnotation(phaseName, a) +: getIncludes(includeGuard)(readAnnotationsFromFile(value))
+        }
+      case x => Seq(x)
+    }
+  }
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = getIncludes()(annotations)
+
+}
diff --git a/src/test/scala/firrtlTests/options/OptionsViewSpec.scala b/src/test/scala/firrtlTests/options/OptionsViewSpec.scala
index dec6a99f..0953027f 100644
--- a/src/test/scala/firrtlTests/options/OptionsViewSpec.scala
+++ b/src/test/scala/firrtlTests/options/OptionsViewSpec.scala
@@ -26,10 +26,7 @@ class OptionsViewSpec extends FlatSpec with Matchers {
       case _                  => foo
     }
 
-    def view(options: AnnotationSeq): Option[Foo] = {
-      val annoSeq = options.foldLeft(Foo())(append)
-      Some(annoSeq)
-    }
+    def view(options: AnnotationSeq): Foo = options.foldLeft(Foo())(append)
   }
 
   /* An OptionsView that converts an AnnotationSeq to Option[Bar] */
@@ -39,10 +36,7 @@ class OptionsViewSpec extends FlatSpec with Matchers {
       case _                 => bar
     }
 
-    def view(options: AnnotationSeq): Option[Bar] = {
-      val annoSeq = options.foldLeft(Bar())(append)
-      Some(annoSeq)
-    }
+    def view(options: AnnotationSeq): Bar = options.foldLeft(Bar())(append)
   }
 
   behavior of "OptionsView"
@@ -52,10 +46,10 @@ class OptionsViewSpec extends FlatSpec with Matchers {
     val annos = Seq(NameAnnotation("foo"), ValueAnnotation(42))
 
     info("Foo conversion okay")
-    FooView.view(annos) should be (Some(Foo(Some("foo"), Some(42))))
+    FooView.view(annos) should be (Foo(Some("foo"), Some(42)))
 
     info("Bar conversion okay")
-    BarView.view(annos) should be (Some(Bar("foo")))
+    BarView.view(annos) should be (Bar("foo"))
   }
 
   behavior of "Viewer"
@@ -67,9 +61,9 @@ class OptionsViewSpec extends FlatSpec with Matchers {
     val annos = Seq[Annotation]()
 
     info("Foo view okay")
-    view[Foo](annos) should be (Some(Foo(None, None)))
+    view[Foo](annos) should be (Foo(None, None))
 
     info("Bar view okay")
-    view[Bar](annos) should be (Some(Bar()))
+    view[Bar](annos) should be (Bar())
   }
 }