Change firrtl.options API, add Phase

This breaks firrtl.options.Stage into a small type hierarchy:

  * Phase: something that transforms an AnnotationSeq
  * Stage extends Phase: a Phase with a Command Line Interface

Some of the old "common options" (input annotation file and target
directory) are moved into firrtl.options and provided as part of the Stage
class. Stage will automatically preprocess an input annotation sequence to
resolve all input annotation files and add a default target directory.

Minor changes:

  * Adds ViewException
  * Stops mixing in the DoNotTerminateOnExit trait into the default Shell
    parser
  * Add StageOptionsView

Signed-off-by: Schuyler Eldridge <schuyler.eldridge@ibm.com>

12 files changed, 307 insertions, 44 deletions

diff --git a/src/main/scala/firrtl/options/OptionsView.scala b/src/main/scala/firrtl/options/OptionsView.scala
index dade5c56..aacd997e 100644
--- a/src/main/scala/firrtl/options/OptionsView.scala
+++ b/src/main/scala/firrtl/options/OptionsView.scala
@@ -12,7 +12,8 @@ trait OptionsView[T] {
   /** Convert an [[AnnotationSeq]] to some other type
     * @param options some annotations
     */
-  def view(options: AnnotationSeq): Option[T]
+  def view(options: AnnotationSeq): T
+
 }
 
 /** A shim to manage multiple "views" of an [[AnnotationSeq]] */
@@ -23,5 +24,6 @@ object Viewer {
     * @param optionsView a converter of options to the requested type
     * @tparam T the type to which the input [[AnnotationSeq]] should be viewed as
     */
-  def view[T](options: AnnotationSeq)(implicit optionsView: OptionsView[T]): Option[T] = optionsView.view(options)
+  def view[T](options: AnnotationSeq)(implicit optionsView: OptionsView[T]): T = optionsView.view(options)
+
 }
diff --git a/src/main/scala/firrtl/options/Phase.scala b/src/main/scala/firrtl/options/Phase.scala
new file mode 100644
index 00000000..22715bc2
--- /dev/null
+++ b/src/main/scala/firrtl/options/Phase.scala
@@ -0,0 +1,20 @@
+// See LICENSE for license details.
+
+package firrtl.options
+
+import firrtl.AnnotationSeq
+
+/** A transformation of an [[AnnotationSeq]]
+  *
+  * A [[Phase]] forms one block in the Chisel/FIRRTL Hardware Compiler Framework (HCF). Note that a [[Phase]] may
+  * consist of multiple phases internally.
+  */
+abstract class Phase {
+
+  /** A transformation of an [[AnnotationSeq]]
+    * @param annotations some annotations
+    * @return transformed annotations
+    */
+  def transform(annotations: AnnotationSeq): AnnotationSeq
+
+}
diff --git a/src/main/scala/firrtl/options/Shell.scala b/src/main/scala/firrtl/options/Shell.scala
index b9278f30..4fb89450 100644
--- a/src/main/scala/firrtl/options/Shell.scala
+++ b/src/main/scala/firrtl/options/Shell.scala
@@ -19,7 +19,7 @@ case class OptionsException(msg: String, cause: Throwable = null) extends Except
 class Shell(val applicationName: String) {
 
   /** Command line argument parser (OptionParser) with modifications */
-  final val parser = new OptionParser[AnnotationSeq](applicationName) with DoNotTerminateOnExit with DuplicateHandling
+  final val parser = new OptionParser[AnnotationSeq](applicationName) with DuplicateHandling
 
   /** Contains all discovered [[RegisteredLibrary]] */
   lazy val registeredLibraries: Seq[RegisteredLibrary] = {
@@ -60,4 +60,8 @@ class Shell(val applicationName: String) {
       .getOrElse(throw new OptionsException("Failed to parse command line options", new IllegalArgumentException))
   }
 
+  parser.note("Shell Options")
+  Seq( InputAnnotationFileAnnotation(),
+       TargetDirAnnotation() )
+    .map(_.addOptions(parser))
 }
diff --git a/src/main/scala/firrtl/options/Stage.scala b/src/main/scala/firrtl/options/Stage.scala
index cc651943..4e74652f 100644
--- a/src/main/scala/firrtl/options/Stage.scala
+++ b/src/main/scala/firrtl/options/Stage.scala
@@ -4,38 +4,17 @@ package firrtl.options
 
 import firrtl.AnnotationSeq
 
-/** Utilities mixed into something that looks like a [[Stage]] */
-object StageUtils {
-  /** Print a warning message (in yellow)
-    * @param message error message
-    */
-  //scalastyle:off regex
-  def dramaticWarning(message: String): Unit = {
-    println(Console.YELLOW + "-"*78)
-    println(s"Warning: $message")
-    println("-"*78 + Console.RESET)
-  }
+case class StageException(val str: String, cause: Throwable = null) extends RuntimeException(str, cause)
 
-  /** Print an error message (in red)
-    * @param message error message
-    * @note This does not stop the Driver.
-    */
-  //scalastyle:off regex
-  def dramaticError(message: String): Unit = {
-    println(Console.RED + "-"*78)
-    println(s"Error: $message")
-    println("-"*78 + Console.RESET)
-  }
-}
-
-/** A [[Stage]] represents one stage in the FIRRTL hardware compiler framework
+/** A [[Stage]] represents one stage in the FIRRTL hardware compiler framework. A [[Stage]] is, conceptually, a
+  * [[Phase]] that includes a command line interface.
   *
   * The FIRRTL compiler is a stage as well as any frontend or backend that runs before/after FIRRTL. Concretely, Chisel
   * is a [[Stage]] as is FIRRTL's Verilog emitter. Each stage performs a mathematical transformation on an
   * [[AnnotationSeq]] where some input annotations are processed to produce different annotations. Command line options
   * may be pulled in if available.
   */
-abstract class Stage {
+abstract class Stage extends Phase {
 
   /** A utility that helps convert command line options to annotations */
   val shell: Shell
@@ -44,18 +23,33 @@ abstract class Stage {
     * @param annotations input annotations
     * @return output annotations
     */
-  def execute(annotations: AnnotationSeq): AnnotationSeq
+  def run(annotations: AnnotationSeq): AnnotationSeq
+
+  /** Execute this [[Stage]] on some input annotations. Annotations will be read from any input annotation files.
+    * @param annotations input annotations
+    * @return output annotations
+    */
+  final def transform(annotations: AnnotationSeq): AnnotationSeq = {
+    val preprocessing: Seq[Phase] = Seq(
+      phases.GetIncludes,
+      phases.ConvertLegacyAnnotations,
+      phases.AddDefaults )
+
+    val a = preprocessing.foldLeft(annotations)((a, p) => p.transform(a))
+
+    run(a)
+  }
 
   /** Run this [[Stage]] on on a mix of arguments and annotations
     * @param args command line arguments
     * @param initialAnnotations annotation
     * @return output annotations
     */
-  def execute(args: Array[String], annotations: AnnotationSeq): AnnotationSeq =
-    execute(shell.parse(args, annotations))
+  final def execute(args: Array[String], annotations: AnnotationSeq): AnnotationSeq =
+    transform(shell.parse(args, annotations))
 
   /** The main function that serves as this [[Stage]]'s command line interface
     * @param args command line arguments
     */
-  def main(args: Array[String]): Unit = execute(args, Seq.empty)
+  final def main(args: Array[String]): Unit = execute(args, Seq.empty)
 }
diff --git a/src/main/scala/firrtl/options/StageAnnotations.scala b/src/main/scala/firrtl/options/StageAnnotations.scala
new file mode 100644
index 00000000..fdad07c3
--- /dev/null
+++ b/src/main/scala/firrtl/options/StageAnnotations.scala
@@ -0,0 +1,42 @@
+// See LICENSE for license details
+
+package firrtl.options
+
+import firrtl.AnnotationSeq
+import firrtl.annotations.NoTargetAnnotation
+
+import scopt.OptionParser
+
+sealed trait StageOption extends HasScoptOptions
+
+/** Holds a filename containing one or more [[annotations.Annotation]] to be read
+  *  - this is not stored in [[FirrtlExecutionOptions]]
+  *  - set with `-faf/--annotation-file`
+  * @param value input annotation filename
+  */
+case class InputAnnotationFileAnnotation(value: String) extends NoTargetAnnotation with StageOption {
+  def addOptions(p: OptionParser[AnnotationSeq]): Unit = p.opt[String]("annotation-file")
+    .abbr("faf")
+    .unbounded()
+    .valueName("<input-anno-file>")
+    .action( (x, c) => c :+ InputAnnotationFileAnnotation(x) )
+    .text("Used to specify annotation file")
+}
+
+object InputAnnotationFileAnnotation {
+  private [firrtl] def apply(): InputAnnotationFileAnnotation = InputAnnotationFileAnnotation("")
+}
+
+/** Holds the name of the target directory
+  *  - set with `-td/--target-dir`
+  *  - if unset, a [[TargetDirAnnotation]] will be generated with the
+  * @param value target directory name
+  */
+case class TargetDirAnnotation(dir: String = ".") extends NoTargetAnnotation with StageOption {
+  def addOptions(p: OptionParser[AnnotationSeq]): Unit = p.opt[String]("target-dir")
+    .abbr("td")
+    .valueName("<target-directory>")
+    .action( (x, c) => c ++ Seq(TargetDirAnnotation(x)) )
+    .unbounded() // See [Note 1]
+    .text(s"Work directory for intermediate files/blackboxes, default is '.' (current directory)")
+}
diff --git a/src/main/scala/firrtl/options/StageOptions.scala b/src/main/scala/firrtl/options/StageOptions.scala
new file mode 100644
index 00000000..af5ea17b
--- /dev/null
+++ b/src/main/scala/firrtl/options/StageOptions.scala
@@ -0,0 +1,51 @@
+// See LICENSE for license details.
+
+package firrtl.options
+
+import java.io.File
+
+/** Options that every stage shares
+  * @param targetDirName a target (build) directory
+  * @param an input annotation file
+  */
+final case class StageOptions(
+  targetDir:       String      = TargetDirAnnotation().dir,
+  annotationFiles: Seq[String] = Seq.empty ) {
+
+  /** Generate a filename (with an optional suffix) and create any parent directories. Suffix is only added if it is not
+    * already there.
+    * @param filename the name of the file
+    * @param suffix an optional suffix that the file must end in
+    * @return the name of the file
+    * @note the filename may include a path
+    * @throws IllegalArgumentException if the filename is empty or if the suffix doesn't start with a '.'
+    */
+  def getBuildFileName(filename: String, suffix: Option[String] = None): String = {
+    require(filename.nonEmpty, "requested filename must not be empty")
+    require(suffix.isEmpty || suffix.get.startsWith("."), s"suffix must start with '.', but got ${suffix.get}")
+
+    /* Mangle the file in the following ways:
+     *   1. Ensure that the file ends in the requested suffix
+     *   2. Prepend the target directory if this is not an absolute path
+     */
+    val file = {
+      val f = if (suffix.nonEmpty && !filename.endsWith(suffix.get)) {
+        new File(filename + suffix.get)
+      } else {
+        new File(filename)
+      }
+      if (f.isAbsolute) {
+        f
+      } else {
+        new File(targetDir + "/" + f)
+      }
+    }.getCanonicalFile
+
+    val parent = file.getParentFile
+
+    if (!parent.exists) { parent.mkdirs() }
+
+    file.toString
+  }
+
+}
diff --git a/src/main/scala/firrtl/options/StageUtils.scala b/src/main/scala/firrtl/options/StageUtils.scala
new file mode 100644
index 00000000..ebbbdd03
--- /dev/null
+++ b/src/main/scala/firrtl/options/StageUtils.scala
@@ -0,0 +1,33 @@
+// See LICENSE for license details.
+
+package firrtl.options
+
+import java.io.File
+
+/** Utilities related to working with a [[Stage]] */
+object StageUtils {
+  /** Print a warning message (in yellow)
+    * @param message error message
+    */
+  //scalastyle:off regex
+  def dramaticWarning(message: String): Unit = {
+    println(Console.YELLOW + "-"*78)
+    println(s"Warning: $message")
+    println("-"*78 + Console.RESET)
+  }
+
+  /** Print an error message (in red)
+    * @param message error message
+    * @note This does not stop the Driver.
+    */
+  //scalastyle:off regex
+  def dramaticError(message: String): Unit = {
+    println(Console.RED + "-"*78)
+    println(s"Error: $message")
+    println("-"*78 + Console.RESET)
+  }
+
+  // def canonicalFileName(suffix: String, directory: String = TargetDirAnnotation().targetDirName) {
+  // }
+
+}
diff --git a/src/main/scala/firrtl/options/package.scala b/src/main/scala/firrtl/options/package.scala
new file mode 100644
index 00000000..a0dcc194
--- /dev/null
+++ b/src/main/scala/firrtl/options/package.scala
@@ -0,0 +1,18 @@
+// See LICENSE for license details.
+
+package firrtl
+
+package object options {
+
+  implicit object StageOptionsView extends OptionsView[StageOptions] {
+    def view(options: AnnotationSeq): StageOptions = options
+      .collect { case a: StageOption => a }
+      .foldLeft(StageOptions())((c, x) =>
+        x match {
+          case TargetDirAnnotation(a) => c.copy(targetDir = a)
+          case InputAnnotationFileAnnotation(a) => c.copy(annotationFiles = a +: c.annotationFiles)
+        }
+      )
+  }
+
+}
diff --git a/src/main/scala/firrtl/options/phases/AddDefaults.scala b/src/main/scala/firrtl/options/phases/AddDefaults.scala
new file mode 100644
index 00000000..f0749b22
--- /dev/null
+++ b/src/main/scala/firrtl/options/phases/AddDefaults.scala
@@ -0,0 +1,26 @@
+// See LICENSE for license details.
+
+package firrtl.options.phases
+
+import firrtl.AnnotationSeq
+import firrtl.options.{Phase, StageOption, TargetDirAnnotation}
+
+/** Add default annotations for a [[Stage]]
+  *
+  * This currently only adds a [[TargetDirAnnotation]]. This isn't necessary for a [[StageOptionsView]], but downstream
+  * tools may expect a [[TargetDirAnnotation]] to exist.
+  */
+object AddDefaults extends Phase {
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = {
+    var td = true
+    annotations.collect { case a: StageOption => a }.map {
+      case _: TargetDirAnnotation => td = false
+      case _ =>
+    }
+
+    (if (td) Seq(TargetDirAnnotation()) else Seq()) ++
+      annotations
+  }
+
+}
diff --git a/src/main/scala/firrtl/options/phases/ConvertLegacyAnnotations.scala b/src/main/scala/firrtl/options/phases/ConvertLegacyAnnotations.scala
new file mode 100644
index 00000000..37667160
--- /dev/null
+++ b/src/main/scala/firrtl/options/phases/ConvertLegacyAnnotations.scala
@@ -0,0 +1,14 @@
+// See LICENSE for license details.
+
+package firrtl.options.phases
+
+import firrtl.AnnotationSeq
+import firrtl.annotations.LegacyAnnotation
+import firrtl.options.Phase
+
+/** Convert any [[LegacyAnnotation]]s to non-legacy variants */
+object ConvertLegacyAnnotations extends Phase {
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = LegacyAnnotation.convertLegacyAnnos(annotations)
+
+}
diff --git a/src/main/scala/firrtl/options/phases/GetIncludes.scala b/src/main/scala/firrtl/options/phases/GetIncludes.scala
new file mode 100644
index 00000000..8156dbbf
--- /dev/null
+++ b/src/main/scala/firrtl/options/phases/GetIncludes.scala
@@ -0,0 +1,65 @@
+// See LICENSE for license details.
+
+package firrtl.options.phases
+
+import net.jcazevedo.moultingyaml._
+
+import firrtl.AnnotationSeq
+import firrtl.annotations.{AnnotationFileNotFoundException, DeletedAnnotation, JsonProtocol, LegacyAnnotation}
+import firrtl.annotations.AnnotationYamlProtocol._
+import firrtl.options.{InputAnnotationFileAnnotation, Phase, StageUtils}
+
+import java.io.File
+
+import scala.collection.mutable
+import scala.util.{Try, Failure}
+
+/** Recursively expand all [[InputAnnotationFileAnnotation]]s in an [[AnnotationSeq]] */
+object GetIncludes extends Phase {
+
+  /** Read all [[annotations.Annotation]] from a file in JSON or YAML format
+    * @param filename a JSON or YAML file of [[annotations.Annotation]]
+    * @throws annotations.AnnotationFileNotFoundException if the file does not exist
+    */
+  private def readAnnotationsFromFile(filename: String): AnnotationSeq = {
+    val file = new File(filename).getCanonicalFile
+    if (!file.exists) { throw new AnnotationFileNotFoundException(file) }
+    JsonProtocol.deserializeTry(file).recoverWith { case jsonException =>
+      // Try old protocol if new one fails
+      Try {
+        val yaml = io.Source.fromFile(file).getLines().mkString("\n").parseYaml
+        val result = yaml.convertTo[List[LegacyAnnotation]]
+        val msg = s"$file is a YAML file!\n" + (" "*9) + "YAML Annotation files are deprecated! Use JSON"
+        StageUtils.dramaticWarning(msg)
+        result
+      }.orElse { // Propagate original JsonProtocol exception if YAML also fails
+        Failure(jsonException)
+      }
+    }.get
+  }
+
+  /** Recursively read all [[Annotation]]s from any [[InputAnnotationFileAnnotation]]s while making sure that each file is
+    * only read once
+    * @param includeGuard filenames that have already been read
+    * @param annos a sequence of annotations
+    * @return the original annotation sequence with any discovered annotations added
+    */
+  private def getIncludes(includeGuard: mutable.Set[String] = mutable.Set())
+                         (annos: AnnotationSeq): AnnotationSeq = {
+    val phaseName = this.getClass.getName
+    annos.flatMap {
+      case a @ InputAnnotationFileAnnotation(value) =>
+        if (includeGuard.contains(value)) {
+          StageUtils.dramaticWarning("Tried to import the same annotation file twice! (Did you include it twice?)")
+          Seq(DeletedAnnotation(phaseName, a))
+        } else {
+          includeGuard += value
+          DeletedAnnotation(phaseName, a) +: getIncludes(includeGuard)(readAnnotationsFromFile(value))
+        }
+      case x => Seq(x)
+    }
+  }
+
+  def transform(annotations: AnnotationSeq): AnnotationSeq = getIncludes()(annotations)
+
+}
diff --git a/src/test/scala/firrtlTests/options/OptionsViewSpec.scala b/src/test/scala/firrtlTests/options/OptionsViewSpec.scala
index dec6a99f..0953027f 100644
--- a/src/test/scala/firrtlTests/options/OptionsViewSpec.scala
+++ b/src/test/scala/firrtlTests/options/OptionsViewSpec.scala
@@ -26,10 +26,7 @@ class OptionsViewSpec extends FlatSpec with Matchers {
       case _                  => foo
     }
 
-    def view(options: AnnotationSeq): Option[Foo] = {
-      val annoSeq = options.foldLeft(Foo())(append)
-      Some(annoSeq)
-    }
+    def view(options: AnnotationSeq): Foo = options.foldLeft(Foo())(append)
   }
 
   /* An OptionsView that converts an AnnotationSeq to Option[Bar] */
@@ -39,10 +36,7 @@ class OptionsViewSpec extends FlatSpec with Matchers {
       case _                 => bar
     }
 
-    def view(options: AnnotationSeq): Option[Bar] = {
-      val annoSeq = options.foldLeft(Bar())(append)
-      Some(annoSeq)
-    }
+    def view(options: AnnotationSeq): Bar = options.foldLeft(Bar())(append)
   }
 
   behavior of "OptionsView"
@@ -52,10 +46,10 @@ class OptionsViewSpec extends FlatSpec with Matchers {
     val annos = Seq(NameAnnotation("foo"), ValueAnnotation(42))
 
     info("Foo conversion okay")
-    FooView.view(annos) should be (Some(Foo(Some("foo"), Some(42))))
+    FooView.view(annos) should be (Foo(Some("foo"), Some(42)))
 
     info("Bar conversion okay")
-    BarView.view(annos) should be (Some(Bar("foo")))
+    BarView.view(annos) should be (Bar("foo"))
   }
 
   behavior of "Viewer"
@@ -67,9 +61,9 @@ class OptionsViewSpec extends FlatSpec with Matchers {
     val annos = Seq[Annotation]()
 
     info("Foo view okay")
-    view[Foo](annos) should be (Some(Foo(None, None)))
+    view[Foo](annos) should be (Foo(None, None))
 
     info("Bar view okay")
-    view[Bar](annos) should be (Some(Bar()))
+    view[Bar](annos) should be (Bar())
   }
 }