Protobuf (#832)

Add support for ProtoBuf serialization and deserialization

* Add support for additional features in .proto description

Features added: Info, Fixed[Type|Literal], AnalogType, Attach, Params

* Add support for .pb input files

This involves an API change where FIRRTL no longer implicitly adds .fir
to input file names

3 files changed, 173 insertions, 9 deletions

diff --git a/src/test/scala/firrtlTests/DriverSpec.scala b/src/test/scala/firrtlTests/DriverSpec.scala
index 4afd5674..4cc7bc90 100644
--- a/src/test/scala/firrtlTests/DriverSpec.scala
+++ b/src/test/scala/firrtlTests/DriverSpec.scala
@@ -2,22 +2,18 @@
 
 package firrtlTests
 
-import java.io.{File, FileWriter}
-import org.scalatest.{FreeSpec, Matchers}
+import java.io.{File, FileInputStream, FileWriter}
 
+import org.scalatest.{FreeSpec, Matchers}
 import firrtl.passes.{InlineAnnotation, InlineInstances}
-import firrtl.passes.memlib.{
-  InferReadWrite,
-  InferReadWriteAnnotation,
-  ReplSeqMem,
-  ReplSeqMemAnnotation
-}
+import firrtl.passes.memlib.{InferReadWrite, InferReadWriteAnnotation, ReplSeqMem, ReplSeqMemAnnotation}
 import firrtl.transforms.BlackBoxTargetDirAnno
 import firrtl._
 import firrtl.annotations._
 import firrtl.util.BackendCompilationUtilities
 
-import scala.util.{Try, Success, Failure}
+import scala.io.Source
+import scala.util.{Failure, Success, Try}
 
 class ExceptingTransform extends Transform {
   def inputForm = HighForm
@@ -416,6 +412,28 @@ class DriverSpec extends FreeSpec with Matchers with BackendCompilationUtilities
     }
   }
 
+  "The Driver is sensitive to the file extension of input files" - {
+    val design = "GCDTester"
+    val outputDir = createTestDirectory("DriverFileExtensionSensitivity")
+    val verilogFromFir = new File(outputDir, s"$design.fromfir.v")
+    val verilogFromPb = new File(outputDir, s"$design.frompb.v")
+    val commonArgs = Array("-X", "verilog", "--info-mode", "use")
+    ".fir means FIRRTL file" in {
+      val inFile = new File(getClass.getResource(s"/integration/$design.fir").getFile)
+      val args = Array("-i", inFile.getAbsolutePath, "-o", verilogFromFir.getAbsolutePath) ++ commonArgs
+      Driver.execute(args)
+    }
+    ".pb means ProtoBuf file" in {
+      val inFile = new File(getClass.getResource(s"/integration/$design.pb").getFile)
+      val args = Array("-i", inFile.getAbsolutePath, "-o", verilogFromPb.getAbsolutePath) ++ commonArgs
+      Driver.execute(args)
+    }
+    "Both paths do the same thing" in {
+      val s1 = Source.fromFile(verilogFromFir).mkString
+      val s2 = Source.fromFile(verilogFromPb).mkString
+      s1 should equal (s2)
+    }
+  }
 
   "Directory deleter is handy for cleaning up after tests" - {
     "for example making a directory tree, and deleting it looks like" in {
diff --git a/src/test/scala/firrtlTests/FirrtlSpec.scala b/src/test/scala/firrtlTests/FirrtlSpec.scala
index 861d1745..01ae0431 100644
--- a/src/test/scala/firrtlTests/FirrtlSpec.scala
+++ b/src/test/scala/firrtlTests/FirrtlSpec.scala
@@ -101,6 +101,12 @@ trait FirrtlMatchers extends Matchers {
     require(!s.contains("\n"))
     s.replaceAll("\\s+", " ").trim
   }
+  /** Helper to make circuits that are the same appear the same */
+  def canonicalize(circuit: Circuit): Circuit = {
+    import firrtl.Mappers._
+    def onModule(mod: DefModule) = mod.map(firrtl.Utils.squashEmpty)
+    circuit.map(onModule)
+  }
   def parse(str: String) = Parser.parse(str.split("\n").toIterator, UseInfo)
   /** Helper for executing tests
     * compiler will be run on input then emitted result will each be split into
diff --git a/src/test/scala/firrtlTests/ProtoBufSpec.scala b/src/test/scala/firrtlTests/ProtoBufSpec.scala
new file mode 100644
index 00000000..2a60eab5
--- /dev/null
+++ b/src/test/scala/firrtlTests/ProtoBufSpec.scala
@@ -0,0 +1,140 @@
+// See LICENSE for license details.
+
+package firrtlTests
+
+import java.io.{ByteArrayInputStream, ByteArrayOutputStream}
+
+import firrtl.FirrtlProtos.Firrtl
+import firrtl._
+import firrtl.ir._
+import firrtl.Mappers._
+
+class ProtoBufSpec extends FirrtlFlatSpec {
+
+  /** Tests in src/test/resource/ in .fir format
+    *
+    * @note These tests rely on the ANTLR Parser
+    */
+  case class FirrtlResourceTest(name: String, resourceDir: String)
+
+  val firrtlResourceTests = List(
+    FirrtlResourceTest("GCDTester", "/integration"),
+    FirrtlResourceTest("RightShiftTester", "/integration"),
+    FirrtlResourceTest("MemTester", "/integration"),
+    FirrtlResourceTest("PipeTester", "/integration"),
+    FirrtlResourceTest("Rob", "/regress"),
+    FirrtlResourceTest("RocketCore", "/regress"),
+    FirrtlResourceTest("ICache", "/regress"),
+    FirrtlResourceTest("FPU", "/regress")
+  )
+
+  for (FirrtlResourceTest(name, dir) <- firrtlResourceTests) {
+    s"$name" should "work with Protobuf serialization and deserialization" in {
+      val stream = getClass.getResourceAsStream(s"$dir/$name.fir")
+      val circuit = parse(scala.io.Source.fromInputStream(stream).getLines.mkString("\n"))
+
+      // Test ToProto and FromProto
+      val protobuf = proto.ToProto.convert(circuit)
+      val pcircuit = proto.FromProto.convert(protobuf)
+      canonicalize(circuit).serialize should equal(canonicalize(pcircuit).serialize)
+
+      // Test ProtoBuf generated serialization and deserialization
+      val ostream = new java.io.ByteArrayOutputStream()
+      protobuf.writeTo(ostream)
+      val istream = new java.io.ByteArrayInputStream(ostream.toByteArray)
+      val cistream = com.google.protobuf.CodedInputStream.newInstance(istream)
+      cistream.setRecursionLimit(Integer.MAX_VALUE)
+      val protobuf2 = firrtl.FirrtlProtos.Firrtl.parseFrom(cistream)
+      protobuf2 should equal (protobuf)
+
+      // Test that our faster serialization matches generated serialization
+      val ostream2 = new java.io.ByteArrayOutputStream
+      proto.ToProto.writeToStream(ostream2, circuit)
+      ostream2.toByteArray.toList should equal (ostream.toByteArray.toList)
+    }
+  }
+
+  // ********** Focused Tests **********
+  // The goal is to fill coverage holes left after the above
+
+  behavior of "ProtoBuf serialization and deserialization"
+  import firrtl.proto._
+
+  it should "support UnknownWidth" in {
+    // Note that this has to be handled in the parent object so we need to test everything that has a width
+    val uint = ir.UIntType(ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(uint).build) should equal (uint)
+
+    val sint = ir.SIntType(ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(sint).build) should equal (sint)
+
+    val ftpe = ir.FixedType(ir.UnknownWidth, ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(ftpe).build) should equal (ftpe)
+
+    val atpe = ir.AnalogType(ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(atpe).build) should equal (atpe)
+
+    val ulit = ir.UIntLiteral(123, ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(ulit).build) should equal (ulit)
+
+    val slit = ir.SIntLiteral(-123, ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(slit).build) should equal (slit)
+
+    val flit = ir.FixedLiteral(-123, ir.UnknownWidth, ir.UnknownWidth)
+    FromProto.convert(ToProto.convert(flit).build) should equal (flit)
+  }
+
+  it should "support all Primops" in {
+    val builtInOps = PrimOps.listing.map(PrimOps.fromString(_))
+    for (op <- builtInOps) {
+      val expr = DoPrim(op, List.empty, List.empty, ir.UnknownType)
+      FromProto.convert(ToProto.convert(expr).build) should equal (expr)
+    }
+  }
+
+  it should "support all ExtModule features (except info which isn't yet supported by Chisel)" in {
+    val ports = Seq(
+      Port(ir.NoInfo, "port1", ir.Input, ir.UIntType(ir.IntWidth(8))),
+      Port(ir.NoInfo, "port2", ir.Output, ir.SIntType(ir.IntWidth(8)))
+    )
+    val params = Seq(
+      IntParam("param1", BigInt(Long.MinValue)),
+      DoubleParam("param2", Double.NegativeInfinity),
+      StringParam("param3", ir.StringLit("quite the string!")),
+      RawStringParam("param4", "get some raw strings")
+    )
+    val ext = ir.ExtModule(ir.NoInfo, "MyModule", ports, "DefNameHere", params)
+    FromProto.convert(ToProto.convert(ext).build) should equal (ext)
+  }
+
+  it should "supported FixedType" in {
+    val ftpe = ir.FixedType(IntWidth(8), IntWidth(4))
+    FromProto.convert(ToProto.convert(ftpe).build) should equal (ftpe)
+  }
+
+  it should "supported FixedLiteral" in {
+    val flit = ir.FixedLiteral(3, IntWidth(8), IntWidth(4))
+    FromProto.convert(ToProto.convert(flit).build) should equal (flit)
+  }
+
+  it should "support Analog and Attach" in {
+    val analog = ir.AnalogType(IntWidth(8))
+    FromProto.convert(ToProto.convert(analog).build) should equal (analog)
+
+    val attach = ir.Attach(ir.NoInfo, Seq(Reference("hi", ir.UnknownType)))
+    FromProto.convert(ToProto.convert(attach).head.build) should equal (attach)
+  }
+
+  // Regression tests were generated before Chisel could emit else
+  it should "support whens with elses" in {
+    val expr = Reference("hi", ir.UnknownType)
+    val stmt = Connect(ir.NoInfo, expr, expr)
+    val when = ir.Conditionally(ir.NoInfo, expr, stmt, stmt)
+    FromProto.convert(ToProto.convert(when).head.build) should equal (when)
+  }
+
+  it should "support SIntLiteral with a width" in {
+    val slit = ir.SIntLiteral(-123)
+    FromProto.convert(ToProto.convert(slit).build) should equal (slit)
+  }
+}