From 73bd4ee6b9b510725b692c33e075362a19512d2c Mon Sep 17 00:00:00 2001 From: Jack Koenig Date: Fri, 20 Aug 2021 11:30:27 -0700 Subject: Remove chisel3's own firrtl Emitter, use firrtl Serializer This will be slightly slower as it involves converting from Chisel modules to FIRRTL modules before turning them into Strings. This cost is somewhat mitigated by doing that conversion lazily such that we never materialize the entire firrtl Circuit in memory, only 1 module at a time. --- .../scala/chisel3/internal/firrtl/Emitter.scala | 192 +-------------------- src/test/scala/chiselTests/PrintableSpec.scala | 7 +- src/test/scala/chiselTests/VecLiteralSpec.scala | 8 +- .../verification/VerificationSpec.scala | 10 +- 4 files changed, 17 insertions(+), 200 deletions(-) (limited to 'src') diff --git a/src/main/scala/chisel3/internal/firrtl/Emitter.scala b/src/main/scala/chisel3/internal/firrtl/Emitter.scala index 47849d91..53329908 100644 --- a/src/main/scala/chisel3/internal/firrtl/Emitter.scala +++ b/src/main/scala/chisel3/internal/firrtl/Emitter.scala @@ -1,194 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 package chisel3.internal.firrtl -import chisel3._ -import chisel3.experimental.{Interval, _} -import chisel3.internal.BaseBlackBox +import firrtl.{ir => fir} private[chisel3] object Emitter { - def emit(circuit: Circuit): String = new Emitter(circuit).toString -} - -private class Emitter(circuit: Circuit) { - override def toString: String = res.toString - - private def emitPort(e: Port, topDir: SpecifiedDirection=SpecifiedDirection.Unspecified): String = { - val resolvedDir = SpecifiedDirection.fromParent(topDir, e.dir) - val dirString = resolvedDir match { - case SpecifiedDirection.Unspecified | SpecifiedDirection.Output => "output" - case SpecifiedDirection.Flip | SpecifiedDirection.Input => "input" - } - val clearDir = resolvedDir match { - case SpecifiedDirection.Input | SpecifiedDirection.Output => true - case SpecifiedDirection.Unspecified | SpecifiedDirection.Flip => false - } - s"$dirString ${e.id.getRef.name} : ${emitType(e.id, clearDir)}" - } - - private def emitType(d: Data, clearDir: Boolean = false): String = d match { - case d: Clock => "Clock" - case _: AsyncReset => "AsyncReset" - case _: ResetType => "Reset" - case d: chisel3.experimental.EnumType => s"UInt${d.width}" - case d: UInt => s"UInt${d.width}" - case d: SInt => s"SInt${d.width}" - case d: FixedPoint => s"Fixed${d.width}${d.binaryPoint}" - case d: Interval => - val binaryPointString = d.binaryPoint match { - case UnknownBinaryPoint => "" - case KnownBinaryPoint(value) => s".$value" - } - d.toType - case d: Analog => s"Analog${d.width}" - case d: Vec[_] => s"${emitType(d.sample_element, clearDir)}[${d.length}]" - case d: Record => { - val childClearDir = clearDir || - d.specifiedDirection == SpecifiedDirection.Input || d.specifiedDirection == SpecifiedDirection.Output - def eltPort(elt: Data): String = (childClearDir, firrtlUserDirOf(elt)) match { - case (true, _) => - s"${elt.getRef.name} : ${emitType(elt, true)}" - case (false, SpecifiedDirection.Unspecified | SpecifiedDirection.Output) => - s"${elt.getRef.name} : ${emitType(elt, false)}" - case (false, SpecifiedDirection.Flip | SpecifiedDirection.Input) => - s"flip ${elt.getRef.name} : ${emitType(elt, false)}" - } - d.elements.toIndexedSeq.reverse.map(e => eltPort(e._2)).mkString("{", ", ", "}") - } - } - - private def firrtlUserDirOf(d: Data): SpecifiedDirection = d match { - case d: Vec[_] => - SpecifiedDirection.fromParent(d.specifiedDirection, firrtlUserDirOf(d.sample_element)) - case d => d.specifiedDirection - } - - private def emit(e: Command, ctx: Component): String = { - val firrtlLine = e match { - case e: DefPrim[_] => s"node ${e.name} = ${e.op.name}(${e.args.map(_.fullName(ctx)).mkString(", ")})" - case e: DefWire => s"wire ${e.name} : ${emitType(e.id)}" - case e: DefReg => s"reg ${e.name} : ${emitType(e.id)}, ${e.clock.fullName(ctx)}" - case e: DefRegInit => s"reg ${e.name} : ${emitType(e.id)}, ${e.clock.fullName(ctx)} with : (reset => (${e.reset.fullName(ctx)}, ${e.init.fullName(ctx)}))" - case e: DefMemory => s"cmem ${e.name} : ${emitType(e.t)}[${e.size}]" - case e: DefSeqMemory => s"smem ${e.name} : ${emitType(e.t)}[${e.size}], ${e.readUnderWrite}" - case e: DefMemPort[_] => s"${e.dir} mport ${e.name} = ${e.source.fullName(ctx)}[${e.index.fullName(ctx)}], ${e.clock.fullName(ctx)}" - case e: Connect => s"${e.loc.fullName(ctx)} <= ${e.exp.fullName(ctx)}" - case e: BulkConnect => s"${e.loc1.fullName(ctx)} <- ${e.loc2.fullName(ctx)}" - case e: Attach => e.locs.map(_.fullName(ctx)).mkString("attach (", ", ", ")") - case e: Stop => s"stop(${e.clock.fullName(ctx)}, UInt<1>(1), ${e.ret})" - case e: chisel3.internal.firrtl.Printf => - val (fmt, args) = e.pable.unpack(ctx) - val printfArgs = Seq(e.clock.fullName(ctx), "UInt<1>(1)", - "\"" + printf.format(fmt) + "\"") ++ args - (printfArgs mkString ("printf(", ", ", ")")) + s": ${e.name}" - case e: Verification[_] => - s"""${e.op}(${e.clock.fullName(ctx)}, ${e.predicate.fullName(ctx)}, UInt<1>(1), "${printf.format(e.message)}") : ${e.name}""" - case e: DefInvalid => s"${e.arg.fullName(ctx)} is invalid" - case e: DefInstance => s"inst ${e.name} of ${e.id.name}" - case w: WhenBegin => - // When consequences are always indented - indent() - s"when ${w.pred.fullName(ctx)} :" - case w: WhenEnd => - // If a when has no else, the indent level must be reset to the enclosing block - unindent() - if (!w.hasAlt) { for (i <- 0 until w.firrtlDepth) { unindent() } } - s"skip" - case a: AltBegin => - // Else blocks are always indented - indent() - s"else :" - case o: OtherwiseEnd => - // Chisel otherwise: ends all FIRRTL associated a Chisel when, resetting indent level - for (i <- 0 until o.firrtlDepth) { unindent() } - s"skip" - } - firrtlLine + e.sourceInfo.makeMessage(" " + _) + def emit(circuit: Circuit): String = { + val fcircuit = Converter.convertLazily(circuit) + fir.Serializer.serialize(fcircuit) } - - private def emitParam(name: String, p: Param): String = { - val str = p match { - case IntParam(value) => value.toString - case DoubleParam(value) => value.toString - case StringParam(str) => "\"" + str + "\"" - case RawParam(str) => "'" + str + "'" - } - s"parameter $name = $str" - } - - /** Generates the FIRRTL module declaration. - */ - private def moduleDecl(m: Component): String = m.id match { - case _: BaseBlackBox => newline + s"extmodule ${m.name} : " - case _: RawModule => newline + s"module ${m.name} : " - } - - /** Generates the FIRRTL module definition. - */ - private def moduleDefn(m: Component): String = { - val body = new StringBuilder - withIndent { - for (p <- m.ports) { - val portDef = m match { - case bb: DefBlackBox => emitPort(p, bb.topDir) - case mod: DefModule => emitPort(p) - } - body ++= newline + portDef - } - body ++= newline - - m match { - case bb: DefBlackBox => - // Firrtl extmodule can overrule name - body ++= newline + s"defname = ${bb.id.desiredName}" - body ++= newline + (bb.params map { case (n, p) => emitParam(n, p) } mkString newline) - case mod: DefModule => { - // Preprocess whens & elsewhens, marking those that have no alternative - val procMod = mod.copy(commands = processWhens(mod.commands)) - for (cmd <- procMod.commands) { body ++= newline + emit(cmd, procMod)} - } - } - body ++= newline - } - body.toString() - } - - /** Returns the FIRRTL declaration and body of a module, or nothing if it's a - * duplicate of something already emitted (on the basis of simple string - * matching). - */ - private def emit(m: Component): String = { - // Generate the body. - val sb = new StringBuilder - sb.append(moduleDecl(m)) - sb.append(moduleDefn(m)) - sb.result - } - - /** Preprocess the command queue, marking when/elsewhen statements - * that have no alternatives (elsewhens or otherwise). These - * alternative-free statements reset the indent level to the - * enclosing block upon emission. - */ - private def processWhens(cmds: Seq[Command]): Seq[Command] = { - if (cmds.isEmpty) { - Seq.empty - } else { - cmds.zip(cmds.tail).map{ - case (a: WhenEnd, b: AltBegin) => a.copy(hasAlt = true) - case (a, b) => a - } ++ cmds.lastOption - } - } - - private var indentLevel = 0 - private def newline = "\n" + (" " * indentLevel) - private def indent(): Unit = indentLevel += 1 - private def unindent() { require(indentLevel > 0); indentLevel -= 1 } - private def withIndent(f: => Unit) { indent(); f; unindent() } - - private val res = new StringBuilder() - res ++= s";${BuildInfo.toString}\n" - res ++= s"circuit ${circuit.name} : " - withIndent { circuit.components.foreach(c => res ++= emit(c)) } - res ++= newline } + diff --git a/src/test/scala/chiselTests/PrintableSpec.scala b/src/test/scala/chiselTests/PrintableSpec.scala index 0325d3bc..25b54966 100644 --- a/src/test/scala/chiselTests/PrintableSpec.scala +++ b/src/test/scala/chiselTests/PrintableSpec.scala @@ -150,7 +150,6 @@ class PrintableSpec extends AnyFlatSpec with Matchers { printf(p"${FullName(myInst.io.fizz)}") } val firrtl = ChiselStage.emitChirrtl(new MyModule) - println(firrtl) getPrintfs(firrtl) match { case Seq(Printf("foo", Seq()), Printf("myWire.foo", Seq()), @@ -256,8 +255,8 @@ class PrintableSpec extends AnyFlatSpec with Matchers { val firLines = scala.io.Source.fromFile(firFile).getLines.toList // check that verification components have expected names - exactly(1, firLines) should include ("""printf(clock, UInt<1>(1), "hello AnonymousBundle(foo -> %d, bar -> %d)", myBun.foo, myBun.bar): howdy""") - exactly(1, firLines) should include ("""printf(clock, UInt<1>(1), "goodbye AnonymousBundle(foo -> %d, bar -> %d)", myBun.foo, myBun.bar): SIM""") - exactly(1, firLines) should include ("""printf(clock, UInt<1>(1), "adieu AnonymousBundle(foo -> %d, bar -> %d)", myBun.foo, myBun.bar): farewell""") + exactly(1, firLines) should include ("""printf(clock, UInt<1>("h1"), "hello AnonymousBundle(foo -> %d, bar -> %d)", myBun.foo, myBun.bar) : howdy""") + exactly(1, firLines) should include ("""printf(clock, UInt<1>("h1"), "goodbye AnonymousBundle(foo -> %d, bar -> %d)", myBun.foo, myBun.bar) : SIM""") + exactly(1, firLines) should include ("""printf(clock, UInt<1>("h1"), "adieu AnonymousBundle(foo -> %d, bar -> %d)", myBun.foo, myBun.bar) : farewell""") } } diff --git a/src/test/scala/chiselTests/VecLiteralSpec.scala b/src/test/scala/chiselTests/VecLiteralSpec.scala index d11289e1..d91cd2f4 100644 --- a/src/test/scala/chiselTests/VecLiteralSpec.scala +++ b/src/test/scala/chiselTests/VecLiteralSpec.scala @@ -461,10 +461,10 @@ class VecLiteralSpec extends ChiselFreeSpec with Utils { "vec literals can contain bundles" in { val chirrtl = (new chisel3.stage.ChiselStage).emitChirrtl(new VecExample, args = Array("--full-stacktrace")) - chirrtl should include("""out[0].bar <= UInt<5>("h016")""") - chirrtl should include("""out[0].foo <= UInt<6>("h02a")""") - chirrtl should include("""out[1].bar <= UInt<2>("h03")""") - chirrtl should include("""out[1].foo <= UInt<3>("h07")""") + chirrtl should include("""out[0].bar <= UInt<5>("h16")""") + chirrtl should include("""out[0].foo <= UInt<6>("h2a")""") + chirrtl should include("""out[1].bar <= UInt<2>("h3")""") + chirrtl should include("""out[1].foo <= UInt<3>("h7")""") } diff --git a/src/test/scala/chiselTests/experimental/verification/VerificationSpec.scala b/src/test/scala/chiselTests/experimental/verification/VerificationSpec.scala index a1fc2a1d..1e080739 100644 --- a/src/test/scala/chiselTests/experimental/verification/VerificationSpec.scala +++ b/src/test/scala/chiselTests/experimental/verification/VerificationSpec.scala @@ -104,9 +104,9 @@ class VerificationSpec extends ChiselPropSpec with Matchers { val firLines = scala.io.Source.fromFile(firFile).getLines.toList // check that verification components have expected names - exactly(1, firLines) should include ("cover(clock, _T, UInt<1>(1), \"\") : cov") - exactly(1, firLines) should include ("assume(clock, _T_3, UInt<1>(1), \"\") : assm") - exactly(1, firLines) should include ("assert(clock, _T_6, UInt<1>(1), \"\") : asst") + exactly(1, firLines) should include ("cover(clock, _T, UInt<1>(\"h1\"), \"\") : cov") + exactly(1, firLines) should include ("assume(clock, _T_3, UInt<1>(\"h1\"), \"\") : assm") + exactly(1, firLines) should include ("assert(clock, _T_6, UInt<1>(\"h1\"), \"\") : asst") } property("annotation of verification constructs with suggested name should work") { @@ -148,7 +148,7 @@ class VerificationSpec extends ChiselPropSpec with Matchers { val firLines = scala.io.Source.fromFile(firFile).getLines.toList // check that verification components have expected names - exactly(1, firLines) should include ("assert(clock, _T, UInt<1>(1), \"\") : hello") - exactly(1, firLines) should include ("assume(clock, _T_3, UInt<1>(1), \"\") : howdy") + exactly(1, firLines) should include ("assert(clock, _T, UInt<1>(\"h1\"), \"\") : hello") + exactly(1, firLines) should include ("assume(clock, _T_3, UInt<1>(\"h1\"), \"\") : howdy") } } -- cgit v1.2.3 From d9c30ea0fdd90f3e47cae161f4a7224a861a6bf0 Mon Sep 17 00:00:00 2001 From: Jack Koenig Date: Fri, 20 Aug 2021 12:00:08 -0700 Subject: Emit .fir lazily, overcomes JVM 2 GiB String limit --- src/main/scala/chisel3/internal/firrtl/Emitter.scala | 15 +++++++++++++-- src/main/scala/chisel3/stage/ChiselAnnotations.scala | 7 +++++-- 2 files changed, 18 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/main/scala/chisel3/internal/firrtl/Emitter.scala b/src/main/scala/chisel3/internal/firrtl/Emitter.scala index 53329908..a94558ce 100644 --- a/src/main/scala/chisel3/internal/firrtl/Emitter.scala +++ b/src/main/scala/chisel3/internal/firrtl/Emitter.scala @@ -1,12 +1,23 @@ // SPDX-License-Identifier: Apache-2.0 package chisel3.internal.firrtl -import firrtl.{ir => fir} + +import scala.collection.immutable.LazyList // Needed for 2.12 alias +import firrtl.ir.Serializer private[chisel3] object Emitter { def emit(circuit: Circuit): String = { val fcircuit = Converter.convertLazily(circuit) - fir.Serializer.serialize(fcircuit) + Serializer.serialize(fcircuit) + } + + def emitLazily(circuit: Circuit): Iterable[String] = { + val result = LazyList(s"circuit ${circuit.name} :\n") + val modules = circuit.components.view.map(Converter.convert) + val moduleStrings = modules.flatMap { m => + Array(Serializer.serialize(m, 1), "\n\n") + } + result ++ moduleStrings } } diff --git a/src/main/scala/chisel3/stage/ChiselAnnotations.scala b/src/main/scala/chisel3/stage/ChiselAnnotations.scala index b071c60f..6cace05f 100644 --- a/src/main/scala/chisel3/stage/ChiselAnnotations.scala +++ b/src/main/scala/chisel3/stage/ChiselAnnotations.scala @@ -133,9 +133,12 @@ case class CircuitSerializationAnnotation(circuit: Circuit, filename: String, fo protected def suffix: Option[String] = Some(format.extension) - // TODO Use lazy Iterables so that we don't have to materialize full intermediate data structures override def getBytes: Iterable[Byte] = format match { - case FirrtlFileFormat => OldEmitter.emit(circuit).getBytes + case FirrtlFileFormat => + OldEmitter.emitLazily(circuit) + .map(_.getBytes) + .flatten + // TODO Use lazy Iterables so that we don't have to materialize full intermediate data structures case ProtoBufFileFormat => val ostream = new java.io.ByteArrayOutputStream val modules = circuit.components.map(m => () => chisel3.internal.firrtl.Converter.convert(m)) -- cgit v1.2.3 From a6eb2ad8b6ff50bf245d610891808e436b19ed01 Mon Sep 17 00:00:00 2001 From: Jack Koenig Date: Fri, 20 Aug 2021 17:34:17 -0700 Subject: Use BufferedCustomFileEmission in CircuitSerializationAnnotation --- src/main/scala/chisel3/stage/ChiselAnnotations.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/main/scala/chisel3/stage/ChiselAnnotations.scala b/src/main/scala/chisel3/stage/ChiselAnnotations.scala index 6cace05f..de47ef36 100644 --- a/src/main/scala/chisel3/stage/ChiselAnnotations.scala +++ b/src/main/scala/chisel3/stage/ChiselAnnotations.scala @@ -3,7 +3,7 @@ package chisel3.stage import firrtl.annotations.{Annotation, NoTargetAnnotation} -import firrtl.options.{CustomFileEmission, HasShellOptions, OptionsException, ShellOption, StageOptions, Unserializable} +import firrtl.options.{BufferedCustomFileEmission, CustomFileEmission, HasShellOptions, OptionsException, ShellOption, StageOptions, Unserializable} import firrtl.options.Viewer.view import chisel3.{ChiselException, Module} import chisel3.RawModule @@ -123,7 +123,7 @@ import CircuitSerializationAnnotation._ */ case class CircuitSerializationAnnotation(circuit: Circuit, filename: String, format: Format) extends NoTargetAnnotation - with CustomFileEmission { + with BufferedCustomFileEmission { /* Caching the hashCode for a large circuit is necessary due to repeated queries. * Not caching the hashCode will cause severe performance degredations for large [[Circuit]]s. */ @@ -133,17 +133,16 @@ case class CircuitSerializationAnnotation(circuit: Circuit, filename: String, fo protected def suffix: Option[String] = Some(format.extension) - override def getBytes: Iterable[Byte] = format match { + override def getBytesBuffered: Iterable[Array[Byte]] = format match { case FirrtlFileFormat => OldEmitter.emitLazily(circuit) .map(_.getBytes) - .flatten // TODO Use lazy Iterables so that we don't have to materialize full intermediate data structures case ProtoBufFileFormat => val ostream = new java.io.ByteArrayOutputStream val modules = circuit.components.map(m => () => chisel3.internal.firrtl.Converter.convert(m)) firrtl.proto.ToProto.writeToStreamFast(ostream, firrtl.ir.NoInfo, modules, circuit.name) - ostream.toByteArray + List(ostream.toByteArray) } } -- cgit v1.2.3