From 30bff5f8b627001f5ad220404e33cd087f463ed8 Mon Sep 17 00:00:00 2001 From: Albert Magyar Date: Thu, 25 Jul 2019 13:49:13 -0700 Subject: Allow name of blackbox resource .f file to change from static value (#1129) * Allow name of blackbox resource .f file to change from static value * Restore fileListName as a deprecated def per Jack's feedback * Support both local and absolute paths for .f resource files --- .../firrtl/transforms/BlackBoxSourceHelper.scala | 57 ++++---- .../firrtl/util/BackendCompilationUtilities.scala | 12 +- .../transforms/BlackBoxSourceHelperSpec.scala | 161 +++++++++++++++++++++ .../transforms/BlacklBoxSourceHelperSpec.scala | 161 --------------------- 4 files changed, 201 insertions(+), 190 deletions(-) create mode 100644 src/test/scala/firrtlTests/transforms/BlackBoxSourceHelperSpec.scala delete mode 100644 src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala (limited to 'src') diff --git a/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala b/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala index 4a253740..4fe11d12 100644 --- a/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala +++ b/src/main/scala/firrtl/transforms/BlackBoxSourceHelper.scala @@ -34,6 +34,11 @@ case class BlackBoxPathAnno(target: ModuleName, path: String) extends BlackBoxHe override def serialize: String = s"path\n$path" } +case class BlackBoxResourceFileNameAnno(resourceFileName: String) extends BlackBoxHelperAnno + with NoTargetAnnotation { + override def serialize: String = s"resourceFileName\n$resourceFileName" +} + /** Exception indicating that a blackbox wasn't found * @param fileName the name of the BlackBox file (only used for error message generation) * @param e an underlying exception that generated this @@ -50,8 +55,8 @@ class BlackBoxNotFoundException(fileName: String, e: Throwable = null) extends F * set by the execution harness, or directly in the tests */ class BlackBoxSourceHelper extends firrtl.Transform { + import BlackBoxSourceHelper._ private val DefaultTargetDir = new File(".") - override def inputForm: CircuitForm = LowForm override def outputForm: CircuitForm = LowForm @@ -59,15 +64,16 @@ class BlackBoxSourceHelper extends firrtl.Transform { * @param annos a list of generic annotations for this transform * @return BlackBoxHelperAnnos and target directory */ - def collectAnnos(annos: Seq[Annotation]): (ListSet[BlackBoxHelperAnno], File) = - annos.foldLeft((ListSet.empty[BlackBoxHelperAnno], DefaultTargetDir)) { - case ((acc, tdir), anno) => anno match { + def collectAnnos(annos: Seq[Annotation]): (ListSet[BlackBoxHelperAnno], File, File) = + annos.foldLeft((ListSet.empty[BlackBoxHelperAnno], DefaultTargetDir, new File(defaultFileListName))) { + case ((acc, tdir, flistName), anno) => anno match { case BlackBoxTargetDirAnno(dir) => val targetDir = new File(dir) if (!targetDir.exists()) { FileUtils.makeDirectory(targetDir.getAbsolutePath) } - (acc, targetDir) - case a: BlackBoxHelperAnno => (acc + a, tdir) - case _ => (acc, tdir) + (acc, targetDir, flistName) + case BlackBoxResourceFileNameAnno(fileName) => (acc, tdir, new File(fileName)) + case a: BlackBoxHelperAnno => (acc + a, tdir, flistName) + case _ => (acc, tdir, flistName) } } @@ -79,17 +85,17 @@ class BlackBoxSourceHelper extends firrtl.Transform { * @throws BlackBoxNotFoundException if a Verilog source cannot be found */ override def execute(state: CircuitState): CircuitState = { - val (annos, targetDir) = collectAnnos(state.annotations) + val (annos, targetDir, flistName) = collectAnnos(state.annotations) val resourceFiles: ListSet[File] = annos.collect { case BlackBoxResourceAnno(_, resourceId) => - BlackBoxSourceHelper.writeResourceToDirectory(resourceId, targetDir) + writeResourceToDirectory(resourceId, targetDir) case BlackBoxPathAnno(_, path) => val fileName = path.split("/").last val fromFile = new File(path) val toFile = new File(targetDir, fileName) - val inputStream = BlackBoxSourceHelper.safeFile(fromFile.toString)(new FileInputStream(fromFile).getChannel) + val inputStream = safeFile(fromFile.toString)(new FileInputStream(fromFile).getChannel) val outputStream = new FileOutputStream(toFile).getChannel outputStream.transferFrom(inputStream, 0, Long.MaxValue) @@ -101,13 +107,23 @@ class BlackBoxSourceHelper extends firrtl.Transform { val outFile = new File(targetDir, name) (text, outFile) }.map { case (text, file) => - BlackBoxSourceHelper.writeTextToFile(text, file) + writeTextToFile(text, file) file } // Issue #917 - We don't want to list Verilog header files ("*.vh") in our file list - they will automatically be included by reference. val verilogSourcesOnly = (resourceFiles ++ inlineFiles).filterNot( _.getName().endsWith(".vh")) - BlackBoxSourceHelper.writeFileList(verilogSourcesOnly, targetDir) + val filelistFile = if (flistName.isAbsolute()) flistName else new File(targetDir, flistName.getName()) + + // We need the canonical path here, so verilator will create a path to the file that works from the targetDir, + // and, so we can compare the list of files automatically included, with an explicit list provided by the client + // and reject duplicates. + // If the path isn't canonical, when make tries to determine dependencies based on the *__ver.d file, we end up with errors like: + // make[1]: *** No rule to make target `test_run_dir/examples.AccumBlackBox_PeekPokeTest_Verilator345491158/AccumBlackBox.v', needed by `.../chisel-testers/test_run_dir/examples.AccumBlackBox_PeekPokeTest_Verilator345491158/VAccumBlackBoxWrapper.h'. Stop. + // or we end up including the same file multiple times. + if (verilogSourcesOnly.nonEmpty) { + writeTextToFile(verilogSourcesOnly.map(_.getCanonicalPath).mkString("\n"), filelistFile) + } state } @@ -150,19 +166,10 @@ object BlackBoxSourceHelper { out.close() } - val fileListName = "firrtl_black_box_resource_files.f" - - def writeFileList(files: ListSet[File], targetDir: File): Unit = { - if (files.nonEmpty) { - // We need the canonical path here, so verilator will create a path to the file that works from the targetDir, - // and, so we can compare the list of files automatically included, with an explicit list provided by the client - // and reject duplicates. - // If the path isn't canonical, when make tries to determine dependencies based on the *__ver.d file, we end up with errors like: - // make[1]: *** No rule to make target `test_run_dir/examples.AccumBlackBox_PeekPokeTest_Verilator345491158/AccumBlackBox.v', needed by `.../chisel-testers/test_run_dir/examples.AccumBlackBox_PeekPokeTest_Verilator345491158/VAccumBlackBoxWrapper.h'. Stop. - // or we end up including the same file multiple times. - writeTextToFile(files.map(_.getCanonicalPath).mkString("\n"), new File(targetDir, fileListName)) - } - } + val defaultFileListName = "firrtl_black_box_resource_files.f" + + @deprecated("Renamed to defaultFileListName, as the file list name may now be changed with an annotation", "1.3") + def fileListName = defaultFileListName def writeTextToFile(text: String, file: File): Unit = { val out = new PrintWriter(file) diff --git a/src/main/scala/firrtl/util/BackendCompilationUtilities.scala b/src/main/scala/firrtl/util/BackendCompilationUtilities.scala index dfa00eba..e0341bf1 100644 --- a/src/main/scala/firrtl/util/BackendCompilationUtilities.scala +++ b/src/main/scala/firrtl/util/BackendCompilationUtilities.scala @@ -81,7 +81,8 @@ trait BackendCompilationUtilities { * all the files which are not included elsewhere. If multiple ones exist, * the compilation will fail. * - * If the file BlackBoxSourceHelper.fileListName exists in the output directory, + * If the file BlackBoxSourceHelper.fileListName (or an overridden .f resource filename that is + * specified with the optional resourceFileName parameter) exists in the output directory, * it contains a list of source files to be included. Filter out any files in the vSources * sequence that are in this file so we don't include the same file multiple times. * This complication is an attempt to work-around the fact that clients used to have to @@ -91,18 +92,21 @@ trait BackendCompilationUtilities { * @param dir output directory * @param vSources list of additional Verilog sources to compile * @param cppHarness C++ testharness to compile/link against + * @param suppressVcd specifies if VCD tracing should be suppressed + * @param resourceFileName specifies what filename to look for to find a .f file */ def verilogToCpp( dutFile: String, dir: File, vSources: Seq[File], cppHarness: File, - suppressVcd: Boolean = false + suppressVcd: Boolean = false, + resourceFileName: String = firrtl.transforms.BlackBoxSourceHelper.defaultFileListName ): ProcessBuilder = { val topModule = dutFile - val list_file = new File(dir, firrtl.transforms.BlackBoxSourceHelper.fileListName) + val list_file = new File(dir, resourceFileName) val blackBoxVerilogList = { if(list_file.exists()) { Seq("-f", list_file.getAbsolutePath) @@ -113,7 +117,7 @@ trait BackendCompilationUtilities { } // Don't include the same file multiple times. - // If it's in BlackBoxSourceHelper.fileListName, don't explicitly include it on the command line. + // If it's in the main .f resource file, don't explicitly include it on the command line. // Build a set of canonical file paths to use as a filter to exclude already included additional Verilog sources. val blackBoxHelperFiles: Set[String] = { if(list_file.exists()) { diff --git a/src/test/scala/firrtlTests/transforms/BlackBoxSourceHelperSpec.scala b/src/test/scala/firrtlTests/transforms/BlackBoxSourceHelperSpec.scala new file mode 100644 index 00000000..089e837a --- /dev/null +++ b/src/test/scala/firrtlTests/transforms/BlackBoxSourceHelperSpec.scala @@ -0,0 +1,161 @@ +// See LICENSE for license details. + +package firrtlTests.transforms + +import firrtl.annotations.{Annotation, CircuitName, ModuleName} +import firrtl.transforms._ +import firrtl.{FIRRTLException, Transform, VerilogCompiler, VerilogEmitter} +import firrtlTests.{HighTransformSpec, LowTransformSpec} +import org.scalacheck.Test.Failed +import org.scalatest.{FreeSpec, Matchers, Succeeded} + + +class BlacklBoxSourceHelperTransformSpec extends LowTransformSpec { + def transform: Transform = new BlackBoxSourceHelper + + private val moduleName = ModuleName("Top", CircuitName("Top")) + private val input = """ + |circuit Top : + | + | extmodule AdderExtModule : + | input foo : UInt<16> + | output bar : UInt<16> + | + | defname = BBFAdd + | + | module Top : + | input x : UInt<16> + | output y : UInt<16> + | + | inst a1 of AdderExtModule + | a1.foo <= x + | y <= a1.bar + """.stripMargin + private val output = """ + |circuit Top : + | + | extmodule AdderExtModule : + | input foo : UInt<16> + | output bar : UInt<16> + | + | defname = BBFAdd + | + | module Top : + | input x : UInt<16> + | output y : UInt<16> + | + | inst a1 of AdderExtModule + | y <= a1.bar + | a1.foo <= x + """.stripMargin + + "annotated external modules with absolute path" should "appear in output directory" in { + + val absPath = new java.io.File("src/test/resources/blackboxes/AdderExtModule.v").getCanonicalPath + val annos = Seq( + BlackBoxTargetDirAnno("test_run_dir"), + BlackBoxPathAnno(moduleName, absPath) + ) + + execute(input, output, annos) + + val module = new java.io.File("test_run_dir/AdderExtModule.v") + val fileList = new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.defaultFileListName}") + + module.exists should be (true) + fileList.exists should be (true) + + module.delete() + fileList.delete() + } + + "annotated external modules with relative path" should "appear in output directory" in { + + val annos = Seq( + BlackBoxTargetDirAnno("test_run_dir"), + BlackBoxPathAnno(moduleName, "src/test/resources/blackboxes/AdderExtModule.v") + ) + + execute(input, output, annos) + + val module = new java.io.File("test_run_dir/AdderExtModule.v") + val fileList = new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.defaultFileListName}") + + module.exists should be (true) + fileList.exists should be (true) + + module.delete() + fileList.delete() + } + + "annotated external modules" should "appear in output directory" in { + + val annos = Seq( + BlackBoxTargetDirAnno("test_run_dir"), + BlackBoxResourceAnno(moduleName, "/blackboxes/AdderExtModule.v") + ) + + execute(input, output, annos) + + new java.io.File("test_run_dir/AdderExtModule.v").exists should be (true) + new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.defaultFileListName}").exists should be (true) + } + + "verilog compiler" should "have BlackBoxSourceHelper transform" in { + val verilogCompiler = new VerilogEmitter + verilogCompiler.transforms.map { x => x.getClass } should contain (classOf[BlackBoxSourceHelper]) + } + + "verilog header files" should "be available but not mentioned in the file list" in { + // Issue #917 - We don't want to list Verilog header files ("*.vh") in our file list. + // We don't actually verify that the generated verilog code works, + // we just ensure that the correct files end up where they're expected. + + // We're taking the liberty of recycling the above code with a few minor edits to change the external module name. + val sourceModuleName = "AdderExtModule" + val replacedModuleName = "ParameterizedViaHeaderAdderExtModule" + val pInput = input.replaceAll(sourceModuleName, replacedModuleName) + val pOutput = output.replaceAll(sourceModuleName, replacedModuleName) + + // We'll copy the following resources to the test_run_dir via BlackBoxResourceAnno's + val resourceNames = Seq("ParameterizedViaHeaderAdderExtModule.v", "VerilogHeaderFile.vh") + + val annos = Seq( + BlackBoxTargetDirAnno("test_run_dir")) ++ resourceNames.map{ n => BlackBoxResourceAnno(moduleName, "/blackboxes/" + n)} + + execute(pInput, pOutput, annos) + + // Our resource files should exist in the test_run_dir, + for (n <- resourceNames) + new java.io.File("test_run_dir/" + n).exists should be (true) + + // but our file list should not include the verilog header file. + val fileListFile = new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.defaultFileListName}") + fileListFile.exists should be (true) + val fileListFileSource = io.Source.fromFile(fileListFile) + val fileList = fileListFileSource.getLines.mkString + fileListFileSource.close() + fileList.contains("ParameterizedViaHeaderAdderExtModule.v") should be (true) + fileList.contains("VerilogHeaderFile.vh") should be (false) + } + + behavior of "BlackBox resources that do not exist" + + it should "provide a useful error message for BlackBoxResourceAnno" in { + val annos = Seq( BlackBoxTargetDirAnno("test_run_dir"), + BlackBoxResourceAnno(moduleName, "/blackboxes/IDontExist.v") ) + + (the [BlackBoxNotFoundException] thrownBy { execute(input, "", annos) }) + .getMessage should include ("Did you misspell it?") + } + + it should "provide a useful error message for BlackBoxPathAnno" in { + val absPath = new java.io.File("src/test/resources/blackboxes/IDontExist.v").getCanonicalPath + val annos = Seq( BlackBoxTargetDirAnno("test_run_dir"), + BlackBoxPathAnno(moduleName, absPath) ) + + (the [BlackBoxNotFoundException] thrownBy { execute(input, "", annos) }) + .getMessage should include ("Did you misspell it?") + } + +} diff --git a/src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala b/src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala deleted file mode 100644 index b7ecc37a..00000000 --- a/src/test/scala/firrtlTests/transforms/BlacklBoxSourceHelperSpec.scala +++ /dev/null @@ -1,161 +0,0 @@ -// See LICENSE for license details. - -package firrtlTests.transforms - -import firrtl.annotations.{Annotation, CircuitName, ModuleName} -import firrtl.transforms._ -import firrtl.{FIRRTLException, Transform, VerilogCompiler, VerilogEmitter} -import firrtlTests.{HighTransformSpec, LowTransformSpec} -import org.scalacheck.Test.Failed -import org.scalatest.{FreeSpec, Matchers, Succeeded} - - -class BlacklBoxSourceHelperTransformSpec extends LowTransformSpec { - def transform: Transform = new BlackBoxSourceHelper - - private val moduleName = ModuleName("Top", CircuitName("Top")) - private val input = """ - |circuit Top : - | - | extmodule AdderExtModule : - | input foo : UInt<16> - | output bar : UInt<16> - | - | defname = BBFAdd - | - | module Top : - | input x : UInt<16> - | output y : UInt<16> - | - | inst a1 of AdderExtModule - | a1.foo <= x - | y <= a1.bar - """.stripMargin - private val output = """ - |circuit Top : - | - | extmodule AdderExtModule : - | input foo : UInt<16> - | output bar : UInt<16> - | - | defname = BBFAdd - | - | module Top : - | input x : UInt<16> - | output y : UInt<16> - | - | inst a1 of AdderExtModule - | y <= a1.bar - | a1.foo <= x - """.stripMargin - - "annotated external modules with absolute path" should "appear in output directory" in { - - val absPath = new java.io.File("src/test/resources/blackboxes/AdderExtModule.v").getCanonicalPath - val annos = Seq( - BlackBoxTargetDirAnno("test_run_dir"), - BlackBoxPathAnno(moduleName, absPath) - ) - - execute(input, output, annos) - - val module = new java.io.File("test_run_dir/AdderExtModule.v") - val fileList = new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.fileListName}") - - module.exists should be (true) - fileList.exists should be (true) - - module.delete() - fileList.delete() - } - - "annotated external modules with relative path" should "appear in output directory" in { - - val annos = Seq( - BlackBoxTargetDirAnno("test_run_dir"), - BlackBoxPathAnno(moduleName, "src/test/resources/blackboxes/AdderExtModule.v") - ) - - execute(input, output, annos) - - val module = new java.io.File("test_run_dir/AdderExtModule.v") - val fileList = new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.fileListName}") - - module.exists should be (true) - fileList.exists should be (true) - - module.delete() - fileList.delete() - } - - "annotated external modules" should "appear in output directory" in { - - val annos = Seq( - BlackBoxTargetDirAnno("test_run_dir"), - BlackBoxResourceAnno(moduleName, "/blackboxes/AdderExtModule.v") - ) - - execute(input, output, annos) - - new java.io.File("test_run_dir/AdderExtModule.v").exists should be (true) - new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.fileListName}").exists should be (true) - } - - "verilog compiler" should "have BlackBoxSourceHelper transform" in { - val verilogCompiler = new VerilogEmitter - verilogCompiler.transforms.map { x => x.getClass } should contain (classOf[BlackBoxSourceHelper]) - } - - "verilog header files" should "be available but not mentioned in the file list" in { - // Issue #917 - We don't want to list Verilog header files ("*.vh") in our file list. - // We don't actually verify that the generated verilog code works, - // we just ensure that the correct files end up where they're expected. - - // We're taking the liberty of recycling the above code with a few minor edits to change the external module name. - val sourceModuleName = "AdderExtModule" - val replacedModuleName = "ParameterizedViaHeaderAdderExtModule" - val pInput = input.replaceAll(sourceModuleName, replacedModuleName) - val pOutput = output.replaceAll(sourceModuleName, replacedModuleName) - - // We'll copy the following resources to the test_run_dir via BlackBoxResourceAnno's - val resourceNames = Seq("ParameterizedViaHeaderAdderExtModule.v", "VerilogHeaderFile.vh") - - val annos = Seq( - BlackBoxTargetDirAnno("test_run_dir")) ++ resourceNames.map{ n => BlackBoxResourceAnno(moduleName, "/blackboxes/" + n)} - - execute(pInput, pOutput, annos) - - // Our resource files should exist in the test_run_dir, - for (n <- resourceNames) - new java.io.File("test_run_dir/" + n).exists should be (true) - - // but our file list should not include the verilog header file. - val fileListFile = new java.io.File(s"test_run_dir/${BlackBoxSourceHelper.fileListName}") - fileListFile.exists should be (true) - val fileListFileSource = io.Source.fromFile(fileListFile) - val fileList = fileListFileSource.getLines.mkString - fileListFileSource.close() - fileList.contains("ParameterizedViaHeaderAdderExtModule.v") should be (true) - fileList.contains("VerilogHeaderFile.vh") should be (false) - } - - behavior of "BlackBox resources that do not exist" - - it should "provide a useful error message for BlackBoxResourceAnno" in { - val annos = Seq( BlackBoxTargetDirAnno("test_run_dir"), - BlackBoxResourceAnno(moduleName, "/blackboxes/IDontExist.v") ) - - (the [BlackBoxNotFoundException] thrownBy { execute(input, "", annos) }) - .getMessage should include ("Did you misspell it?") - } - - it should "provide a useful error message for BlackBoxPathAnno" in { - val absPath = new java.io.File("src/test/resources/blackboxes/IDontExist.v").getCanonicalPath - val annos = Seq( BlackBoxTargetDirAnno("test_run_dir"), - BlackBoxPathAnno(moduleName, absPath) ) - - (the [BlackBoxNotFoundException] thrownBy { execute(input, "", annos) }) - .getMessage should include ("Did you misspell it?") - } - -} -- cgit v1.2.3