Mill support for Chisel3 (#1035)

Co-Authored-By: Jack Koenig <jack.koenig3@gmail.com>
Co-Authored-By: Jim Lawson <ucbjrl@berkeley.edu>

4 files changed, 388 insertions, 31 deletions

diff --git a/CommonBuild.sc b/CommonBuild.sc
new file mode 100644
index 00000000..a9a4e1a2
--- /dev/null
+++ b/CommonBuild.sc
@@ -0,0 +1,94 @@
+import ammonite.ops._
+import mill._
+import mill.scalalib._
+
+def scalacOptionsVersion(scalaVersion: String): Seq[String] = {
+  Seq() ++ {
+    // If we're building with Scala > 2.11, enable the compile option
+    //  switch to support our anonymous Bundle definitions:
+    //  https://github.com/scala/bug/issues/10047
+    if (scalaVersion.startsWith("2.11.")) {
+      Seq()
+    } else {
+      Seq(
+        "-Xsource:2.11",
+        "-Ywarn-unused:imports",
+        "-Ywarn-unused:locals"
+      )
+    }
+  }
+}
+
+def javacOptionsVersion(scalaVersion: String): Seq[String] = {
+  Seq() ++ {
+    // Scala 2.12 requires Java 8. We continue to generate
+    //  Java 7 compatible code for Scala 2.11
+    //  for compatibility with old clients.
+    if (scalaVersion.startsWith("2.11.")) {
+      Seq("-source", "1.7", "-target", "1.7")
+    } else {
+      Seq("-source", "1.8", "-target", "1.8")
+    }
+  }
+}
+
+// Define our own BuildInfo since mill doesn't currently have one.
+trait BuildInfo extends ScalaModule { outer =>
+
+  def buildInfoObjectName: String = "BuildInfo"
+
+  def buildInfoMembers: T[Map[String, String]] = T {
+    Map.empty[String, String]
+  }
+
+  private def generateBuildInfo(outputPath: Path, members: Map[String, String]) = {
+    val outputFile = outputPath / "BuildInfo.scala"
+    val packageName = members.getOrElse("buildInfoPackage", "")
+    val packageDef = if (packageName != "") {
+      s"package ${packageName}"
+    } else {
+      ""
+    }
+    val internalMembers =
+      members
+        .map {
+          case (name, value) => s"""  val ${name}: String = "${value}""""
+        }
+        .mkString("\n")
+    write(outputFile,
+      s"""
+         |${packageDef}
+         |case object ${buildInfoObjectName}{
+         |$internalMembers
+         |  override val toString: String = {
+         |    "buildInfoPackage: %s, version: %s, scalaVersion: %s" format (
+         |        buildInfoPackage, version, scalaVersion
+         |    )
+         |  }
+         |}
+       """.stripMargin)
+    outputPath
+  }
+
+  override def generatedSources = T {
+    super.generatedSources() :+ PathRef(generateBuildInfo(T.ctx().dest, buildInfoMembers()))
+  }
+}
+
+// Define some file filters to exclude unwanted files from created jars.
+type JarFileFilter = (Path, RelPath) => Boolean
+// Exclude any `.DS_Store` files
+val noDS_StoreFiles: JarFileFilter = (p: Path, relPath: RelPath) => {
+  relPath.last != ".DS_Store"
+}
+
+// Exclude non-source files - accept all resource files, but only *.{java,scala} from source paths
+val onlySourceFiles: JarFileFilter = (p: Path, relPath: RelPath) => {
+  p.last == "resources" || (relPath.ext == "scala" || relPath.ext == "java")
+}
+
+// Apply a sequence of file filters - only accept files which satisfy all filters.
+// We expect this to be curried, the resulting file filter passed to createJar()
+def forallFilters(fileFilters: Seq[JarFileFilter])(p: Path, relPath: RelPath): Boolean = {
+  fileFilters.forall(f => f(p, relPath))
+}
diff --git a/Makefile b/Makefile
index 53740363..9184c09c 100644
--- a/Makefile
+++ b/Makefile
@@ -1,31 +1,35 @@
-# Retain all intermediate files.
-.SECONDARY:
-
 SBT		?= sbt
 SBT_FLAGS	?= -Dsbt.log.noformat=true
+MKDIR ?= mkdir -p
+CURL ?= curl -L
+MILL_BIN ?= $(HOME)/bin/mill
+MILL ?= $(MILL_BIN) --color false
+MILL_REMOTE_RELEASE ?= https://github.com/lihaoyi/mill/releases/download/0.3.5/0.3.5
+
+# Fetch mill (if we don't have it).
+$(MILL_BIN):
+	$(MKDIR) $(dir $@)
+	@echo $(CURL) --silent --output $@.curl --write-out "%{http_code}" $(MILL_REMOTE_RELEASE)
+	STATUSCODE=$(shell $(CURL) --silent --output $@.curl --write-out "%{http_code}" $(MILL_REMOTE_RELEASE)) && \
+	if test $$STATUSCODE -eq 200; then \
+	  mv $@.curl $@ && chmod +x $@ ;\
+	else \
+	  echo "Can't fetch $(MILL_REMOTE_RELEASE)" && cat $@.curl && echo ;\
+	  false ;\
+	fi
+
+mill-tools:	$(MILL_BIN)
 
 CHISEL_VERSION = $(shell "$(SBT)" $(SBT_FLAGS) "show version" | tail -n 1 | cut -d ' ' -f 2)
 
-SRC_DIR	?= .
-CHISEL_BIN ?= $(abspath $(SRC_DIR)/bin)
-export CHISEL_BIN
-
 #$(info Build Chisel $(CHISEL_VERSION))
 
-# The targetDir will be rm -rf'ed when "make clean"
-targetDir ?= ./generated
 # The TEST_OUTPUT_DIR will be rm -rf'ed when "make clean"
-TEST_OUTPUT_DIR ?= ./test-outputs
-RM_DIRS 	:= $(TEST_OUTPUT_DIR) test-reports $(targetDir)
-#CLEAN_DIRS	:= doc
-
-test_src_dir := src/test/scala/ChiselTests
-test_results := $(filter-out main DirChange Pads SIntOps,$(notdir $(basename $(wildcard $(test_src_dir)/*.scala))))
-c_resources_dir := src/main/resources
-
-test_outs    := $(addprefix $(targetDir)/, $(addsuffix .out, $(test_results)))
+TEST_OUTPUT_DIR ?= ./test_run_dur
+RM_DIRS 	:= $(TEST_OUTPUT_DIR)
 
-.PHONY:	smoke publish-local pubishLocal check clean jenkins-build coverage scaladoc test checkstyle compile
+.PHONY:	smoke publish-local pubishLocal check clean jenkins-build coverage scaladoc test checkstyle compile \
+	mill.build mill.test mill.publishLocal mill.build.all mill.test.all mill.publishLocal.all mill-tools
 
 default:	publishLocal
 
@@ -38,8 +42,6 @@ publish-local publishLocal:
 test:
 	$(SBT) $(SBT_FLAGS) test
 
-check:	test $(test_outs)
-
 checkstyle:
 	$(SBT) $(SBT_FLAGS) scalastyle test:scalastyle
 
@@ -73,20 +75,34 @@ jenkins-build: clean
 	$(SBT) $(SBT_FLAGS) scalastyle coverage test
 	$(SBT) $(SBT_FLAGS) coverageReport
 
-$(targetDir)/%.fir: $(test_src_dir)/%.scala
-	$(SBT) $(SBT_FLAGS) "test:runMain ChiselTests.MiniChisel $(notdir $(basename $<)) $(CHISEL_FLAGS)"
+# Compile and package jar
+mill.build: mill-tools
+	$(MILL) chisel3.jar
+
+# Compile and test
+mill.test: mill-tools
+	$(MILL) chisel3.test
+
+# Build and publish jar
+mill.publishLocal: mill-tools
+	$(MILL) chisel3.publishLocal
 
-$(targetDir)/%.flo: $(targetDir)/%.fir
-	$(CHISEL_BIN)/fir2flo.sh $(targetDir)/$*
+# Compile and package all jar
+mill.build.all: mill-tools
+	$(MILL) chisel3[_].jar
 
-$(targetDir)/%: $(targetDir)/%.flo $(targetDir)/emulator.h $(targetDir)/emulator_mod.h $(targetDir)/emulator_api.h
-	(cd $(targetDir); $(CHISEL_BIN)/flo2app.sh $*)
+# Compile and test
+mill.test.all: mill-tools
+	$(MILL) chisel3[_].test
 
-$(targetDir)/%.h:	$(c_resources_dir)/%.h
-	cp $< $@
+# Build and publish jar
+mill.publishLocal.all: mill-tools
+	$(MILL) chisel3[_].publishLocal
 
-$(targetDir)/%.out:	$(targetDir)/%
-	$(SBT) $(SBT_FLAGS) "test:runMain ChiselTests.MiniChisel $(notdir $(basename $<)) $(CHISEL_FLAGS) --test --targetDir $(targetDir)"
+# Remove all generated code.
+# Until "mill clean" makes it into a release.
+mill.clean:
+	$(RM) -rf out
 
 # The "last-resort" rule.
 # We assume the target is something like "+clean".
diff --git a/build.sbt b/build.sbt
index 0b6c9bb4..f403697f 100644
--- a/build.sbt
+++ b/build.sbt
@@ -125,6 +125,18 @@ lazy val coreMacros = (project in file("coreMacros")).
 lazy val chiselFrontend = (project in file("chiselFrontend")).
   settings(commonSettings: _*).
   settings(publishArtifact := false).
+  settings(
+    scalacOptions := scalacOptions.value ++ Seq(
+      "-deprecation",
+      "-explaintypes",
+      "-feature",
+      "-language:reflectiveCalls",
+      "-unchecked",
+      "-Xcheckinit",
+      "-Xlint:infer-any"
+//      "-Xlint:missing-interpolator"
+    )
+  ).
   dependsOn(coreMacros)
 
 // This will always be the root project, even if we are a sub-project.
diff --git a/build.sc b/build.sc
new file mode 100644
index 00000000..f080c522
--- /dev/null
+++ b/build.sc
@@ -0,0 +1,235 @@
+import ammonite.ops._
+import ammonite.ops.ImplicitWd._
+import mill._
+import mill.scalalib._
+import mill.scalalib.publish._
+import mill.eval.{Evaluator, Result}
+import mill.define.Task
+import mill.modules.Jvm._
+
+import $file.CommonBuild
+
+/** Utility types for changing a dependency between Ivy and Module (source) */
+sealed trait IvyOrModuleDep
+case class IvyDep(dep: Dep) extends IvyOrModuleDep
+case class ModuleDep(dep: PublishModule) extends IvyOrModuleDep
+
+object chiselCompileOptions {
+  def scalacOptions = Seq(
+    "-deprecation",
+    "-explaintypes",
+    "-feature",
+    "-language:reflectiveCalls",
+    "-unchecked",
+    "-Xcheckinit",
+    "-Xlint:infer-any"
+/*    "-Xlint:missing-interpolator" // this causes a:
+//[error] .../chisel3/chiselFrontend/src/main/scala/chisel3/core/Aggregate.scala:605:48: recursive value outer needs type
+//[error]             val outer = clazz.getDeclaredField("$outer").get(this)
+//[error]                                                ^
+//[error] one error found
+ */
+  )
+}
+
+val crossVersions = Seq("2.12.6", "2.11.12")
+
+// Provide a managed dependency on X if -DXVersion="" is supplied on the command line.
+val defaultVersions = Map("firrtl" -> "1.2-SNAPSHOT")
+
+def getVersion(dep: String, org: String = "edu.berkeley.cs") = {
+  val version = sys.env.getOrElse(dep + "Version", defaultVersions(dep))
+  ivy"$org::$dep:$version"
+}
+
+// Define the common chisel module.
+trait CommonChiselModule extends SbtModule {
+  // Normally defined in CrossSbtModule, our submodules don't have it by default
+  def crossScalaVersion: String
+  // This build uses an ivy dependency but allows overriding with a module (source) dependency
+  def firrtlDep: IvyOrModuleDep
+  override def scalacOptions = chiselCompileOptions.scalacOptions ++ CommonBuild.scalacOptionsVersion(crossScalaVersion)
+  override def javacOptions = CommonBuild.javacOptionsVersion(crossScalaVersion)
+  val macroPlugins = Agg(ivy"org.scalamacros:::paradise:2.1.0")
+  def scalacPluginIvyDeps = macroPlugins
+  def compileIvyDeps = macroPlugins
+  def chiselDeps = firrtlDep match {
+    case IvyDep(dep) => Agg(dep)
+    case ModuleDep(_) => Agg()
+  }
+  override def ivyDeps = T { chiselDeps }
+}
+
+trait PublishChiselModule extends CommonChiselModule with PublishModule {
+  override def artifactName = "chisel3"
+  def publishVersion = "3.2-SNAPSHOT"
+
+  def pomSettings = PomSettings(
+    description = artifactName(),
+    organization = "edu.berkeley.cs",
+    url = "https://chisel.eecs.berkeley.edu",
+    licenses = Seq(License.`BSD-3-Clause`),
+    versionControl = VersionControl.github("freechipsproject", "chisel3"),
+    developers = Seq(
+      Developer("jackbackrack",    "Jonathan Bachrach",      "https://eecs.berkeley.edu/~jrb/")
+    )
+  )
+}
+
+// Make this available to external tools.
+object chisel3 extends Cross[ChiselTopModule](crossVersions: _*) {
+  def defaultVersion(ev: Evaluator) = T.command{
+    println(crossVersions.head)
+  }
+
+  def compile = T{
+    chisel3(crossVersions.head).compile()
+  }
+
+  def jar = T{
+    chisel3(crossVersions.head).jar()
+  }
+
+  def test = T{
+    chisel3(crossVersions.head).test.test()
+  }
+
+  def publishLocal = T{
+    chisel3(crossVersions.head).publishLocal()
+  }
+
+  def docJar = T{
+    chisel3(crossVersions.head).docJar()
+  }
+}
+
+class ChiselTopModule(val crossScalaVersion: String) extends AbstractChiselModule {
+  // This build uses an ivy dependency but allows overriding with a module (source) dependency
+  def firrtlDep: IvyOrModuleDep = IvyDep(getVersion("firrtl"))
+}
+
+trait AbstractChiselModule extends PublishChiselModule with CommonBuild.BuildInfo with CrossSbtModule { top =>
+
+  // If would be nice if we didn't need to do this, but PublishModule may only be dependent on
+  //  other PublishModules.
+  trait UnpublishedChiselModule extends PublishChiselModule
+
+
+  object coreMacros extends UnpublishedChiselModule {
+    def crossScalaVersion = top.crossScalaVersion
+    def scalaVersion = crossScalaVersion
+    def firrtlDep: IvyOrModuleDep = top.firrtlDep
+  }
+
+  object chiselFrontend extends UnpublishedChiselModule {
+    def crossScalaVersion = top.crossScalaVersion
+    def scalaVersion = crossScalaVersion
+    def moduleDeps = Seq(coreMacros) ++ (firrtlDep match {
+      case ModuleDep(dep) => Seq(dep)
+      case IvyDep(_) => Seq()
+    })
+    def firrtlDep: IvyOrModuleDep = top.firrtlDep
+  }
+
+  override def moduleDeps = Seq(coreMacros, chiselFrontend)
+
+  // This submodule is unrooted - its source directory is in the top level directory.
+  override def millSourcePath = super.millSourcePath / ammonite.ops.up
+
+  // In order to preserve our "all-in-one" policy for published jars,
+  //  we define allModuleSources() to include transitive sources, and define
+  //  allModuleClasspath() to include transitive classes.
+  def transitiveSources = T {
+    Task.traverse(moduleDeps)(m =>
+      T.task{m.allSources()}
+    )().flatten
+  }
+
+  def allModuleSources = T {
+    allSources() ++ transitiveSources()
+  }
+
+  def transitiveResources = T {
+    Task.traverse(moduleDeps)(m =>
+      T.task{m.resources()}
+    )().flatten
+  }
+
+  def allModuleResources = T {
+    resources() ++ transitiveResources()
+  }
+
+  // We package all classes in a singe jar.
+  def allModuleClasspath = T {
+    localClasspath() ++ transitiveLocalClasspath()
+  }
+
+  // Override publishXmlDeps so we don't include dependencies on our sub-modules.
+  override def publishXmlDeps = T.task {
+    val ivyPomDeps = ivyDeps().map(resolvePublishDependency().apply(_))
+    ivyPomDeps
+  }
+
+  // We need to copy (and override) the `jar` and `docJar` targets so we can build
+  //  single jars implementing our "all-in-one" policy.
+  override def jar = T {
+    createJar(
+      allModuleClasspath().map(_.path).filter(exists),
+      mainClass()
+    )
+  }
+
+
+  override def docJar = T {
+    val outDir = T.ctx().dest
+
+    val javadocDir = outDir / 'javadoc
+    mkdir(javadocDir)
+
+    val files = for{
+      ref <- allModuleSources()
+      if exists(ref.path)
+      p <- (if (ref.path.isDir) ls.rec(ref.path) else Seq(ref.path))
+      if (p.isFile && ((p.ext == "scala") || (p.ext == "java")))
+    } yield p.toNIO.toString
+
+    val pluginOptions = scalacPluginClasspath().map(pluginPathRef => s"-Xplugin:${pluginPathRef.path}")
+    val options = Seq("-d", javadocDir.toNIO.toString, "-usejavacp") ++ pluginOptions ++ scalacOptions()
+
+    if (files.nonEmpty) runSubprocess(
+      "scala.tools.nsc.ScalaDoc",
+      scalaCompilerClasspath().map(_.path) ++ compileClasspath().filter(_.path.ext != "pom").map(_.path),
+      mainArgs = (files ++ options).toSeq
+    )
+
+    createJar(Agg(javadocDir), None)(outDir)
+  }
+
+  def sourceJar = T {
+    createJar((allModuleSources() ++ allModuleResources()).map(_.path).filter(exists), None)
+  }
+
+  override def ivyDeps = Agg(
+    ivy"com.github.scopt::scopt:3.6.0"
+  ) ++ chiselDeps
+
+  object test extends Tests {
+    override def ivyDeps = Agg(
+      ivy"org.scalatest::scalatest:3.0.1",
+      ivy"org.scalacheck::scalacheck:1.13.4"
+    )
+    def testFrameworks = Seq("org.scalatest.tools.Framework")
+  }
+
+  // This is required for building a library, but not for a `run` target.
+  // In the latter case, mill will determine this on its own.
+  def mainClass = Some("chisel3.Driver")
+
+  override def buildInfoMembers = T {
+    Map[String, String](
+      "buildInfoPackage" -> artifactName(),
+      "version" -> publishVersion(),
+      "scalaVersion" -> scalaVersion()
+    )
+  }
+}