building scaladoc or javadoc and pushing to ghpages

[erikerlandson]

Looking for mill module (or code snippets) that are equivalent to sbt clean unidoc ghpagesPushSite

[jodersky]

The most customizable option is to call the zinc worker directly to build the scaldoc, as is done by the docJar task https://github.com/lihaoyi/mill/blob/4ad87fab1e3c477d527016da3ef7795a38f1928c/scalalib/src/ScalaModule.scala#L177. This allows you to customize which sources should be included in the documentation.

I use something similar to the following:

// This module isn't really a ScalaModule, but we use it to generate
// consolidated documentation using the Scaladoc tool.
object docs extends ScalaModule {
  def scalaVersion = "3.0.0-M3"

  def docSource = T.source(millSourcePath)

  def moduleDeps = Seq(
    ...
  )

  // generate the static website
  def site = T {
    import mill.eval.Result

    for {
      child <- os.walk(docSource().path)
      if os.isFile(child)
    } {
      os.copy.over(child, T.dest / child.subRelativeTo(docSource().path), createFolders = true)
    }
    val files: Seq[os.Path] = T.traverse(moduleDeps)(_.allSourceFiles)().flatten.map(_.path)

    // format: off
    val options = Seq(
      "-classpath", compileClasspath().map(_.path).mkString(":"),
      "-siteroot", T.dest.toString,
      "-project-url", "<url>",
      "-project-logo", "logo.svg",
      "-project-version", "<version>",
      "-project", "<name>"
    ) ++ scalaDocPluginClasspath().map(pluginPathRef => s"-Xplugin:${pluginPathRef.path}")
    // format: on

    zincWorker.worker().docJar(
      scalaVersion(),
      scalaOrganization(),
      scalaDocClasspath().map(_.path),
      scalacPluginClasspath().map(_.path),
      files.map(_.toString) ++ options
    ) match{
      case true =>
        Result.Success(PathRef(T.dest / "_site"))
      case false =>
        Result.Failure("doc generation failed")
    }
  }

  // preview the site locally
  def serve() = T.command{
    os.proc("python3", "-m", "http.server", "--directory", site().path).call()
  }
}

It's for Scala 3, but it should be very easy to adapt. The key parts really are:

• zincWorker.worker().docJar(...)
• and the way to gather source files from multiple projects: val files: Seq[os.Path] = T.traverse(moduleDeps)(_.allSourceFiles)().flatten.map(_.path)

You can generate the site with mill docs.site, or even directly preview it with mill docs.serve

[erikerlandson]

Thanks, this will be helpful!

[jodersky]

The dest directory of docs.site can then be published on github by pushing it to the gh-pages branch. Here's a script do to this:

#!/bin/bash
set -o errexit

./mill -i docs.site

docdir=out/docs/site/dest/_site
git -C "$docdir" init
git -C "$docdir" config user.name "Joe Doc Champion"
git -C "$docdir" config user.email "[email protected]"
git -C "$docdir" add .
git -C "$docdir" commit -m "A new version is out!"
git -C "$docdir" push -f [email protected]:<username>/<project> master:gh-pages

[erikerlandson]

I adapted @jodersky 's answer above to create a new module trait ScaladocSiteModule, with targets:

• stage: generate and stage doc into out.<yourmodule>.stage.dest
• serve: stage and then serve the doc locally on http://localhost:8000
• push: stage and then push to a repo (use branch gh-pages by default)

(I also tweaked the scaladoc logic to work for scala-2)

The example below is an excerpt from here: https://github.com/erikerlandson/mill-testbed

object site extends ScaladocSiteModule {
    // standard scalaVersion method is a task, which only works inside other tasks
    def scaladocScalaVersion = "2.13.6"

    // specify subdirectory for scaladoc
    override def scaladocSubPath: os.SubPath = os.sub / "api" / "latest"

    // example of bridging non-cross ScalaSiteModule to cross-compiled modules
    override def scaladocModules = Seq(lib1(scaladocScalaVersion), lib2(scaladocScalaVersion))

    def scaladocPushGitURI = "[email protected]:erikerlandson/mill-testbed.git"
}

// This module isn't really a ScalaModule, but we use it to generate
// consolidated documentation using the Scaladoc tool.
trait ScaladocSiteModule extends ScalaModule {
    // would be preferable to just use standard scalaVersion here to be more DRY, but
    // scalaVersion is a Task and I haven't figured out how to invoke a Task in a non-Task method
    def scaladocScalaVersion: String

    def scaladocModules: Seq[JavaModule] = List.empty[JavaModule]

    // stage the static website and/or doc into the 'stage' task destination directory
    // adapted from:
    // https://github.com/com-lihaoyi/mill/discussions/1194
    def stage = T {
        import mill.eval.Result

        if (!os.isDir(millSourcePath)) {
            T.log.error(s"""Source path "${millSourcePath}" not found, ignoring""")
            T.log.error(s"Staging index.html from method defaultSiteIndex")
            os.write.over(T.dest / "index.html", defaultSiteIndex)
        } else {
            val sitefiles = os.walk(millSourcePath, skip = (p: os.Path) => !os.isFile(p))
            T.log.info(s"Staging ${sitefiles.length} site files from site source path ${millSourcePath}") 
            for {
                f <- sitefiles
            } {
                os.copy.over(f, T.dest / f.subRelativeTo(millSourcePath), createFolders = true)
            }
        }

        val scaladocFiles: Seq[os.Path] =
            T.traverse(scaladocModules)(_.allSourceFiles)().flatten.map(_.path)

        T.log.info(s"Staging scaladoc for ${scaladocFiles.length} files")

        val scaladocPath: os.Path = T.dest / scaladocSubPath
        os.makeDir.all(scaladocPath)

        // the details of the options and zincWorker call are significantly
        // different between scala-2 scaladoc and scala-3 scaladoc
        // below is for scala-2 variant
        val options: Seq[String] = Seq(
            "-doc-title", projectContext.projectName,
            "-doc-version", projectContext.projectVersion,
            "-d", scaladocPath.toString,
            "-classpath", compileClasspath().map(_.path).mkString(":"),
        )

        val docReturn = zincWorker.worker().docJar(
            scalaVersion(),
            scalaOrganization(),
            scalaDocClasspath().map(_.path),
            scalacPluginClasspath().map(_.path),
            options ++ scaladocFiles.map(_.toString)
        ) match{
            case true =>  Result.Success(PathRef(T.dest))
            case false => Result.Failure("doc generation failed")
        }

        docReturn
    }

    // preview the site locally
    // use './mill -i ...', or python http server may remain as zombie
    def serve() = T.command {
        val port = scaladocServePort
        require((port > 0) && (port < 65536))

        // this also runs 'stage' target as a dependency
        val stageDir = (stage().path).toString

        try {
            T.log.info(s"serving on http://localhost:${port}")
            os.proc("python3", "-m", "http.server", s"${port}", "--directory", stageDir).call()
            ()
        } catch {
            case _: Throwable =>
                T.log.error("Server startup failed - is python3 installed?")
                ()
        }
    }

    def push() = T.command {
        // this also runs 'stage' target as a dependency
        val stageDir = (stage().path).toString

        val workBranch = scaladocPushWorkingBranch
        val remoteBranch = scaladocPushRemoteBranch
        val gitURI = scaladocPushGitURI
        val username = scaladocPushUserName
        val useremail = scaladocPushUserEmail

        T.log.info(s"pushing site to branch $remoteBranch of $gitURI")

        os.proc("git", "-C", stageDir, "init", "--quiet", s"--initial-branch=${workBranch}").call()
        os.proc("git", "-C", stageDir, "config", "user.name", username).call()
        os.proc("git", "-C", stageDir, "config", "user.email", useremail).call()
        os.proc("git", "-C", stageDir, "add", ".").call()
        os.proc("git", "-C", stageDir, "commit", "-m", "push from mill").call()
        os.proc("git", "-C", stageDir, "push", "-f", gitURI, s"${workBranch}:${remoteBranch}").call()

        T.log.info("cleaning up git working directory")
        os.proc("rm", "-rf", s"${stageDir}/.git").call()
    }

    // currently no default for this
    def scaladocPushGitURI: String

    def scaladocPushWorkingBranch: String = "main"

    def scaladocPushRemoteBranch: String = "gh-pages"

    def scaladocPushUserName: String = os.proc("git", "config", "user.name").call().out.string

    def scaladocPushUserEmail: String = os.proc("git", "config", "user.email").call().out.string

    def scalaVersion = scaladocScalaVersion

    // scaladoc goes in this subdirectory
    // TODO: add javadoc support, similar to sbt unidoc?
    def scaladocSubPath: os.SubPath = os.sub / "api" / "latest"

    def scaladocServePort: Int = 8000

    def defaultSiteIndex: String =
        s"""|<!DOCTYPE html>
            |<html lang="en">
            |<head>
            |    <meta charset="UTF-8">
            |    <title>Project Documentation</title>
            |    <script language="JavaScript">
            |    <!--
            |    function doRedirect()
            |    {
            |        window.location.replace("${scaladocSubPath}");
            |    }
            |    doRedirect();
            |    //-->
            |    </script>
            |</head>
            |<body>
            |<a href="${scaladocSubPath}">Go to the project documentation
            |</a>
            |</body>
            |</html>        
            |""".stripMargin
}

[lefou]

Very Cool. Thanks for posting your solution.

Just a small remark about the stale/zombie process after exiting mill. If you use mill.modules.Jvm.runSubprocess, it should handle shutdown of spawned processes better.

[erikerlandson]

@lefou I am trying to decide whether to submit this as a contrib module or just publish it myself as a 3rd-party module. If you are interested in a contrib module PR, let me know.

[erikerlandson]

PS also using runSubprocess is working, thanks!

mill.modules.Jvm.runSubprocess(List("python3", "-m", "http.server", s"${port}", "--directory", stageDir), T.env(T.ctx()), os.pwd)