diff --git a/admin/jenkins/ci_build_all.scala b/admin/jenkins/ci_build_all.scala --- a/admin/jenkins/ci_build_all.scala +++ b/admin/jenkins/ci_build_all.scala @@ -1,257 +1,257 @@ object profile extends isabelle.CI_Profile { import isabelle._ import java.io.FileReader import scala.sys.process._ import org.apache.commons.configuration2._ override def clean = false val afp = Path.explode("$AFP_BASE") val afp_thys = afp + Path.explode("thys") val afp_id = hg_id(afp) sealed abstract class Status(val str: String) { def merge(that: Status): Status = (this, that) match { case (Ok, s) => s case (Failed, s) => Failed case (Skipped, Failed) => Failed case (Skipped, s) => Skipped } } object Status { def merge(statuses: List[Status]): Status = statuses.foldLeft(Ok: Status)(_ merge _) def from_results(results: Build.Results, session: String): Status = if (results.cancelled(session)) Skipped else if (results(session).ok) Ok else Failed } case object Ok extends Status("ok") case object Skipped extends Status("skipped") case object Failed extends Status("failed") case class Mail(subject: String, recipients: List[String], text: String) { import java.util._ import javax.mail._ import javax.mail.internet._ import javax.activation._ def send(): Unit = { val authenticator = new Authenticator() { override def getPasswordAuthentication() = new PasswordAuthentication(System.getProperty("mail.smtp.user"), System.getProperty("mail.smtp.password")) } val session = Session.getDefaultInstance(System.getProperties(), authenticator) val message = new MimeMessage(session) message.setFrom(new InternetAddress("ci@isabelle.systems", "Isabelle/Jenkins")) message.setSender(new InternetAddress("ge73ruk@mytum.de")) message.setSubject(subject) message.setText(text, "UTF-8") message.setSentDate(new java.util.Date()) recipients.foreach { recipient => message.addRecipient(Message.RecipientType.TO, new InternetAddress(recipient)) } try { Transport.send(message) } catch { case ex: MessagingException => println(s"Sending mail failed: ${ex.getMessage}") } } } class Metadata(ini: INIConfiguration) { def maintainers(entry: String): List[String] = { val config = ini.getSection(entry) val raw = if (config.containsKey("notify")) config.getString("notify") else "" List(raw.split(','): _*).map(_.trim).filterNot(_.isEmpty) } def entry_of_session(info: Sessions.Info): Option[String] = if (info.dir.dir.file == afp_thys.file) Some(info.dir.base.implode) else None def notify(name: String, result: Process_Result, info: Sessions.Info): Unit = entry_of_session(info).foreach { entry => val mails = maintainers(entry) val text = s"""|The build for the session | $name, |belonging to the AFP entry | $entry |failed. | |You are receiving this mail because you are the maintainer of that AFP |entry. | |The following information might help you with resolving the problem. | |Build log: ${Isabelle_System.getenv("BUILD_URL")} |Isabelle ID: $isabelle_id |AFP ID: $afp_id |Timeout? ${result.timeout} |Exit code: ${result.rc} | |Last 50 lines from stdout (if available): | |${result.out_lines.takeRight(50).mkString("\n")} | |Last 50 lines from stderr (if available): | |${result.err_lines.takeRight(50).mkString("\n")} |""".stripMargin val subject = s"Build of AFP entry $entry failed" if (mails.isEmpty) println(s"Entry $entry: WARNING no maintainers specified") else Mail(text = text, subject = subject, recipients = mails).send() } def group_by_entry(results: Build.Results): Map[Option[String], List[String]] = results.sessions.toList.map { name => entry_of_session(results.info(name)) -> name - }.groupBy(_._1).mapValues(_.map(_._2)) + }.groupBy(_._1).view.mapValues(_.map(_._2)).toMap def status_as_json(status: Map[Option[String], Status]): String = { val entries_strings = status.collect { case (Some(entry), status) => s"""{"entry": "$entry", "status": "${status.str}"}""" } val entries_string = entries_strings.mkString("[", ",\n", "]") s""" {"build_data": {"isabelle_id": "$isabelle_id", "afp_id": "$afp_id", "time": "$start_time", "url": "${Isabelle_System.getenv("BUILD_URL")}", "job": "${Isabelle_System.getenv("JOB_NAME")}" }, "entries": $entries_string } """ } def status_as_html(status: Map[Option[String], Status]): String = { val entries_strings = status.collect { case (None, Failed) => s"
  • Distribution
  • " case (Some(entry), Failed) => s"""
  • $entry
  • """ } if (entries_strings.isEmpty) "" else entries_strings.mkString("Failed entries: ") } } val status_file = Path.explode("$ISABELLE_HOME/status.json").file val report_file = Path.explode("$ISABELLE_HOME/report.html").file val deps_file = Path.explode("$ISABELLE_HOME/dependencies.json").file def can_send_mails = System.getProperties().containsKey("mail.smtp.host") def include = List(afp_thys) def select = Nil def pre_hook(args: List[String]) = { println(s"AFP id $afp_id") if (status_file.exists()) status_file.delete() if (report_file.exists()) report_file.delete() File.write(report_file, "") if (!can_send_mails) println(s"Mail configuration not found.") } def post_hook(results: Build.Results) = { print_section("DEPENDENCIES") println("Generating dependencies file ...") val result = Isabelle_System.bash("isabelle afp_dependencies") result.check println("Writing dependencies file ...") File.write(deps_file, result.out) val metadata = { val path = afp + Path.explode("metadata/metadata") val ini = new INIConfiguration() if (path.is_file) { val reader = new FileReader(path.file) ini.read(reader) reader.close() } new Metadata(ini) } val status = - metadata.group_by_entry(results).mapValues { sessions => + metadata.group_by_entry(results).view.mapValues { sessions => Status.merge(sessions.map(Status.from_results(results, _))) - } + }.toMap print_section("REPORT") println("Writing report file ...") File.write(report_file, metadata.status_as_html(status)) print_section("SITEGEN") println("Writing status file ...") File.write(status_file, metadata.status_as_json(status)) println("Running sitegen ...") val script = afp + Path.explode("admin/sitegen-devel") val sitegen_result = List(script.file.toString, status_file.toString, deps_file.toString).! if (sitegen_result > 0) println("sitegen failed") if (!results.ok) { print_section("NOTIFICATIONS") for (name <- results.sessions) { val result = results(name) if (!result.ok && !results.cancelled(name) && can_send_mails) metadata.notify(name, result, results.info(name)) } } print_section("COMPLETED") } def selection = Sessions.Selection( all_sessions = true, exclude_session_groups = List("slow")) } diff --git a/admin/jenkins/ci_build_testboard.scala b/admin/jenkins/ci_build_testboard.scala --- a/admin/jenkins/ci_build_testboard.scala +++ b/admin/jenkins/ci_build_testboard.scala @@ -1,114 +1,114 @@ object profile extends isabelle.CI_Profile { import isabelle._ import java.io.FileReader import scala.sys.process._ import org.apache.commons.configuration2._ override def clean = false val afp = Path.explode("$AFP_BASE") val afp_thys = afp + Path.explode("thys") val afp_id = hg_id(afp) sealed abstract class Status(val str: String) { def merge(that: Status): Status = (this, that) match { case (Ok, s) => s case (Failed, s) => Failed case (Skipped, Failed) => Failed case (Skipped, s) => Skipped } } object Status { def merge(statuses: List[Status]): Status = statuses.foldLeft(Ok: Status)(_ merge _) def from_results(results: Build.Results, session: String): Status = if (results.cancelled(session)) Skipped else if (results(session).ok) Ok else Failed } case object Ok extends Status("ok") case object Skipped extends Status("skipped") case object Failed extends Status("failed") class Metadata(ini: INIConfiguration) { def entry_of_session(info: Sessions.Info): Option[String] = if (info.dir.dir.file == afp_thys.file) Some(info.dir.base.implode) else None def group_by_entry(results: Build.Results): Map[Option[String], List[String]] = results.sessions.toList.map { name => entry_of_session(results.info(name)) -> name - }.groupBy(_._1).mapValues(_.map(_._2)) + }.groupBy(_._1).view.mapValues(_.map(_._2)).toMap def status_as_html(status: Map[Option[String], Status]): String = { val entries_strings = status.collect { case (None, Failed) => s"
  • Distribution
  • " case (Some(entry), Failed) => s"""
  • $entry
  • """ } if (entries_strings.isEmpty) "" else entries_strings.mkString("Failed entries: ") } } val report_file = Path.explode("$ISABELLE_HOME/report.html").file def include = List(afp_thys) def select = Nil def pre_hook(args: List[String]) = { println(s"AFP id $afp_id") if (report_file.exists()) report_file.delete() File.write(report_file, "") } def post_hook(results: Build.Results) = { val metadata = { val path = afp + Path.explode("metadata/metadata") val ini = new INIConfiguration() if (path.is_file) { val reader = new FileReader(path.file) ini.read(reader) reader.close() } new Metadata(ini) } val status = - metadata.group_by_entry(results).mapValues { sessions => + metadata.group_by_entry(results).view.mapValues { sessions => Status.merge(sessions.map(Status.from_results(results, _))) - } + }.toMap print_section("REPORT") println("Writing report file ...") File.write(report_file, metadata.status_as_html(status)) print_section("COMPLETED") } def selection = Sessions.Selection( all_sessions = true, exclude_session_groups = List("slow")) } diff --git a/tools/afp_check_roots.scala b/tools/afp_check_roots.scala --- a/tools/afp_check_roots.scala +++ b/tools/afp_check_roots.scala @@ -1,134 +1,134 @@ // DESCRIPTION: check ROOT files of AFP sessions object AFP_Check_Roots extends isabelle.Isabelle_Tool.Body { import isabelle._ val afp_dir = Path.explode("$AFP").expand val excludes = List("ROOTS", "LICENSE", "LICENSE.LGPL", ".DS_Store") def print_good(string: String): Unit = println(Console.BOLD + Console.GREEN + string + Console.RESET) def print_bad(string: String): Unit = println(Console.BOLD + Console.RED + string + Console.RESET) class Check[T]( run: (Sessions.Structure, List[String]) => List[T], failure_msg: String, failure_format: T => String) { def apply(tree: Sessions.Structure, selected: List[String]): Boolean = run(tree, selected) match { case Nil => true case offenders => print_bad(failure_msg) offenders.foreach(offender => println(" " + failure_format(offender))) false } } val check_timeout = new Check[(String, List[String])]( run = { (tree, selected) => selected.flatMap { name => val info = tree(name) val entry = info.dir.base.implode val timeout = info.options.real("timeout") if (timeout == 0 || timeout % 300 != 0) Some((entry, name)) else None - }.groupBy(_._1).mapValues(_.map(_._2)).toList + }.groupBy(_._1).view.mapValues(_.map(_._2)).toList }, failure_msg = "The following entries contain sessions without timeouts or with timeouts not divisible by 300:", failure_format = { case (entry, sessions) => s"""$entry ${sessions.mkString("(", ", ", ")")}""" } ) val check_paths = new Check[(String, Path)]( run = { (tree, selected) => selected.flatMap { name => val info = tree(name) val dir = info.dir if (dir.dir.expand.file != afp_dir.file) Some((name, dir)) else None } }, failure_msg = "The following sessions are in the wrong directory:", failure_format = { case (session, dir) => s"""$session ($dir)""" } ) val check_chapter = new Check[String]( run = { (tree, selected) => selected.flatMap { name => val info = tree(name) val entry = info.dir.base.implode if (info.chapter != "AFP") Some(entry) else None }.distinct }, failure_msg = "The following entries are not in the AFP chapter:", failure_format = identity ) val check_groups = new Check[(String, List[String])]( run = { (tree, selected) => selected.flatMap { name => val info = tree(name) if (!info.groups.toSet.subsetOf(AFP.groups.keySet + "AFP") || !info.groups.contains("AFP")) Some((name, info.groups)) else None } }, failure_msg = "The following sessions have wrong groups:", failure_format = { case (session, groups) => s"""$session ${groups.mkString("{", ", ", "}")}""" } ) val check_presence = new Check[String]( run = { (tree, selected) => val fs_entries = File.read_dir(afp_dir).filterNot(excludes.contains) fs_entries.flatMap { name => if (!selected.contains(name) || tree(name).dir.base.implode != name) Some(name) else None } }, failure_msg = "The following entries (according to the file system) are not registered in ROOTS, or registered in the wrong ROOT:", failure_format = identity ) def apply(args: List[String]): Unit = { val full_tree = Sessions.load_structure(Options.init(), Nil, List(afp_dir)) val selected = full_tree.build_selection(Sessions.Selection.empty) val checks = List( check_timeout, // check_paths, check_chapter, check_groups, check_presence) val bad = checks.exists(check => !check(full_tree, selected)) if (bad) { print_bad("Errors found.") System.exit(1) } else { print_good(s"${selected.length} sessions have been checked") print_good(s"${checks.length} checks have found no errors") } } }