diff --git a/src/Pure/Admin/build_jdk.scala b/src/Pure/Admin/build_jdk.scala --- a/src/Pure/Admin/build_jdk.scala +++ b/src/Pure/Admin/build_jdk.scala @@ -1,235 +1,235 @@ /* Title: Pure/Admin/build_jdk.scala Author: Makarius Build Isabelle jdk component from original platform installations. */ package isabelle import java.io.{File => JFile} import java.nio.file.Files import java.nio.file.attribute.PosixFilePermission import scala.util.matching.Regex object Build_JDK { /* platform and version information */ sealed case class JDK_Platform( platform_name: String, platform_regex: Regex, exe: String = "java", macos_home: Boolean = false, jdk_version: String = "" ) { override def toString: String = platform_name def platform_path: Path = Path.explode(platform_name) def detect(jdk_dir: Path): Option[JDK_Platform] = { val major_version = { val Major_Version = """.*jdk(\d+).*$""".r val jdk_name = jdk_dir.file.getName jdk_name match { case Major_Version(s) => s case _ => error("Cannot determine major version from " + quote(jdk_name)) } } val path = jdk_dir + Path.explode("bin") + Path.explode(exe) if (path.is_file) { val file_descr = Isabelle_System.bash("file -b " + File.bash_path(path)).check.out if (platform_regex.pattern.matcher(file_descr).matches) { val Version = ("^(" + major_version + """[0-9.+]+)(?:-LTS)?$""").r val version_lines = Isabelle_System.bash("strings " + File.bash_path(path)).check .out_lines.flatMap({ case Version(s) => Some(s) case _ => None }) version_lines match { case List(jdk_version) => Some(copy(jdk_version = jdk_version)) case _ => error("Expected unique version within executable " + path) } } else None } else None } } val templates: List[JDK_Platform] = List( JDK_Platform("arm64-darwin", """.*Mach-O 64-bit.*arm64.*""".r, macos_home = true), JDK_Platform("arm64-linux", """.*ELF 64-bit.*ARM aarch64.*""".r), JDK_Platform("x86_64-darwin", """.*Mach-O 64-bit.*x86[-_]64.*""".r, macos_home = true), JDK_Platform("x86_64-linux", """.*ELF 64-bit.*x86[-_]64.*""".r), JDK_Platform("x86_64-windows", """.*PE32\+ executable.*x86[-_]64.*""".r, exe = "java.exe")) /* README */ def readme(jdk_version: String): String = -"""This is OpenJDK """ + jdk_version + """ based on downloads by Azul, see also + """This is OpenJDK """ + jdk_version + """ based on downloads by Azul, see also https://www.azul.com/downloads/zulu-community/?package=jdk The main license is GPL2, but some modules are covered by other (more liberal) licenses, see legal/* for details. Linux, Windows, macOS all work uniformly, depending on platform-specific subdirectories. """ /* settings */ val settings: String = -"""# -*- shell-script -*- :mode=shellscript: + """# -*- shell-script -*- :mode=shellscript: case "$ISABELLE_PLATFORM_FAMILY" in linux) ISABELLE_JAVA_PLATFORM="$ISABELLE_PLATFORM64" ISABELLE_JDK_HOME="$COMPONENT/$ISABELLE_JAVA_PLATFORM" ;; windows) ISABELLE_JAVA_PLATFORM="$ISABELLE_WINDOWS_PLATFORM64" ISABELLE_JDK_HOME="$COMPONENT/$ISABELLE_JAVA_PLATFORM" ;; macos) if [ -n "$ISABELLE_APPLE_PLATFORM64" -a -d "$COMPONENT/$ISABELLE_APPLE_PLATFORM64" ] then ISABELLE_JAVA_PLATFORM="$ISABELLE_APPLE_PLATFORM64" else ISABELLE_JAVA_PLATFORM="$ISABELLE_PLATFORM64" fi ISABELLE_JDK_HOME="$COMPONENT/$ISABELLE_JAVA_PLATFORM" ;; esac """ /* extract archive */ def extract_archive(dir: Path, archive: Path): JDK_Platform = { try { val tmp_dir = Isabelle_System.make_directory(dir + Path.explode("tmp")) if (archive.get_ext == "zip") { Isabelle_System.bash( "unzip -x " + File.bash_path(archive.absolute), cwd = tmp_dir.file).check } else { Isabelle_System.gnutar("-xzf " + File.bash_path(archive), dir = tmp_dir).check } val dir_entry = File.read_dir(tmp_dir) match { case List(s) => s case _ => error("Archive contains multiple directories") } val jdk_dir = tmp_dir + Path.explode(dir_entry) val platform = templates.view.flatMap(_.detect(jdk_dir)) .headOption.getOrElse(error("Failed to detect JDK platform")) val platform_dir = dir + platform.platform_path if (platform_dir.is_dir) error("Directory already exists: " + platform_dir) Isabelle_System.move_file(jdk_dir, platform_dir) platform } catch { case ERROR(msg) => cat_error(msg, "The error(s) above occurred for " + archive) } } /* build jdk */ def build_jdk( archives: List[Path], progress: Progress = new Progress, target_dir: Path = Path.current ): Unit = { if (Platform.is_windows) error("Cannot build jdk on Windows") Isabelle_System.with_tmp_dir("jdk") { dir => progress.echo("Extracting ...") val platforms = archives.map(extract_archive(dir, _)) val jdk_version = platforms.map(_.jdk_version).distinct match { case List(version) => version case Nil => error("No archives") case versions => error("Archives contain multiple JDK versions: " + commas_quote(versions)) } templates.filterNot(p1 => platforms.exists(p2 => p1.platform_name == p2.platform_name)) match { case Nil => case missing => error("Missing platforms: " + commas_quote(missing.map(_.platform_name))) } val jdk_name = "jdk-" + jdk_version val jdk_path = Path.explode(jdk_name) val component_dir = dir + jdk_path Isabelle_System.make_directory(component_dir + Path.explode("etc")) File.write(Components.settings(component_dir), settings) File.write(component_dir + Path.explode("README"), readme(jdk_version)) for (platform <- platforms) { Isabelle_System.move_file(dir + platform.platform_path, component_dir) } for (file <- File.find_files(component_dir.file, include_dirs = true)) { val path = file.toPath val perms = Files.getPosixFilePermissions(path) perms.add(PosixFilePermission.OWNER_READ) perms.add(PosixFilePermission.GROUP_READ) perms.add(PosixFilePermission.OTHERS_READ) perms.add(PosixFilePermission.OWNER_WRITE) if (file.isDirectory) { perms.add(PosixFilePermission.OWNER_WRITE) perms.add(PosixFilePermission.OWNER_EXECUTE) perms.add(PosixFilePermission.GROUP_EXECUTE) perms.add(PosixFilePermission.OTHERS_EXECUTE) } Files.setPosixFilePermissions(path, perms) } progress.echo("Archiving ...") Isabelle_System.gnutar( "-czf " + File.bash_path(target_dir + jdk_path.tar.gz) + " " + jdk_name, dir = dir).check } } /* Isabelle tool wrapper */ val isabelle_tool = Isabelle_Tool("build_jdk", "build Isabelle jdk component from original archives", Scala_Project.here, { args => var target_dir = Path.current val getopts = Getopts(""" Usage: isabelle build_jdk [OPTIONS] ARCHIVES... Options are: -D DIR target directory (default ".") Build jdk component from tar.gz archives, with original jdk archives for Linux, Windows, and macOS. """, "D:" -> (arg => target_dir = Path.explode(arg))) val more_args = getopts(args) if (more_args.isEmpty) getopts.usage() val archives = more_args.map(Path.explode) val progress = new Console_Progress() build_jdk(archives = archives, progress = progress, target_dir = target_dir) }) } diff --git a/src/Pure/General/symbol.scala b/src/Pure/General/symbol.scala --- a/src/Pure/General/symbol.scala +++ b/src/Pure/General/symbol.scala @@ -1,668 +1,669 @@ /* Title: Pure/General/symbol.scala Author: Makarius Isabelle text symbols. */ package isabelle import scala.collection.mutable import scala.util.matching.Regex import scala.annotation.tailrec object Symbol { type Symbol = String // counting Isabelle symbols, starting from 1 type Offset = Text.Offset type Range = Text.Range /* spaces */ val space_char = ' ' val space = " " private val static_spaces = space * 4000 def spaces(n: Int): String = { require(n >= 0, "negative spaces") if (n < static_spaces.length) static_spaces.substring(0, n) else space * n } /* char symbols */ private val char_symbols: Array[Symbol] = (0 until 0x500).iterator.map(i => new String(Array(i.toChar))).toArray def char_symbol(c: Char): String = if (c < char_symbols.length) char_symbols(c) else c.toString /* ASCII characters */ def is_ascii_printable(c: Char): Boolean = space_char <= c && c <= '~' def is_ascii_letter(c: Char): Boolean = 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' def is_ascii_digit(c: Char): Boolean = '0' <= c && c <= '9' def is_ascii_hex(c: Char): Boolean = '0' <= c && c <= '9' || 'A' <= c && c <= 'F' || 'a' <= c && c <= 'f' def is_ascii_quasi(c: Char): Boolean = c == '_' || c == '\'' def is_ascii_blank(c: Char): Boolean = " \t\n\u000b\f\r".contains(c) def is_ascii_line_terminator(c: Char): Boolean = "\r\n".contains(c) def is_ascii_letdig(c: Char): Boolean = is_ascii_letter(c) || is_ascii_digit(c) || is_ascii_quasi(c) def is_ascii_identifier(s: String): Boolean = s.nonEmpty && is_ascii_letter(s(0)) && s.forall(is_ascii_letdig) def ascii(c: Char): Symbol = { if (c > 127) error("Non-ASCII character: " + quote(c.toString)) else char_symbol(c) } def is_ascii(s: Symbol): Boolean = s.length == 1 && s(0) < 128 /* symbol matching */ def is_malformed(s: Symbol): Boolean = s.length match { case 1 => val c = s(0) Character.isHighSurrogate(c) || Character.isLowSurrogate(c) || c == '\ufffd' case 2 => val c1 = s(0) val c2 = s(1) !(c1 == '\r' && c2 == '\n' || Character.isSurrogatePair(c1, c2)) case _ => !s.endsWith(">") || s == "\\<>" || s == "\\<^>" } def is_newline(s: Symbol): Boolean = s == "\n" || s == "\r" || s == "\r\n" class Matcher(text: CharSequence) { private def ok(i: Int): Boolean = 0 <= i && i < text.length private def char(i: Int): Char = if (ok(i)) text.charAt(i) else 0 private def maybe_char(c: Char, i: Int): Int = if (char(i) == c) i + 1 else i @tailrec private def many_ascii_letdig(i: Int): Int = if (is_ascii_letdig(char(i))) many_ascii_letdig(i + 1) else i private def maybe_ascii_id(i: Int): Int = if (is_ascii_letter(char(i))) many_ascii_letdig(i + 1) else i def match_length(i: Int): Int = { val a = char(i) val b = char(i + 1) if (Character.isHighSurrogate(a) && Character.isLowSurrogate(b) || a == '\r' && b == '\n') 2 else if (a == '\\' && b == '<') maybe_char('>', maybe_ascii_id(maybe_char('^', i + 2))) - i else if (ok(i)) 1 else 0 } def match_symbol(i: Int): String = match_length(i) match { case 0 => "" case 1 => char_symbol(text.charAt(i)) case n => text.subSequence(i, i + n).toString } } /* iterator */ def iterator(text: CharSequence): Iterator[Symbol] = new Iterator[Symbol] { private val matcher = new Matcher(text) private var i = 0 def hasNext: Boolean = i < text.length def next(): Symbol = { val s = matcher.match_symbol(i) i += s.length s } } def explode(text: CharSequence): List[Symbol] = iterator(text).toList def length(text: CharSequence): Int = iterator(text).length def trim_blanks(text: CharSequence): String = Library.trim(is_blank, explode(text)).mkString def all_blank(str: String): Boolean = iterator(str).forall(is_blank) def trim_blank_lines(text: String): String = cat_lines(split_lines(text).dropWhile(all_blank).reverse.dropWhile(all_blank).reverse) /* decoding offsets */ object Index { private sealed case class Entry(chr: Int, sym: Int) val empty: Index = new Index(Nil) def apply(text: CharSequence): Index = { val matcher = new Matcher(text) val buf = new mutable.ListBuffer[Entry] var chr = 0 var sym = 0 while (chr < text.length) { val n = matcher.match_length(chr) chr += n sym += 1 if (n > 1) buf += Entry(chr, sym) } if (buf.isEmpty) empty else new Index(buf.toList) } } final class Index private(entries: List[Index.Entry]) { private val hash: Int = entries.hashCode private val index: Array[Index.Entry] = entries.toArray def decode(symbol_offset: Offset): Text.Offset = { val sym = symbol_offset - 1 val end = index.length @tailrec def bisect(a: Int, b: Int): Int = { if (a < b) { val c = (a + b) / 2 if (sym < index(c).sym) bisect(a, c) else if (c + 1 == end || sym < index(c + 1).sym) c else bisect(c + 1, b) } else -1 } val i = bisect(0, end) if (i < 0) sym else index(i).chr + sym - index(i).sym } def decode(symbol_range: Range): Text.Range = symbol_range.map(decode) override def hashCode: Int = hash override def equals(that: Any): Boolean = that match { case other: Index => index.sameElements(other.index) case _ => false } } /* symbolic text chunks -- without actual text */ object Text_Chunk { sealed abstract class Name case object Default extends Name case class Id(id: Document_ID.Generic) extends Name case class File(name: String) extends Name def apply(text: CharSequence): Text_Chunk = new Text_Chunk(Text.Range(0, text.length), Index(text)) } final class Text_Chunk private(val range: Text.Range, private val index: Index) { override def hashCode: Int = (range, index).hashCode override def equals(that: Any): Boolean = that match { case other: Text_Chunk => range == other.range && index == other.index case _ => false } override def toString: String = "Text_Chunk" + range.toString def decode(symbol_offset: Offset): Text.Offset = index.decode(symbol_offset) def decode(symbol_range: Range): Text.Range = index.decode(symbol_range) def incorporate(symbol_range: Range): Option[Text.Range] = { - def in(r: Range): Option[Text.Range] = + def in(r: Range): Option[Text.Range] = { range.try_restrict(decode(r)) match { case Some(r1) if !r1.is_singularity => Some(r1) case _ => None } - in(symbol_range) orElse in(symbol_range - 1) + } + in(symbol_range) orElse in(symbol_range - 1) } } /* recoding text */ private class Recoder(list: List[(String, String)]) { private val (min, max) = { var min = '\uffff' var max = '\u0000' for ((x, _) <- list) { val c = x(0) if (c < min) min = c if (c > max) max = c } (min, max) } private val table = { var tab = Map[String, String]() for ((x, y) <- list) { tab.get(x) match { case None => tab += (x -> y) case Some(z) => error("Duplicate symbol mapping of " + quote(x) + " to " + quote(y) + " vs. " + quote(z)) } } tab } def recode(text: String): String = { val len = text.length val matcher = new Symbol.Matcher(text) val result = new StringBuilder(len) var i = 0 while (i < len) { val c = text(i) if (min <= c && c <= max) { val s = matcher.match_symbol(i) result.append(table.getOrElse(s, s)) i += s.length } else { result.append(c); i += 1 } } result.toString } } /** defined symbols **/ object Argument extends Enumeration { val none, cartouche, space_cartouche = Value def unapply(s: String): Option[Value] = try { Some(withName(s)) } catch { case _: NoSuchElementException => None} } object Entry { private val Name = new Regex("""\\<\^?([A-Za-z][A-Za-z0-9_']*)>""") private val Argument = new Properties.String("argument") private val Abbrev = new Properties.String("abbrev") private val Code = new Properties.String("code") private val Font = new Properties.String("font") private val Group = new Properties.String("group") def apply(symbol: Symbol, props: Properties.T): Entry = { def err(msg: String): Nothing = error(msg + " for symbol " + quote(symbol)) val name = symbol match { case Name(a) => a case _ => err("Cannot determine name") } val argument = props match { case Argument(arg) => Symbol.Argument.unapply(arg) getOrElse error("Bad argument: " + quote(arg) + " for symbol " + quote(symbol)) case _ => Symbol.Argument.none } val code = props match { case Code(s) => try { val code = Integer.decode(s).intValue if (code >= 128) Some(code) else err("Illegal ASCII code") } catch { case _: NumberFormatException => err("Bad code") } case _ => None } val groups = proper_list(for ((Group.name, a) <- props) yield a).getOrElse(List("unsorted")) val abbrevs = for ((Abbrev.name, a) <- props) yield a new Entry(symbol, name, argument, code, Font.unapply(props), groups, abbrevs) } } class Entry private( val symbol: Symbol, val name: String, val argument: Symbol.Argument.Value, val code: Option[Int], val font: Option[String], val groups: List[String], val abbrevs: List[String] ) { override def toString: String = symbol val decode: Option[String] = code.map(c => new String(Character.toChars(c))) } lazy val symbols: Symbols = Symbols.load() object Symbols { def load(static: Boolean = false): Symbols = { val paths = if (static) List(Path.explode("~~/etc/symbols")) else Path.split(Isabelle_System.getenv("ISABELLE_SYMBOLS")) make(cat_lines(for (path <- paths if path.is_file) yield File.read(path))) } def make(symbols_spec: String): Symbols = { val No_Decl = new Regex("""(?xs) ^\s* (?: \#.* )? $ """) val Key = new Regex("""(?xs) (.+): """) def read_decl(decl: String): (Symbol, Properties.T) = { def err() = error("Bad symbol declaration: " + decl) def read_props(props: List[String]): Properties.T = { props match { case Nil => Nil case _ :: Nil => err() case Key(x) :: y :: rest => (x -> y.replace('\u2423', ' ')) :: read_props(rest) case _ => err() } } decl.split("\\s+").toList match { case sym :: props if sym.length > 1 && !is_malformed(sym) => (sym, read_props(props)) case _ => err() } } new Symbols( split_lines(symbols_spec).reverse. foldLeft((List.empty[Entry], Set.empty[Symbol])) { case (res, No_Decl()) => res case ((list, known), decl) => val (sym, props) = read_decl(decl) if (known(sym)) (list, known) else (Entry(sym, props) :: list, known + sym) }._1) } } class Symbols(val entries: List[Entry]) { override def toString: String = entries.mkString("Symbols(", ", ", ")") /* basic properties */ private val entries_map: Map[Symbol, Entry] = (for (entry <- entries.iterator) yield entry.symbol -> entry).toMap def get(sym: Symbol): Option[Entry] = entries_map.get(sym) def defined(sym: Symbol): Boolean = entries_map.isDefinedAt(sym) def defined_code(sym: Symbol): Boolean = get(sym) match { case Some(entry) => entry.code.isDefined case _ => false } val code_defined: Int => Boolean = (for (entry <- entries.iterator; code <- entry.code) yield code).toSet val groups_code: List[(String, List[Symbol])] = (for { entry <- entries.iterator if entry.code.isDefined group <- entry.groups.iterator } yield entry.symbol -> group) .toList.groupBy(_._2).toList.map({ case (g, list) => (g, list.map(_._1)) }).sortBy(_._1) def get_abbrevs(sym: Symbol): List[String] = get(sym) match { case Some(entry) => entry.abbrevs case _ => Nil } /* recoding */ private val (decoder, encoder) = { val mapping = for (entry <- entries; s <- entry.decode) yield entry.symbol -> s (new Recoder(mapping), new Recoder(for ((x, y) <- mapping) yield (y, x))) } def decode(text: String): String = decoder.recode(text) def encode(text: String): String = encoder.recode(text) private def recode_set(elems: Iterable[String]): Set[String] = (elems.iterator ++ elems.iterator.map(decode)).toSet private def recode_map[A](elems: Iterable[(String, A)]): Map[String, A] = (elems.iterator ++ elems.iterator.map({ case (sym, a) => (decode(sym), a) })).toMap /* user fonts */ val fonts: Map[Symbol, String] = recode_map(for (entry <- entries; font <- entry.font) yield entry.symbol -> font) val font_names: List[String] = fonts.iterator.map(_._2).toSet.toList val font_index: Map[String, Int] = (font_names zip font_names.indices.toList).toMap def lookup_font(sym: Symbol): Option[Int] = fonts.get(sym).map(font_index(_)) /* classification */ val letters: Set[String] = recode_set(List( "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\

", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\

", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\
", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\")) val blanks: Set[String] = recode_set(List(space, "\t", "\n", "\u000B", "\f", "\r", "\r\n")) val sym_chars = Set("!", "#", "$", "%", "&", "*", "+", "-", "/", "<", "=", ">", "?", "@", "^", "_", "|", "~") val symbolic: Set[String] = recode_set(for (entry <- entries if raw_symbolic(entry.symbol)) yield entry.symbol) /* misc symbols */ val newline_decoded: Symbol = decode(newline) val comment_decoded: Symbol = decode(comment) val cancel_decoded: Symbol = decode(cancel) val latex_decoded: Symbol = decode(latex) val marker_decoded: Symbol = decode(marker) val open_decoded: Symbol = decode(open) val close_decoded: Symbol = decode(close) /* control symbols */ val control_decoded: Set[Symbol] = (for (entry <- entries.iterator if entry.symbol.startsWith("\\<^"); s <- entry.decode) yield s).toSet val sub_decoded: Symbol = decode(sub) val sup_decoded: Symbol = decode(sup) val bold_decoded: Symbol = decode(bold) val emph_decoded: Symbol = decode(emph) val bsub_decoded: Symbol = decode(bsub) val esub_decoded: Symbol = decode(esub) val bsup_decoded: Symbol = decode(bsup) val esup_decoded: Symbol = decode(esup) } /* tables */ def decode(text: String): String = symbols.decode(text) def encode(text: String): String = symbols.encode(text) def decode_yxml(text: String, cache: XML.Cache = XML.Cache.none): XML.Body = YXML.parse_body(decode(text), cache = cache) def decode_yxml_failsafe(text: String, cache: XML.Cache = XML.Cache.none): XML.Body = YXML.parse_body_failsafe(decode(text), cache = cache) def encode_yxml(body: XML.Body): String = encode(YXML.string_of_body(body)) def decode_strict(text: String): String = { val decoded = decode(text) if (encode(decoded) == text) decoded else { val bad = new mutable.ListBuffer[Symbol] for (s <- iterator(text) if encode(decode(s)) != s && !bad.contains(s)) bad += s error("Bad Unicode symbols in text: " + commas_quote(bad)) } } def output(unicode_symbols: Boolean, text: String): String = if (unicode_symbols) Symbol.decode(text) else Symbol.encode(text) /* classification */ def is_letter(sym: Symbol): Boolean = symbols.letters.contains(sym) def is_digit(sym: Symbol): Boolean = sym.length == 1 && '0' <= sym(0) && sym(0) <= '9' def is_quasi(sym: Symbol): Boolean = sym == "_" || sym == "'" def is_letdig(sym: Symbol): Boolean = is_letter(sym) || is_digit(sym) || is_quasi(sym) def is_blank(sym: Symbol): Boolean = symbols.blanks.contains(sym) /* symbolic newline */ val newline: Symbol = "\\" def newline_decoded: Symbol = symbols.newline_decoded def print_newlines(str: String): String = if (str.contains('\n')) (for (s <- iterator(str)) yield { if (s == "\n") newline_decoded else s }).mkString else str /* formal comments */ val comment: Symbol = "\\" val cancel: Symbol = "\\<^cancel>" val latex: Symbol = "\\<^latex>" val marker: Symbol = "\\<^marker>" def comment_decoded: Symbol = symbols.comment_decoded def cancel_decoded: Symbol = symbols.cancel_decoded def latex_decoded: Symbol = symbols.latex_decoded def marker_decoded: Symbol = symbols.marker_decoded /* cartouches */ val open: Symbol = "\\" val close: Symbol = "\\" def open_decoded: Symbol = symbols.open_decoded def close_decoded: Symbol = symbols.close_decoded def is_open(sym: Symbol): Boolean = sym == open_decoded || sym == open def is_close(sym: Symbol): Boolean = sym == close_decoded || sym == close def cartouche(s: String): String = open + s + close def cartouche_decoded(s: String): String = open_decoded + s + close_decoded /* symbols for symbolic identifiers */ private def raw_symbolic(sym: Symbol): Boolean = sym.startsWith("\\<") && sym.endsWith(">") && !sym.startsWith("\\<^") def is_symbolic(sym: Symbol): Boolean = !is_open(sym) && !is_close(sym) && (raw_symbolic(sym) || symbols.symbolic.contains(sym)) def is_symbolic_char(sym: Symbol): Boolean = symbols.sym_chars.contains(sym) /* control symbols */ val control_prefix = "\\<^" val control_suffix = ">" def control_name(sym: Symbol): Option[String] = if (is_control_encoded(sym)) Some(sym.substring(control_prefix.length, sym.length - control_suffix.length)) else None def is_control_encoded(sym: Symbol): Boolean = sym.startsWith(control_prefix) && sym.endsWith(control_suffix) def is_control(sym: Symbol): Boolean = is_control_encoded(sym) || symbols.control_decoded.contains(sym) def is_controllable(sym: Symbol): Boolean = !is_blank(sym) && !is_control(sym) && !is_open(sym) && !is_close(sym) && !is_malformed(sym) && sym != "\"" val sub: Symbol = "\\<^sub>" val sup: Symbol = "\\<^sup>" val bold: Symbol = "\\<^bold>" val emph: Symbol = "\\<^emph>" val bsub: Symbol = "\\<^bsub>" val esub: Symbol = "\\<^esub>" val bsup: Symbol = "\\<^bsup>" val esup: Symbol = "\\<^esup>" def sub_decoded: Symbol = symbols.sub_decoded def sup_decoded: Symbol = symbols.sup_decoded def bold_decoded: Symbol = symbols.bold_decoded def emph_decoded: Symbol = symbols.emph_decoded def bsub_decoded: Symbol = symbols.bsub_decoded def esub_decoded: Symbol = symbols.esub_decoded def bsup_decoded: Symbol = symbols.bsup_decoded def esup_decoded: Symbol = symbols.esup_decoded /* metric */ def is_printable(sym: Symbol): Boolean = if (is_ascii(sym)) is_ascii_printable(sym(0)) else !is_control(sym) object Metric extends Pretty.Metric { val unit = 1.0 def apply(str: String): Double = (for (s <- iterator(str)) yield { val sym = encode(s) if (sym.startsWith("\\ empty case head :: tail => new Markup_Tree( tail.foldLeft(head.branches) { case (branches, tree) => tree.branches.foldLeft(branches) { case (bs, (r, entry)) => require(!bs.isDefinedAt(r), "cannot merge markup trees") bs + (r -> entry) } }) } /* tree building blocks */ object Entry { def apply(markup: Text.Markup, subtree: Markup_Tree): Entry = Entry(markup.range, List(markup.info), subtree) } sealed case class Entry( range: Text.Range, rev_markup: List[XML.Elem], subtree: Markup_Tree ) { def markup: List[XML.Elem] = rev_markup.reverse def filter_markup(elements: Markup.Elements): List[XML.Elem] = { var result: List[XML.Elem] = Nil for (elem <- rev_markup if elements(elem.name)) result ::= elem result } def + (markup: Text.Markup): Entry = copy(rev_markup = markup.info :: rev_markup) def \ (markup: Text.Markup): Entry = copy(subtree = subtree + markup) } object Branches { type T = SortedMap[Text.Range, Entry] val empty: T = SortedMap.empty(Text.Range.Ordering) } /* XML representation */ @tailrec private def strip_elems( elems: List[XML.Elem], body: XML.Body): (List[XML.Elem], XML.Body) = body match { case List(XML.Wrapped_Elem(markup1, body1, body2)) => strip_elems(XML.Elem(markup1, body1) :: elems, body2) case List(XML.Elem(markup1, body1)) => strip_elems(XML.Elem(markup1, Nil) :: elems, body1) case _ => (elems, body) } private def make_trees( acc: (Int, List[Markup_Tree]), tree: XML.Tree ): (Int, List[Markup_Tree]) = { val (offset, markup_trees) = acc strip_elems(Nil, List(tree)) match { case (Nil, body) => (offset + XML.text_length(body), markup_trees) case (elems, body) => val (end_offset, subtrees) = body.foldLeft((offset, List.empty[Markup_Tree]))(make_trees) if (offset == end_offset) acc else { val range = Text.Range(offset, end_offset) val entry = Entry(range, elems, merge_disjoint(subtrees)) (end_offset, new Markup_Tree(Branches.empty, entry) :: markup_trees) } } } def from_XML(body: XML.Body): Markup_Tree = merge_disjoint(body.foldLeft((0, List.empty[Markup_Tree]))(make_trees)._2) } final class Markup_Tree private(val branches: Markup_Tree.Branches.T) { import Markup_Tree._ private def this(branches: Markup_Tree.Branches.T, entry: Markup_Tree.Entry) = this(branches + (entry.range -> entry)) private def overlapping(range: Text.Range): Branches.T = if (branches.isEmpty || (range.contains(branches.firstKey.start) && branches.lastKey.stop <= range.stop)) branches else { val start = Text.Range(range.start) val stop = Text.Range(range.stop) val bs = branches.range(start, stop) branches.get(stop) match { case Some(end) if range overlaps end.range => bs + (end.range -> end) case _ => bs } } def restrict(range: Text.Range): Markup_Tree = new Markup_Tree(overlapping(range)) def is_empty: Boolean = branches.isEmpty def + (new_markup: Text.Markup): Markup_Tree = { val new_range = new_markup.range branches.get(new_range) match { case None => new Markup_Tree(branches, Entry(new_markup, empty)) case Some(entry) => if (entry.range == new_range) new Markup_Tree(branches, entry + new_markup) else if (entry.range.contains(new_range)) new Markup_Tree(branches, entry \ new_markup) else if (new_range.contains(branches.head._1) && new_range.contains(branches.last._1)) new Markup_Tree(Branches.empty, Entry(new_markup, this)) else { val body = overlapping(new_range) if (body.forall(e => new_range.contains(e._1))) new Markup_Tree(branches -- body.keys, Entry(new_markup, new Markup_Tree(body))) else { Output.warning("Ignored overlapping markup information: " + new_markup + "\n" + body.filter(e => !new_range.contains(e._1)).values.mkString("\n")) this } } } } def merge(other: Markup_Tree, root_range: Text.Range, elements: Markup.Elements): Markup_Tree = { def merge_trees(tree1: Markup_Tree, tree2: Markup_Tree): Markup_Tree = tree2.branches.foldLeft(tree1) { case (tree, (range, entry)) => if (!range.overlaps(root_range)) tree else { entry.filter_markup(elements).foldLeft(merge_trees(tree, entry.subtree)) { case (t, elem) => t + Text.Info(range, elem) } } } if (this eq other) this else { val tree1 = this.restrict(root_range) val tree2 = other.restrict(root_range) if (tree1.is_empty) tree2 else merge_trees(tree1, tree2) } } def cumulate[A]( root_range: Text.Range, root_info: A, elements: Markup.Elements, result: (A, Text.Markup) => Option[A] ): List[Text.Info[A]] = { def results(x: A, entry: Entry): Option[A] = { var y = x var changed = false for { elem <- entry.filter_markup(elements) y1 <- result(y, Text.Info(entry.range, elem)) } { y = y1; changed = true } if (changed) Some(y) else None } def traverse( last: Text.Offset, stack: List[(Text.Info[A], List[(Text.Range, Entry)])] ): List[Text.Info[A]] = { stack match { case (parent, (range, entry) :: more) :: rest => val subrange = range.restrict(root_range) val subtree = entry.subtree.overlapping(subrange).toList val start = subrange.start results(parent.info, entry) match { case Some(res) => val next = Text.Info(subrange, res) val nexts = traverse(start, (next, subtree) :: (parent, more) :: rest) if (last < start) parent.restrict(Text.Range(last, start)) :: nexts else nexts case None => traverse(last, (parent, subtree ::: more) :: rest) } case (parent, Nil) :: rest => val stop = parent.range.stop val nexts = traverse(stop, rest) if (last < stop) parent.restrict(Text.Range(last, stop)) :: nexts else nexts case Nil => val stop = root_range.stop if (last < stop) List(Text.Info(Text.Range(last, stop), root_info)) else Nil } } traverse(root_range.start, List((Text.Info(root_range, root_info), overlapping(root_range).toList))) } def to_XML(root_range: Text.Range, text: CharSequence, elements: Markup.Elements): XML.Body = { def make_text(start: Text.Offset, stop: Text.Offset): XML.Body = if (start == stop) Nil else List(XML.Text(text.subSequence(start, stop).toString)) def make_elems(rev_markups: List[XML.Elem], body: XML.Body): XML.Body = rev_markups.foldLeft(body) { case (b, elem) => if (!elements(elem.name)) b else if (elem.body.isEmpty) List(XML.Elem(elem.markup, b)) else List(XML.Wrapped_Elem(elem.markup, elem.body, b)) } def make_body( elem_range: Text.Range, elem_markup: List[XML.Elem], entries: Branches.T ) : XML.Body = { val body = new mutable.ListBuffer[XML.Tree] var last = elem_range.start for ((range, entry) <- entries) { val subrange = range.restrict(elem_range) body ++= make_text(last, subrange.start) body ++= make_body(subrange, entry.rev_markup, entry.subtree.overlapping(subrange)) last = subrange.stop } body ++= make_text(last, elem_range.stop) make_elems(elem_markup, body.toList) } - make_body(root_range, Nil, overlapping(root_range)) + make_body(root_range, Nil, overlapping(root_range)) } override def toString: String = branches.toList.map(_._2) match { case Nil => "Empty" case list => list.mkString("Tree(", ",", ")") } } diff --git a/src/Pure/Thy/bibtex.scala b/src/Pure/Thy/bibtex.scala --- a/src/Pure/Thy/bibtex.scala +++ b/src/Pure/Thy/bibtex.scala @@ -1,688 +1,688 @@ /* Title: Pure/Thy/bibtex.scala Author: Makarius BibTeX support. */ package isabelle import java.io.{File => JFile} import scala.collection.mutable import scala.util.parsing.combinator.RegexParsers import scala.util.parsing.input.Reader object Bibtex { /** file format **/ def is_bibtex(name: String): Boolean = name.endsWith(".bib") class File_Format extends isabelle.File_Format { val format_name: String = "bibtex" val file_ext: String = "bib" override def theory_suffix: String = "bibtex_file" override def theory_content(name: String): String = """theory "bib" imports Pure begin bibtex_file """ + Outer_Syntax.quote_string(name) + """ end""" override def html_document(snapshot: Document.Snapshot): Option[Presentation.HTML_Document] = { val name = snapshot.node_name if (detect(name.node)) { val title = "Bibliography " + quote(snapshot.node_name.path.file_name) val content = Isabelle_System.with_tmp_file("bib", "bib") { bib => File.write(bib, snapshot.node.source) Bibtex.html_output(List(bib), style = "unsort", title = title) } Some(Presentation.HTML_Document(title, content)) } else None } } /** bibtex errors **/ def bibtex_errors(dir: Path, root_name: String): List[String] = { val log_path = dir + Path.explode(root_name).ext("blg") if (log_path.is_file) { val Error1 = """^(I couldn't open database file .+)$""".r val Error2 = """^(I found no .+)$""".r val Error3 = """^(.+)---line (\d+) of file (.+)""".r Line.logical_lines(File.read(log_path)).flatMap(line => line match { case Error1(msg) => Some("Bibtex error: " + msg) case Error2(msg) => Some("Bibtex error: " + msg) case Error3(msg, Value.Int(l), file) => val path = File.standard_path(file) if (Path.is_wellformed(path)) { val pos = Position.Line_File(l, (dir + Path.explode(path)).canonical.implode) Some("Bibtex error" + Position.here(pos) + ":\n " + msg) } else None case _ => None }) } else Nil } /** check database **/ def check_database(database: String): (List[(String, Position.T)], List[(String, Position.T)]) = { val chunks = parse(Line.normalize(database)) var chunk_pos = Map.empty[String, Position.T] val tokens = new mutable.ListBuffer[(Token, Position.T)] var line = 1 var offset = 1 def make_pos(length: Int): Position.T = Position.Offset(offset) ::: Position.End_Offset(offset + length) ::: Position.Line(line) def advance_pos(tok: Token): Unit = { for (s <- Symbol.iterator(tok.source)) { if (Symbol.is_newline(s)) line += 1 offset += 1 } } def get_line_pos(l: Int): Position.T = if (0 < l && l <= tokens.length) tokens(l - 1)._2 else Position.none for (chunk <- chunks) { val name = chunk.name if (name != "" && !chunk_pos.isDefinedAt(name)) { chunk_pos += (name -> make_pos(chunk.heading_length)) } for (tok <- chunk.tokens) { tokens += (tok.copy(source = tok.source.replace("\n", " ")) -> make_pos(tok.source.length)) advance_pos(tok) } } Isabelle_System.with_tmp_dir("bibtex") { tmp_dir => File.write(tmp_dir + Path.explode("root.bib"), tokens.iterator.map(p => p._1.source).mkString("", "\n", "\n")) File.write(tmp_dir + Path.explode("root.aux"), "\\bibstyle{plain}\n\\bibdata{root}\n\\citation{*}") Isabelle_System.bash("\"$ISABELLE_BIBTEX\" root", cwd = tmp_dir.file) val Error = """^(.*)---line (\d+) of file root.bib$""".r val Warning = """^Warning--(.+)$""".r val Warning_Line = """--line (\d+) of file root.bib$""".r val Warning_in_Chunk = """^Warning--(.+) in (.+)$""".r val log_file = tmp_dir + Path.explode("root.blg") val lines = if (log_file.is_file) Line.logical_lines(File.read(log_file)) else Nil val (errors, warnings) = if (lines.isEmpty) (Nil, Nil) else { lines.zip(lines.tail ::: List("")).flatMap( { case (Error(msg, Value.Int(l)), _) => Some((true, (msg, get_line_pos(l)))) case (Warning_in_Chunk(msg, name), _) if chunk_pos.isDefinedAt(name) => Some((false, (Word.capitalize(msg + " in entry " + quote(name)), chunk_pos(name)))) case (Warning(msg), Warning_Line(Value.Int(l))) => Some((false, (Word.capitalize(msg), get_line_pos(l)))) case (Warning(msg), _) => Some((false, (Word.capitalize(msg), Position.none))) case _ => None } ).partition(_._1) } (errors.map(_._2), warnings.map(_._2)) } } object Check_Database extends Scala.Fun_String("bibtex_check_database") { val here = Scala_Project.here def apply(database: String): String = { import XML.Encode._ YXML.string_of_body(pair(list(pair(string, properties)), list(pair(string, properties)))( check_database(database))) } } /** document model **/ /* entries */ def entries(text: String): List[Text.Info[String]] = { val result = new mutable.ListBuffer[Text.Info[String]] var offset = 0 for (chunk <- Bibtex.parse(text)) { val end_offset = offset + chunk.source.length if (chunk.name != "" && !chunk.is_command) result += Text.Info(Text.Range(offset, end_offset), chunk.name) offset = end_offset } result.toList } def entries_iterator[A, B <: Document.Model]( models: Map[A, B] ): Iterator[Text.Info[(String, B)]] = { for { (_, model) <- models.iterator info <- model.bibtex_entries.iterator } yield info.map((_, model)) } /* completion */ def completion[A, B <: Document.Model]( history: Completion.History, rendering: Rendering, caret: Text.Offset, models: Map[A, B] ): Option[Completion.Result] = { for { Text.Info(r, name) <- rendering.citations(rendering.before_caret_range(caret)).headOption name1 <- Completion.clean_name(name) original <- rendering.get_text(r) original1 <- Completion.clean_name(Library.perhaps_unquote(original)) entries = (for { Text.Info(_, (entry, _)) <- entries_iterator(models) if entry.toLowerCase.containsSlice(name1.toLowerCase) && entry != original1 } yield entry).toList if entries.nonEmpty items = entries.sorted.map({ case entry => val full_name = Long_Name.qualify(Markup.CITATION, entry) val description = List(entry, "(BibTeX entry)") val replacement = quote(entry) - Completion.Item(r, original, full_name, description, replacement, 0, false) + Completion.Item(r, original, full_name, description, replacement, 0, false) }).sorted(history.ordering).take(rendering.options.int("completion_limit")) } yield Completion.Result(r, original, false, items) } /** content **/ private val months = List( "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec") def is_month(s: String): Boolean = months.contains(s.toLowerCase) private val commands = List("preamble", "string") def is_command(s: String): Boolean = commands.contains(s.toLowerCase) sealed case class Entry( kind: String, required: List[String], optional_crossref: List[String], optional_other: List[String] ) { val optional_standard: List[String] = List("url", "doi", "ee") def is_required(s: String): Boolean = required.contains(s.toLowerCase) def is_optional(s: String): Boolean = optional_crossref.contains(s.toLowerCase) || optional_other.contains(s.toLowerCase) || optional_standard.contains(s.toLowerCase) def fields: List[String] = required ::: optional_crossref ::: optional_other ::: optional_standard def template: String = "@" + kind + "{,\n" + fields.map(x => " " + x + " = {},\n").mkString + "}\n" } val known_entries: List[Entry] = List( Entry("Article", List("author", "title"), List("journal", "year"), List("volume", "number", "pages", "month", "note")), Entry("InProceedings", List("author", "title"), List("booktitle", "year"), List("editor", "volume", "number", "series", "pages", "month", "address", "organization", "publisher", "note")), Entry("InCollection", List("author", "title", "booktitle"), List("publisher", "year"), List("editor", "volume", "number", "series", "type", "chapter", "pages", "edition", "month", "address", "note")), Entry("InBook", List("author", "editor", "title", "chapter"), List("publisher", "year"), List("volume", "number", "series", "type", "address", "edition", "month", "pages", "note")), Entry("Proceedings", List("title", "year"), List(), List("booktitle", "editor", "volume", "number", "series", "address", "month", "organization", "publisher", "note")), Entry("Book", List("author", "editor", "title"), List("publisher", "year"), List("volume", "number", "series", "address", "edition", "month", "note")), Entry("Booklet", List("title"), List(), List("author", "howpublished", "address", "month", "year", "note")), Entry("PhdThesis", List("author", "title", "school", "year"), List(), List("type", "address", "month", "note")), Entry("MastersThesis", List("author", "title", "school", "year"), List(), List("type", "address", "month", "note")), Entry("TechReport", List("author", "title", "institution", "year"), List(), List("type", "number", "address", "month", "note")), Entry("Manual", List("title"), List(), List("author", "organization", "address", "edition", "month", "year", "note")), Entry("Unpublished", List("author", "title", "note"), List(), List("month", "year")), Entry("Misc", List(), List(), List("author", "title", "howpublished", "month", "year", "note"))) def get_entry(kind: String): Option[Entry] = known_entries.find(entry => entry.kind.toLowerCase == kind.toLowerCase) def is_entry(kind: String): Boolean = get_entry(kind).isDefined /** tokens and chunks **/ object Token { object Kind extends Enumeration { val COMMAND = Value("command") val ENTRY = Value("entry") val KEYWORD = Value("keyword") val NAT = Value("natural number") val STRING = Value("string") val NAME = Value("name") val IDENT = Value("identifier") val SPACE = Value("white space") val COMMENT = Value("ignored text") val ERROR = Value("bad input") } } sealed case class Token(kind: Token.Kind.Value, source: String) { def is_kind: Boolean = kind == Token.Kind.COMMAND || kind == Token.Kind.ENTRY || kind == Token.Kind.IDENT def is_name: Boolean = kind == Token.Kind.NAME || kind == Token.Kind.IDENT def is_ignored: Boolean = kind == Token.Kind.SPACE || kind == Token.Kind.COMMENT def is_malformed: Boolean = kind == Token.Kind.ERROR def is_open: Boolean = kind == Token.Kind.KEYWORD && (source == "{" || source == "(") } case class Chunk(kind: String, tokens: List[Token]) { val source = tokens.map(_.source).mkString private val content: Option[List[Token]] = tokens match { case Token(Token.Kind.KEYWORD, "@") :: body if body.nonEmpty => (body.init.filterNot(_.is_ignored), body.last) match { case (tok :: Token(Token.Kind.KEYWORD, "{") :: toks, Token(Token.Kind.KEYWORD, "}")) if tok.is_kind => Some(toks) case (tok :: Token(Token.Kind.KEYWORD, "(") :: toks, Token(Token.Kind.KEYWORD, ")")) if tok.is_kind => Some(toks) case _ => None } case _ => None } def name: String = content match { case Some(tok :: _) if tok.is_name => tok.source case _ => "" } def heading_length: Int = if (name == "") 1 else { tokens.takeWhile(tok => !tok.is_open).foldLeft(0) { case (n, tok) => n + tok.source.length } } def is_ignored: Boolean = kind == "" && tokens.forall(_.is_ignored) def is_malformed: Boolean = kind == "" || tokens.exists(_.is_malformed) def is_command: Boolean = Bibtex.is_command(kind) && name != "" && content.isDefined def is_entry: Boolean = Bibtex.is_entry(kind) && name != "" && content.isDefined } /** parsing **/ // context of partial line-oriented scans abstract class Line_Context case object Ignored extends Line_Context case object At extends Line_Context case class Item_Start(kind: String) extends Line_Context case class Item_Open(kind: String, end: String) extends Line_Context case class Item(kind: String, end: String, delim: Delimited) extends Line_Context case class Delimited(quoted: Boolean, depth: Int) val Closed = Delimited(false, 0) private def token(kind: Token.Kind.Value)(source: String): Token = Token(kind, source) private def keyword(source: String): Token = Token(Token.Kind.KEYWORD, source) // See also https://ctan.org/tex-archive/biblio/bibtex/base/bibtex.web // module @. object Parsers extends RegexParsers { /* white space and comments */ override val whiteSpace = "".r private val space = """[ \t\n\r]+""".r ^^ token(Token.Kind.SPACE) private val spaces = rep(space) /* ignored text */ private val ignored: Parser[Chunk] = rep1("""(?i)([^@]+|@[ \t]*comment)""".r) ^^ { case ss => Chunk("", List(Token(Token.Kind.COMMENT, ss.mkString))) } private def ignored_line: Parser[(Chunk, Line_Context)] = ignored ^^ { case a => (a, Ignored) } /* delimited string: outermost "..." or {...} and body with balanced {...} */ // see also bibtex.web: scan_a_field_token_and_eat_white, scan_balanced_braces private def delimited_depth(delim: Delimited): Parser[(String, Delimited)] = new Parser[(String, Delimited)] { require(if (delim.quoted) delim.depth > 0 else delim.depth >= 0, "bad delimiter depth") def apply(in: Input) = { val start = in.offset val end = in.source.length var i = start var q = delim.quoted var d = delim.depth var finished = false while (!finished && i < end) { val c = in.source.charAt(i) if (c == '"' && d == 0) { i += 1; d = 1; q = true } else if (c == '"' && d == 1 && q) { i += 1; d = 0; q = false; finished = true } else if (c == '{') { i += 1; d += 1 } else if (c == '}') { if (d == 1 && !q || d > 1) { i += 1; d -= 1; if (d == 0) finished = true } else {i = start; finished = true } } else if (d > 0) i += 1 else finished = true } if (i == start) Failure("bad input", in) else { val s = in.source.subSequence(start, i).toString Success((s, Delimited(q, d)), in.drop(i - start)) } } }.named("delimited_depth") private def delimited: Parser[Token] = delimited_depth(Closed) ^? { case (s, delim) if delim == Closed => Token(Token.Kind.STRING, s) } private def recover_delimited: Parser[Token] = """["{][^@]*""".r ^^ token(Token.Kind.ERROR) def delimited_line(ctxt: Item): Parser[(Chunk, Line_Context)] = delimited_depth(ctxt.delim) ^^ { case (s, delim1) => (Chunk(ctxt.kind, List(Token(Token.Kind.STRING, s))), ctxt.copy(delim = delim1)) } | recover_delimited ^^ { case a => (Chunk(ctxt.kind, List(a)), Ignored) } /* other tokens */ private val at = "@" ^^ keyword private val nat = "[0-9]+".r ^^ token(Token.Kind.NAT) private val name = """[\x21-\x7f&&[^"#%'(),={}]]+""".r ^^ token(Token.Kind.NAME) private val identifier = """[\x21-\x7f&&[^"#%'(),={}0-9]][\x21-\x7f&&[^"#%'(),={}]]*""".r private val ident = identifier ^^ token(Token.Kind.IDENT) val other_token = "[=#,]".r ^^ keyword | (nat | (ident | space)) /* body */ private val body = delimited | (recover_delimited | other_token) private def body_line(ctxt: Item) = if (ctxt.delim.depth > 0) delimited_line(ctxt) else delimited_line(ctxt) | other_token ^^ { case a => (Chunk(ctxt.kind, List(a)), ctxt) } | ctxt.end ^^ { case a => (Chunk(ctxt.kind, List(keyword(a))), Ignored) } /* items: command or entry */ private val item_kind = identifier ^^ { case a => val kind = if (is_command(a)) Token.Kind.COMMAND else if (is_entry(a)) Token.Kind.ENTRY else Token.Kind.IDENT Token(kind, a) } private val item_begin = "{" ^^ { case a => ("}", keyword(a)) } | "(" ^^ { case a => (")", keyword(a)) } private def item_name(kind: String) = kind.toLowerCase match { case "preamble" => failure("") case "string" => identifier ^^ token(Token.Kind.NAME) case _ => name } private val item_start = at ~ spaces ~ item_kind ~ spaces ^^ { case a ~ b ~ c ~ d => (c.source, List(a) ::: b ::: List(c) ::: d) } private val item: Parser[Chunk] = (item_start ~ item_begin ~ spaces) into { case (kind, a) ~ ((end, b)) ~ c => opt(item_name(kind)) ~ rep(body) ~ opt(end ^^ keyword) ^^ { case d ~ e ~ f => Chunk(kind, a ::: List(b) ::: c ::: d.toList ::: e ::: f.toList) } } private val recover_item: Parser[Chunk] = at ~ "[^@]*".r ^^ { case a ~ b => Chunk("", List(a, Token(Token.Kind.ERROR, b))) } /* chunks */ val chunk: Parser[Chunk] = ignored | (item | recover_item) def chunk_line(ctxt: Line_Context): Parser[(Chunk, Line_Context)] = { ctxt match { case Ignored => ignored_line | at ^^ { case a => (Chunk("", List(a)), At) } case At => space ^^ { case a => (Chunk("", List(a)), ctxt) } | item_kind ^^ { case a => (Chunk(a.source, List(a)), Item_Start(a.source)) } | recover_item ^^ { case a => (a, Ignored) } | ignored_line case Item_Start(kind) => space ^^ { case a => (Chunk(kind, List(a)), ctxt) } | item_begin ^^ { case (end, a) => (Chunk(kind, List(a)), Item_Open(kind, end)) } | recover_item ^^ { case a => (a, Ignored) } | ignored_line case Item_Open(kind, end) => space ^^ { case a => (Chunk(kind, List(a)), ctxt) } | item_name(kind) ^^ { case a => (Chunk(kind, List(a)), Item(kind, end, Closed)) } | body_line(Item(kind, end, Closed)) | ignored_line case item_ctxt: Item => body_line(item_ctxt) | ignored_line case _ => failure("") } } } /* parse */ def parse(input: CharSequence): List[Chunk] = Parsers.parseAll(Parsers.rep(Parsers.chunk), Scan.char_reader(input)) match { case Parsers.Success(result, _) => result case _ => error("Unexpected failure to parse input:\n" + input.toString) } def parse_line(input: CharSequence, context: Line_Context): (List[Chunk], Line_Context) = { var in: Reader[Char] = Scan.char_reader(input) val chunks = new mutable.ListBuffer[Chunk] var ctxt = context while (!in.atEnd) { val result = Parsers.parse(Parsers.chunk_line(ctxt), in) (result : @unchecked) match { case Parsers.Success((x, c), rest) => chunks += x; ctxt = c; in = rest case Parsers.NoSuccess(_, rest) => error("Unepected failure to parse input:\n" + rest.source.toString) } } (chunks.toList, ctxt) } /** HTML output **/ private val output_styles = List( "" -> "html-n", "plain" -> "html-n", "alpha" -> "html-a", "named" -> "html-n", "paragraph" -> "html-n", "unsort" -> "html-u", "unsortlist" -> "html-u") def html_output(bib: List[Path], title: String = "Bibliography", body: Boolean = false, citations: List[String] = List("*"), style: String = "", chronological: Boolean = false ): String = { Isabelle_System.with_tmp_dir("bibtex") { tmp_dir => /* database files */ val bib_files = bib.map(_.drop_ext) val bib_names = { val names0 = bib_files.map(_.file_name) if (Library.duplicates(names0).isEmpty) names0 else names0.zipWithIndex.map({ case (name, i) => (i + 1).toString + "-" + name }) } for ((a, b) <- bib_files zip bib_names) { Isabelle_System.copy_file(a.ext("bib"), tmp_dir + Path.basic(b).ext("bib")) } /* style file */ val bst = output_styles.toMap.get(style) match { case Some(base) => base + (if (chronological) "c" else "") + ".bst" case None => error("Bad style for bibtex HTML output: " + quote(style) + "\n(expected: " + commas_quote(output_styles.map(_._1)) + ")") } Isabelle_System.copy_file(Path.explode("$BIB2XHTML_HOME/bst") + Path.explode(bst), tmp_dir) /* result */ val in_file = Path.explode("bib.aux") val out_file = Path.explode("bib.html") File.write(tmp_dir + in_file, bib_names.mkString("\\bibdata{", ",", "}\n") + citations.map(cite => "\\citation{" + cite + "}\n").mkString) Isabelle_System.bash( "\"$BIB2XHTML_HOME/main/bib2xhtml.pl\" -B \"$ISABELLE_BIBTEX\"" + " -u -s " + Bash.string(proper_string(style) getOrElse "empty") + (if (chronological) " -c" else "") + (if (title != "") " -h " + Bash.string(title) + " " else "") + " " + File.bash_path(in_file) + " " + File.bash_path(out_file), cwd = tmp_dir.file).check val html = File.read(tmp_dir + out_file) if (body) { cat_lines( split_lines(html). dropWhile(line => !line.startsWith("

", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\", "\\