diff --git a/src/Tools/jEdit/src/jedit_bibtex.scala b/src/Tools/jEdit/src/jedit_bibtex.scala --- a/src/Tools/jEdit/src/jedit_bibtex.scala +++ b/src/Tools/jEdit/src/jedit_bibtex.scala @@ -1,197 +1,199 @@ /* Title: Tools/jEdit/src/jedit_bibtex.scala Author: Makarius BibTeX support in Isabelle/jEdit. */ package isabelle.jedit import isabelle._ import scala.collection.mutable import java.awt.event.{ActionListener, ActionEvent} import javax.swing.text.Segment import javax.swing.tree.DefaultMutableTreeNode import javax.swing.{JMenu, JMenuItem} import org.gjt.sp.jedit.Buffer import org.gjt.sp.jedit.textarea.{JEditTextArea, TextArea} -import org.gjt.sp.jedit.syntax.{Token => JEditToken, TokenMarker, TokenHandler, ParserRuleSet} +import org.gjt.sp.jedit.syntax.{Token => JEditToken, TokenMarker, TokenHandler} import sidekick.{SideKickParser, SideKickParsedData} object JEdit_Bibtex { /** context menu **/ def context_menu(text_area0: JEditTextArea): List[JMenuItem] = { text_area0 match { case text_area: TextArea => text_area.getBuffer match { case buffer: Buffer if (Bibtex.is_bibtex(JEdit_Lib.buffer_name(buffer)) && buffer.isEditable) => val menu = new JMenu("BibTeX entries") for (entry <- Bibtex.known_entries) { val item = new JMenuItem(entry.kind) item.addActionListener(new ActionListener { def actionPerformed(evt: ActionEvent): Unit = Isabelle.insert_line_padding(text_area, entry.template) }) menu.add(item) } List(menu) case _ => Nil } case _ => Nil } } /** token markup **/ /* token style */ private def token_style(context: String, token: Bibtex.Token): Byte = token.kind match { case Bibtex.Token.Kind.COMMAND => JEditToken.KEYWORD2 case Bibtex.Token.Kind.ENTRY => JEditToken.KEYWORD1 case Bibtex.Token.Kind.KEYWORD => JEditToken.OPERATOR case Bibtex.Token.Kind.NAT => JEditToken.LITERAL2 case Bibtex.Token.Kind.STRING => JEditToken.LITERAL1 case Bibtex.Token.Kind.NAME => JEditToken.LABEL case Bibtex.Token.Kind.IDENT => if (Bibtex.is_month(token.source)) JEditToken.LITERAL3 else Bibtex.get_entry(context) match { case Some(entry) if entry.is_required(token.source) => JEditToken.KEYWORD3 case Some(entry) if entry.is_optional(token.source) => JEditToken.KEYWORD4 case _ => JEditToken.DIGIT } case Bibtex.Token.Kind.SPACE => JEditToken.NULL case Bibtex.Token.Kind.COMMENT => JEditToken.COMMENT1 case Bibtex.Token.Kind.ERROR => JEditToken.INVALID } /* line context */ - private val context_rules = new ParserRuleSet("bibtex", "MAIN") + private val mode_rule_set = Token_Markup.mode_rule_set("bibtex") private class Line_Context(val context: Option[Bibtex.Line_Context]) - extends TokenMarker.LineContext(context_rules, null) + extends TokenMarker.LineContext(mode_rule_set, null) { override def hashCode: Int = context.hashCode override def equals(that: Any): Boolean = that match { case other: Line_Context => context == other.context case _ => false } } /* token marker */ class Token_Marker extends TokenMarker { + addRuleSet(mode_rule_set) + override def markTokens(context: TokenMarker.LineContext, handler: TokenHandler, raw_line: Segment): TokenMarker.LineContext = { val line_ctxt = context match { case c: Line_Context => c.context case _ => Some(Bibtex.Ignored) } val line = if (raw_line == null) new Segment else raw_line def no_markup = { val styled_token = (JEditToken.NULL, line.subSequence(0, line.count).toString) (List(styled_token), new Line_Context(None)) } val context1 = { val (styled_tokens, context1) = line_ctxt match { case Some(ctxt) => try { val (chunks, ctxt1) = Bibtex.parse_line(line, ctxt) val styled_tokens = for { chunk <- chunks; tok <- chunk.tokens } yield (token_style(chunk.kind, tok), tok.source) (styled_tokens, new Line_Context(Some(ctxt1))) } catch { case ERROR(msg) => Output.warning(msg); no_markup } case None => no_markup } var offset = 0 for ((style, token) <- styled_tokens) { val length = token.length val end_offset = offset + length if ((offset until end_offset).exists(i => line.charAt(i) == '\t')) { for (i <- offset until end_offset) handler.handleToken(line, style, i, 1, context1) } else handler.handleToken(line, style, offset, length, context1) offset += length } handler.handleToken(line, JEditToken.END, line.count, 0, context1) context1 } val context2 = context1.intern handler.setLineContext(context2) context2 } } /** Sidekick parser **/ class Sidekick_Parser extends SideKickParser("bibtex") { override def supportsCompletion = false private class Asset(label: String, label_html: String, range: Text.Range, source: String) extends Isabelle_Sidekick.Asset(label, range) { override def getShortString: String = label_html override def getLongString: String = source } def parse(buffer: Buffer, error_source: errorlist.DefaultErrorSource): SideKickParsedData = { val data = Isabelle_Sidekick.root_data(buffer) try { var offset = 0 for (chunk <- Bibtex.parse(JEdit_Lib.buffer_text(buffer))) { val kind = chunk.kind val name = chunk.name val source = chunk.source if (kind != "") { val label = kind + (if (name == "") "" else " " + name) val label_html = "" + HTML.output(kind) + "" + (if (name == "") "" else " " + HTML.output(name)) + "" val range = Text.Range(offset, offset + source.length) val asset = new Asset(label, label_html, range, source) data.root.add(new DefaultMutableTreeNode(asset)) } offset += source.length } data } catch { case ERROR(msg) => Output.warning(msg); null } } } } diff --git a/src/Tools/jEdit/src/token_markup.scala b/src/Tools/jEdit/src/token_markup.scala --- a/src/Tools/jEdit/src/token_markup.scala +++ b/src/Tools/jEdit/src/token_markup.scala @@ -1,312 +1,317 @@ /* Title: Tools/jEdit/src/token_markup.scala Author: Makarius Outer syntax token markup. */ package isabelle.jedit import isabelle._ import javax.swing.text.Segment import org.gjt.sp.jedit.{Mode, Buffer} import org.gjt.sp.jedit.syntax.{Token => JEditToken, TokenMarker, TokenHandler, DummyTokenHandler, ParserRuleSet, ModeProvider, XModeHandler} import org.gjt.sp.jedit.indent.IndentRule import org.gjt.sp.jedit.buffer.JEditBuffer object Token_Markup { /* line context */ + def mode_rule_set(mode: String): ParserRuleSet = + new ParserRuleSet(mode, "MAIN") + object Line_Context { def init(mode: String): Line_Context = new Line_Context(mode, Some(Scan.Finished), Line_Structure.init) def before(buffer: JEditBuffer, line: Int): Line_Context = if (line == 0) init(JEdit_Lib.buffer_mode(buffer)) else after(buffer, line - 1) def after(buffer: JEditBuffer, line: Int): Line_Context = { val line_mgr = JEdit_Lib.buffer_line_manager(buffer) def context = line_mgr.getLineContext(line) match { case c: Line_Context => Some(c) case _ => None } context getOrElse { buffer.markTokens(line, DummyTokenHandler.INSTANCE) context getOrElse init(JEdit_Lib.buffer_mode(buffer)) } } } class Line_Context( val mode: String, val context: Option[Scan.Line_Context], val structure: Line_Structure) - extends TokenMarker.LineContext(new ParserRuleSet(mode, "MAIN"), null) + extends TokenMarker.LineContext(mode_rule_set(mode), null) { def get_context: Scan.Line_Context = context.getOrElse(Scan.Finished) override def hashCode: Int = (mode, context, structure).hashCode override def equals(that: Any): Boolean = that match { case other: Line_Context => mode == other.mode && context == other.context && structure == other.structure case _ => false } } /* tokens from line (inclusive) */ private def try_line_tokens(syntax: Outer_Syntax, buffer: JEditBuffer, line: Int) : Option[List[Token]] = { val line_context = Line_Context.before(buffer, line) for { ctxt <- line_context.context text <- JEdit_Lib.get_text(buffer, JEdit_Lib.line_range(buffer, line)) } yield Token.explode_line(syntax.keywords, text, ctxt)._1 } def line_token_iterator( syntax: Outer_Syntax, buffer: JEditBuffer, start_line: Int, end_line: Int): Iterator[Text.Info[Token]] = for { line <- Range(start_line max 0, end_line min buffer.getLineCount).iterator tokens <- try_line_tokens(syntax, buffer, line).iterator starts = tokens.iterator.scanLeft(buffer.getLineStartOffset(line))( (i, tok) => i + tok.source.length) (i, tok) <- starts zip tokens.iterator } yield Text.Info(Text.Range(i, i + tok.source.length), tok) def line_token_reverse_iterator( syntax: Outer_Syntax, buffer: JEditBuffer, start_line: Int, end_line: Int): Iterator[Text.Info[Token]] = for { line <- Range(start_line min (buffer.getLineCount - 1), end_line max -1, -1).iterator tokens <- try_line_tokens(syntax, buffer, line).iterator stops = tokens.reverseIterator.scanLeft(buffer.getLineEndOffset(line) min buffer.getLength)( (i, tok) => i - tok.source.length) (i, tok) <- stops zip tokens.reverseIterator } yield Text.Info(Text.Range(i - tok.source.length, i), tok) /* tokens from offset (inclusive) */ def token_iterator(syntax: Outer_Syntax, buffer: JEditBuffer, offset: Text.Offset): Iterator[Text.Info[Token]] = if (JEdit_Lib.buffer_range(buffer).contains(offset)) line_token_iterator(syntax, buffer, buffer.getLineOfOffset(offset), buffer.getLineCount). dropWhile(info => !info.range.contains(offset)) else Iterator.empty def token_reverse_iterator(syntax: Outer_Syntax, buffer: JEditBuffer, offset: Text.Offset): Iterator[Text.Info[Token]] = if (JEdit_Lib.buffer_range(buffer).contains(offset)) line_token_reverse_iterator(syntax, buffer, buffer.getLineOfOffset(offset), -1). dropWhile(info => !info.range.contains(offset)) else Iterator.empty /* command spans */ def command_span(syntax: Outer_Syntax, buffer: JEditBuffer, offset: Text.Offset) : Option[Text.Info[Command_Span.Span]] = { val keywords = syntax.keywords def maybe_command_start(i: Text.Offset): Option[Text.Info[Token]] = token_reverse_iterator(syntax, buffer, i). find(info => keywords.is_before_command(info.info) || info.info.is_command) def maybe_command_stop(i: Text.Offset): Option[Text.Info[Token]] = token_iterator(syntax, buffer, i). find(info => keywords.is_before_command(info.info) || info.info.is_command) if (JEdit_Lib.buffer_range(buffer).contains(offset)) { val start_info = { val info1 = maybe_command_start(offset) info1 match { case Some(Text.Info(range1, tok1)) if tok1.is_command => val info2 = maybe_command_start(range1.start - 1) info2 match { case Some(Text.Info(_, tok2)) if keywords.is_before_command(tok2) => info2 case _ => info1 } case _ => info1 } } val (start_before_command, start, start_next) = start_info match { case Some(Text.Info(range, tok)) => (keywords.is_before_command(tok), range.start, range.stop) case None => (false, 0, 0) } val stop_info = { val info1 = maybe_command_stop(start_next) info1 match { case Some(Text.Info(range1, tok1)) if tok1.is_command && start_before_command => maybe_command_stop(range1.stop) case _ => info1 } } val stop = stop_info match { case Some(Text.Info(range, _)) => range.start case None => buffer.getLength } val text = JEdit_Lib.get_text(buffer, Text.Range(start, stop)).getOrElse("") val spans = syntax.parse_spans(text) (spans.iterator.scanLeft(start)(_ + _.length) zip spans.iterator). map({ case (i, span) => Text.Info(Text.Range(i, i + span.length), span) }). find(_.range.contains(offset)) } else None } private def _command_span_iterator( syntax: Outer_Syntax, buffer: JEditBuffer, offset: Text.Offset, next_offset: Text.Range => Text.Offset): Iterator[Text.Info[Command_Span.Span]] = new Iterator[Text.Info[Command_Span.Span]] { private var next_span = command_span(syntax, buffer, offset) def hasNext: Boolean = next_span.isDefined def next(): Text.Info[Command_Span.Span] = { val span = next_span.getOrElse(Iterator.empty.next()) next_span = command_span(syntax, buffer, next_offset(span.range)) span } } def command_span_iterator(syntax: Outer_Syntax, buffer: JEditBuffer, offset: Text.Offset) : Iterator[Text.Info[Command_Span.Span]] = _command_span_iterator(syntax, buffer, offset max 0, range => range.stop) def command_span_reverse_iterator(syntax: Outer_Syntax, buffer: JEditBuffer, offset: Text.Offset) : Iterator[Text.Info[Command_Span.Span]] = _command_span_iterator(syntax, buffer, (offset min buffer.getLength) - 1, range => range.start - 1) /* token marker */ class Marker( protected val mode: String, protected val opt_buffer: Option[Buffer]) extends TokenMarker { + addRuleSet(mode_rule_set(mode)) + override def hashCode: Int = (mode, opt_buffer).hashCode override def equals(that: Any): Boolean = that match { case other: Marker => mode == other.mode && opt_buffer == other.opt_buffer case _ => false } override def toString: String = opt_buffer match { case None => "Marker(" + mode + ")" case Some(buffer) => "Marker(" + mode + "," + JEdit_Lib.buffer_name(buffer) + ")" } override def markTokens(context: TokenMarker.LineContext, handler: TokenHandler, raw_line: Segment): TokenMarker.LineContext = { val line = if (raw_line == null) new Segment else raw_line val line_context = context match { case c: Line_Context => c case _ => Line_Context.init(mode) } val structure = line_context.structure val context1 = { val opt_syntax = opt_buffer match { case Some(buffer) => Isabelle.buffer_syntax(buffer) case None => Isabelle.mode_syntax(mode) } val (styled_tokens, context1) = (line_context.context, opt_syntax) match { case (Some(ctxt), _) if mode == "isabelle-ml" || mode == "sml" => val (tokens, ctxt1) = ML_Lex.tokenize_line(mode == "sml", line, ctxt) val styled_tokens = tokens.map(tok => (JEdit_Rendering.ml_token_markup(tok), tok.source)) (styled_tokens, new Line_Context(line_context.mode, Some(ctxt1), structure)) case (Some(ctxt), Some(syntax)) if syntax.has_tokens => val (tokens, ctxt1) = Token.explode_line(syntax.keywords, line, ctxt) val structure1 = structure.update(syntax.keywords, tokens) val styled_tokens = tokens.map(tok => (JEdit_Rendering.token_markup(syntax, tok), tok.source)) (styled_tokens, new Line_Context(line_context.mode, Some(ctxt1), structure1)) case _ => val styled_token = (JEditToken.NULL, line.subSequence(0, line.count).toString) (List(styled_token), new Line_Context(line_context.mode, None, structure)) } val extended = Syntax_Style.extended(line) def special(i: Int): Boolean = extended.isDefinedAt(i) || line.charAt(i) == '\t' var offset = 0 for ((style, token) <- styled_tokens) { val length = token.length if ((offset until (offset + length)).exists(special)) { for ((c, i) <- Codepoint.iterator_offset(token)) { val style1 = extended.get(offset + i) match { case None => style case Some(ext) => ext(style) } handler.handleToken(line, style1, offset + i, Character.charCount(c), context1) } } else handler.handleToken(line, style, offset, length, context1) offset += length } handler.handleToken(line, JEditToken.END, line.count, 0, context1) context1 } val context2 = context1.intern handler.setLineContext(context2) context2 } } /* mode provider */ class Mode_Provider(orig_provider: ModeProvider) extends ModeProvider { for (mode <- orig_provider.getModes) addMode(mode) override def loadMode(mode: Mode, xmh: XModeHandler): Unit = { super.loadMode(mode, xmh) Isabelle.mode_token_marker(mode.getName).foreach(mode.setTokenMarker _) Isabelle.indent_rule(mode.getName).foreach(indent_rule => Untyped.set[java.util.List[IndentRule]]( mode, "indentRules", java.util.List.of(indent_rule))) } } }