CodeMirror/manuskript
1
0
Fork 0
mirror of https://github.com/olivierkes/manuskript.git synced 2024-06-14 17:04:33 +12:00
Code Issues Projects Releases Wiki Activity

Checkpoint: working highlighter. Optimization welcome.

Browse source
This commit is contained in:
Olivier Keshavjee 2017-11-22 21:03:47 +01:00
parent 9be2edeee7
commit da62b823c7
7 changed files with 237 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
manuskript/ui/highlighters/MMDHighlighter.py
View file

@ -35,13 +35,6 @@ class MMDHighlighter(BasicHighlighter):
for key in self.MARKDOWN_REGEX: for key in self.MARKDOWN_REGEX:
self.rules[key] = re.compile(self.MARKDOWN_REGEX[key]) self.rules[key] = re.compile(self.MARKDOWN_REGEX[key])
def highlightBlock(self, text):
BasicHighlighter.highlightBlockBefore(self, text)
self.doHighlightBlock(text)
BasicHighlighter.highlightBlockAfter(self, text)
def doHighlightBlock(self, text): def doHighlightBlock(self, text):
""" """
A quick-n-dirty very basic highlighter, that fails in most non-trivial cases. And is ugly. A quick-n-dirty very basic highlighter, that fails in most non-trivial cases. And is ugly.

6
manuskript/ui/highlighters/__init__.py
View file

@ -3,4 +3,10 @@
from manuskript.ui.highlighters.basicHighlighter import BasicHighlighter from manuskript.ui.highlighters.basicHighlighter import BasicHighlighter
from manuskript.ui.highlighters.MMDHighlighter import MMDHighlighter from manuskript.ui.highlighters.MMDHighlighter import MMDHighlighter
# Markdown highlighter
from manuskript.ui.highlighters.markdownEnums import MarkdownState
from manuskript.ui.highlighters.markdownEnums import MarkdownTokenType
from manuskript.ui.highlighters.markdownEnums import BlockquoteStyle
from manuskript.ui.highlighters.markdownTokenizer import MarkdownTokenizer
from manuskript.ui.highlighters.markdownHighlighter import MarkdownHighlighter from manuskript.ui.highlighters.markdownHighlighter import MarkdownHighlighter

42
manuskript/ui/highlighters/basicHighlighter.py
View file

@ -8,6 +8,9 @@ from PyQt5.QtGui import QBrush, QTextCursor, QColor, QFont, QSyntaxHighlighter
from PyQt5.QtGui import QTextBlockFormat, QTextCharFormat from PyQt5.QtGui import QTextBlockFormat, QTextCharFormat
import manuskript.models.references as Ref import manuskript.models.references as Ref
import manuskript.ui.style as S
from manuskript import settings
from manuskript import functions as F
class BasicHighlighter(QSyntaxHighlighter): class BasicHighlighter(QSyntaxHighlighter):
@ -18,6 +21,11 @@ class BasicHighlighter(QSyntaxHighlighter):
self._misspelledColor = Qt.red self._misspelledColor = Qt.red
self._defaultBlockFormat = QTextBlockFormat() self._defaultBlockFormat = QTextBlockFormat()
self._defaultCharFormat = QTextCharFormat() self._defaultCharFormat = QTextCharFormat()
self.defaultTextColor = QColor(S.text)
self.backgroundColor = QColor(S.base)
self.markupColor = QColor(S.textLight)
self.linkColor = QColor(S.link)
self.spellingErrorColor = QColor(Qt.red)
def setDefaultBlockFormat(self, bf): def setDefaultBlockFormat(self, bf):
self._defaultBlockFormat = bf self._defaultBlockFormat = bf
@ -30,17 +38,47 @@ class BasicHighlighter(QSyntaxHighlighter):
def setMisspelledColor(self, color): def setMisspelledColor(self, color):
self._misspelledColor = color self._misspelledColor = color
def updateColorScheme(self, rehighlight=True):
"""
Generates a base set of colors that will take account of user
preferences, and use system style.
"""
# Reading user settings
opt = settings.textEditor
self.defaultTextColor = QColor(opt["fontColor"])
self.backgroundColor = (QColor(opt["background"])
if not opt["backgroundTransparent"]
else QColor(S.window))
self.markupColor = F.mixColors(self.defaultTextColor,
self.backgroundColor,
.3)
self.linkColor = QColor(S.link)
self.spellingErrorColor = QColor(opt["misspelled"])
self._defaultCharFormat.setForeground(QBrush(self.defaultTextColor))
if rehighlight:
self.rehighlight()
def highlightBlock(self, text): def highlightBlock(self, text):
"""Apply syntax highlighting to the given block of text. """Apply syntax highlighting to the given block of text.
""" """
self.highlightBlockBefore(text) self.highlightBlockBefore(text)
self.doHighlightBlock(text)
self.highlightBlockAfter(text) self.highlightBlockAfter(text)
def doHighlightBlock(self, text):
"""
Virtual funtion to subclass.
"""
pass
def highlightBlockBefore(self, text): def highlightBlockBefore(self, text):
"""Highlighting to do before anything else. """Highlighting to do before anything else.
When subclassing BasicHighlighter, you must call highlightBlockBefore When subclassing BasicHighlighter, you must call highlightBlockBefore
before you do any custom highlighting. before you do any custom highlighting. Or implement doHighlightBlock.
""" """
#print(">", self.currentBlock().document().availableUndoSteps()) #print(">", self.currentBlock().document().availableUndoSteps())
@ -58,7 +96,7 @@ class BasicHighlighter(QSyntaxHighlighter):
"""Highlighting to do after everything else. """Highlighting to do after everything else.
When subclassing BasicHighlighter, you must call highlightBlockAfter When subclassing BasicHighlighter, you must call highlightBlockAfter
after your custom highlighting. after your custom highlighting. Or implement doHighlightBlock.
""" """
# References # References

215
manuskript/ui/highlighters/markdownHighlighter.py
View file

@ -12,10 +12,14 @@ from PyQt5.QtGui import (QSyntaxHighlighter, QTextBlock, QColor, QFont,
QTextCharFormat, QBrush, QPalette) QTextCharFormat, QBrush, QPalette)
from PyQt5.QtWidgets import qApp, QStyle from PyQt5.QtWidgets import qApp, QStyle
from manuskript.ui.highlighters.markdownTokenizer import MarkdownTokenizer from manuskript.ui.highlighters import BasicHighlighter
from manuskript.ui.highlighters.markdownEnums import MarkdownState as MS from manuskript.ui.highlighters import MarkdownTokenizer
from manuskript.ui.highlighters.markdownEnums import MarkdownTokenType as MTT from manuskript.ui.highlighters import MarkdownState as MS
from manuskript.ui.highlighters.markdownEnums import BlockquoteStyle as BS from manuskript.ui.highlighters import MarkdownTokenType as MTT
from manuskript.ui.highlighters import BlockquoteStyle as BS
from manuskript.ui import style as S
from manuskript import settings
from manuskript import functions as F
# Un longue ligne. Un longue ligne. Un longue ligne. Un longue ligne.asdasdasda # Un longue ligne. Un longue ligne. Un longue ligne. Un longue ligne.asdasdasda
@ -26,14 +30,14 @@ GW_FADE_ALPHA = 140
#FIXME: Setext heading don't work anymore #FIXME: Setext heading don't work anymore
class MarkdownHighlighter(QSyntaxHighlighter): class MarkdownHighlighter(BasicHighlighter):
highlightBlockAtPosition = pyqtSignal(int) highlightBlockAtPosition = pyqtSignal(int)
headingFound = pyqtSignal(int, str, QTextBlock) headingFound = pyqtSignal(int, str, QTextBlock)
headingRemoved = pyqtSignal(int) headingRemoved = pyqtSignal(int)
def __init__(self, editor): def __init__(self, editor):
QSyntaxHighlighter.__init__(self, editor.document()) BasicHighlighter.__init__(self, editor)
#default values #default values
self.editor = editor self.editor = editor
@ -42,11 +46,6 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.spellCheckEnabled = False self.spellCheckEnabled = False
#self.typingPaused = True #self.typingPaused = True
self.inBlockquote = False self.inBlockquote = False
self.defaultTextColor = QColor(Qt.black)
self.backgroundColor = QColor(Qt.white)
self.markupColor = QColor(Qt.black)
self.linkColor = QColor(Qt.blue)
self.spellingErrorColor = QColor(Qt.red)
self.blockquoteStyle = BS.BlockquoteStyleFancy self.blockquoteStyle = BS.BlockquoteStyleFancy
# Settings # Settings
@ -56,13 +55,6 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.highlightBlockAtPosition.connect(self.onHighlightBlockAtPosition, self.highlightBlockAtPosition.connect(self.onHighlightBlockAtPosition,
Qt.QueuedConnection) Qt.QueuedConnection)
# font = QFont("Monospace", 12, QFont.Normal, False)
font = self.document().defaultFont()
font.setStyleStrategy(QFont.PreferAntialias)
self.defaultFormat = QTextCharFormat()
self.defaultFormat.setFont(font)
self.defaultFormat.setForeground(QBrush(self.defaultTextColor))
self.theme = self.defaultTheme() self.theme = self.defaultTheme()
self.setupHeadingFontSize(True) self.setupHeadingFontSize(True)
@ -72,16 +64,11 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.searchExpressionRegExp = False self.searchExpressionRegExp = False
self.searchExpressionCase = False self.searchExpressionCase = False
self.customRules = [
("(°).*?(°)", {"background": Qt.yellow,
"markupColor":Qt.lightGray}),
]
#f = self.document().defaultFont() #f = self.document().defaultFont()
#f.setFamily("monospace") #f.setFamily("monospace")
#self.document().setDefaultFont(f) #self.document().setDefaultFont(f)
def highlightBlock(self, text): def doHighlightBlock(self, text):
""" """
Note: Never set the QTextBlockFormat for a QTextBlock from within Note: Never set the QTextBlockFormat for a QTextBlock from within
the highlighter. Depending on how the block format is modified, the highlighter. Depending on how the block format is modified,
@ -97,17 +84,8 @@ class MarkdownHighlighter(QSyntaxHighlighter):
formatting to be triggered yet again. formatting to be triggered yet again.
""" """
if self.currentBlock().blockNumber() == 0:
# This is the title
bf = QTextCharFormat()
bf.setFontPointSize(self.editor.font().pointSize() * 2)
bf.setFontWeight(QFont.Bold)
bf.setForeground(Qt.lightGray)
self.setFormat(0, len(text), bf)
return
lastState = self.currentBlockState() lastState = self.currentBlockState()
self.setFormat(0, len(text), self.defaultFormat) self.setFormat(0, len(text), self._defaultCharFormat)
if self.tokenizer != None: if self.tokenizer != None:
self.tokenizer.clear() self.tokenizer.clear()
@ -144,7 +122,7 @@ class MarkdownHighlighter(QSyntaxHighlighter):
i += 1 i += 1
#if self.currentBlockState() == MS.MarkdownStateBlockquote: #if self.currentBlockState() == MS.MarkdownStateBlockquote:
#fmt = QTextCharFormat(self.defaultFormat) #fmt = QTextCharFormat(self._defaultCharFormat)
#fmt.setForeground(Qt.lightGray) #fmt.setForeground(Qt.lightGray)
#self.setFormat(0, len(text), fmt) #self.setFormat(0, len(text), fmt)
@ -176,87 +154,6 @@ class MarkdownHighlighter(QSyntaxHighlighter):
if self.spellCheckEnabled: if self.spellCheckEnabled:
self.spellCheck(text) self.spellCheck(text)
# HASHTAGS AND HIGHLIGHTS
# Hashtags
s = 0
ht = QRegExp(r'([^#])(#[\w]+)')
while ht.indexIn(text, s) >= 0:
f = self.format(ht.pos()+1)
f.setForeground(QColor("#07c"))
f.setFontWeight(QFont.Bold)
self.setFormat(ht.pos()+1, ht.matchedLength()-1, f)
s = ht.pos() + 1
# Highlighted
for w in self.highlightedWords + self.highlightedTags:
pos = text.lower().find(w.lower())
while pos >= 0:
for i in range(pos, pos + len(w)):
f = self.format(i)
f.setBackground(QBrush(QColor("#fAf")))
self.setFormat(i, 1, f)
pos = text.lower().find(w.lower(), pos+1)
# Searched
#FIXME: consider searchExpressionRegExp
if self.searchExpression:
s = self.searchExpression
if not self.searchExpressionRegExp:
if self.searchExpressionCase:
pos = text.find(s)
else:
pos = text.lower().find(s.lower())
while pos >= 0:
for i in range(pos, pos + len(s)):
f = self.format(i)
f.setBackground(QBrush(QColor("#Aff")))
self.setFormat(i, 1, f)
pos = text.lower().find(s.lower(), pos+1)
else:
# Using QRegExp
rx = QRegExp(s)
if not self.searchExpressionCase:
rx.setCaseSensitivity(Qt.CaseInsensitive)
p = rx.indexIn(text)
while p != -1:
f = self.format(p)
f.setBackground(QBrush(QColor("#Aff")))
self.setFormat(p, rx.matchedLength(), f)
p = rx.indexIn(text, p + 1)
# Using python re
#try:
#for m in re.finditer(s, text):
#f = self.format(m.start())
#f.setBackground(QBrush(QColor("#0ff")))
#self.setFormat(m.start(), len(m.group()), f)
#except:
## Probably malformed regExp
#pass
# Custom rules
for rule, theme in self.customRules:
for m in re.finditer(rule, text):
if not m.groups(): # No groups, therefore no markup
f = self.format(m.start())
f, garbage = self.formatsFromTheme(theme, f)
self.setFormat(m.start(), len(m.group()), f)
else:
mf = self.format(m.start())
f = self.format(m.start() + len(m.group(1)))
f, mf = self.formatsFromTheme(theme, f, mf)
self.setFormat(m.start(1), len(m.group(1)), mf)
self.setFormat(m.start(2), len(m.group(2)), mf)
self.setFormat(m.start(1) + len(m.group(1)),
len(m.group())
- len(m.group(1))
- len(m.group(2)), f)
# If the block has transitioned from previously being a heading to now # If the block has transitioned from previously being a heading to now
# being a non-heading, signal that the position in the document no # being a non-heading, signal that the position in the document no
# longer contains a heading. # longer contains a heading.
@ -270,20 +167,25 @@ class MarkdownHighlighter(QSyntaxHighlighter):
# COLORS & FORMATTING # COLORS & FORMATTING
########################################################################### ###########################################################################
def updateColorScheme(self, rehighlight=True):
BasicHighlighter.updateColorScheme(self, rehighlight)
self.theme = self.defaultTheme()
self.setEnableLargeHeadingSizes(True)
def defaultTheme(self): def defaultTheme(self):
markup = qApp.palette().color(QPalette.Mid) markup = self.markupColor
if markup == Qt.black:
markup = Qt.lightGray
dark = qApp.palette().color(QPalette.Dark) dark = qApp.palette().color(QPalette.Dark)
if dark == Qt.black: if dark == Qt.black:
dark = QColor(Qt.gray) dark = QColor(Qt.gray)
darker = dark.darker(150) darker = dark.darker(150)
# Text background # Text background
background = qApp.palette().color(QPalette.Base) background = self.backgroundColor
lightBackground = background.darker(130) text = self.defaultTextColor
veryLightBackground = background.darker(105) lightBackground = F.mixColors(background, text, .4)
veryLightBackground = F.mixColors(background, text, .7)
link = self.linkColor
theme = { theme = {
"markup": markup} "markup": markup}
@ -311,17 +213,15 @@ class MarkdownHighlighter(QSyntaxHighlighter):
"formatMarkup":True, "formatMarkup":True,
"bold": True, "bold": True,
"monospace": True, "monospace": True,
#"color": Qt.darkBlue if i % 2 == 1 else Qt.darkMagenta,
} }
b = 100
d = 50 color = QColor(S.highlightedTextDark)
color = QColor(Qt.darkBlue)
theme[MTT.TokenAtxHeading1]["color"] = color theme[MTT.TokenAtxHeading1]["color"] = color
theme[MTT.TokenAtxHeading2]["color"] = color.lighter(b + d) theme[MTT.TokenAtxHeading2]["color"] = F.mixColors(color, background, .9)
theme[MTT.TokenAtxHeading3]["color"] = color.lighter(b + 2*d) theme[MTT.TokenAtxHeading3]["color"] = F.mixColors(color, background, .8)
theme[MTT.TokenAtxHeading4]["color"] = color.lighter(b + 3*d) theme[MTT.TokenAtxHeading4]["color"] = F.mixColors(color, background, .7)
theme[MTT.TokenAtxHeading5]["color"] = color.lighter(b + 4*d) theme[MTT.TokenAtxHeading5]["color"] = F.mixColors(color, background, .6)
theme[MTT.TokenAtxHeading6]["color"] = color.lighter(b + 5*d) theme[MTT.TokenAtxHeading6]["color"] = F.mixColors(color, background, .5)
for i in [MTT.TokenSetextHeading1Line2, MTT.TokenSetextHeading2Line2]: for i in [MTT.TokenSetextHeading1Line2, MTT.TokenSetextHeading2Line2]:
theme[i] = { theme[i] = {
@ -352,13 +252,13 @@ class MarkdownHighlighter(QSyntaxHighlighter):
theme[MTT.TokenHtmlEntity] = { theme[MTT.TokenHtmlEntity] = {
"color":Qt.red} "color":Qt.red}
theme[MTT.TokenAutomaticLink] = { theme[MTT.TokenAutomaticLink] = {
"color": qApp.palette().color(QPalette.Link)} "color": link}
theme[MTT.TokenInlineLink] = { theme[MTT.TokenInlineLink] = {
"color": qApp.palette().color(QPalette.Link)} "color": link}
theme[MTT.TokenReferenceLink] = { theme[MTT.TokenReferenceLink] = {
"color": qApp.palette().color(QPalette.Link)} "color": link}
theme[MTT.TokenReferenceDefinition] = { theme[MTT.TokenReferenceDefinition] = {
"color": qApp.palette().color(QPalette.Link)} "color": link}
theme[MTT.TokenImage] = { theme[MTT.TokenImage] = {
"color": Qt.green} "color": Qt.green}
theme[MTT.TokenHtmlComment] = { theme[MTT.TokenHtmlComment] = {
@ -402,25 +302,13 @@ class MarkdownHighlighter(QSyntaxHighlighter):
return theme return theme
def setColorScheme(self, defaultTextColor, backgroundColor, markupColor,
linkColor, spellingErrorColor):
self.defaultTextColor = defaultTextColor
self.backgroundColor = backgroundColor
self.markupColor = markupColor
self.linkColor = linkColor
self.spellingErrorColor = spellingErrorColor
self.defaultFormat.setForeground(QBrush(defaultTextColor))
# FIXME: generate a theme based on that
self.rehighlight()
########################################################################### ###########################################################################
# ACTUAL FORMATTING # ACTUAL FORMATTING
########################################################################### ###########################################################################
def applyFormattingForToken(self, token, text): def applyFormattingForToken(self, token, text):
if token.type != MTT.TokenUnknown: if token.type != MTT.TokenUnknown:
format = self.format(token.position + token.openingMarkupLength) fmt = self.format(token.position + token.openingMarkupLength)
markupFormat = self.format(token.position) markupFormat = self.format(token.position)
if self.theme.get("markup"): if self.theme.get("markup"):
markupFormat.setForeground(self.theme["markup"]) markupFormat.setForeground(self.theme["markup"])
@ -438,14 +326,14 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.currentBlockState(),) self.currentBlockState(),)
) )
#if token.type in range(6, 10): # if token.type in range(6, 10):
#debug() # debug()
theme = self.theme.get(token.type) theme = self.theme.get(token.type)
if theme: if theme:
format, markupFormat = self.formatsFromTheme(theme, fmt, markupFormat = self.formatsFromTheme(theme,
format, fmt,
markupFormat) markupFormat)
# Format openning Markup # Format openning Markup
self.setFormat(token.position, token.openingMarkupLength, self.setFormat(token.position, token.openingMarkupLength,
@ -455,7 +343,7 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.setFormat( self.setFormat(
token.position + token.openingMarkupLength, token.position + token.openingMarkupLength,
token.length - token.openingMarkupLength - token.closingMarkupLength, token.length - token.openingMarkupLength - token.closingMarkupLength,
format) fmt)
# Format closing Markup # Format closing Markup
if token.closingMarkupLength > 0: if token.closingMarkupLength > 0:
@ -468,13 +356,15 @@ class MarkdownHighlighter(QSyntaxHighlighter):
qWarning("MarkdownHighlighter.applyFormattingForToken() was passed" qWarning("MarkdownHighlighter.applyFormattingForToken() was passed"
" in a token of unknown type.") " in a token of unknown type.")
def formatsFromTheme(self, theme, format=QTextCharFormat(), def formatsFromTheme(self, theme, format=None,
markupFormat=QTextCharFormat()): markupFormat=QTextCharFormat()):
# Token # Token
if theme.get("color"): if theme.get("color"):
format.setForeground(theme["color"]) format.setForeground(theme["color"])
if theme.get("deltaSize"): if theme.get("deltaSize"):
format.setFontPointSize(format.fontPointSize() + theme["deltaSize"]) f = format.font()
f.setPointSize(format.font().pointSize() + theme["deltaSize"])
format.setFont(f)
if theme.get("background"): if theme.get("background"):
format.setBackground(theme["background"]) format.setBackground(theme["background"])
if theme.get("monospace"): if theme.get("monospace"):
@ -542,13 +432,13 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.rehighlight() self.rehighlight()
def increaseFontSize(self): def increaseFontSize(self):
self.defaultFormat.setFontPointSize(self.defaultFormat.fontPointSize() self._defaultCharFormat.setFontPointSize(
+ 1.0) self._defaultCharFormat.font().pointSize() + 1.0)
self.rehighlight() self.rehighlight()
def decreaseFontSize(self): def decreaseFontSize(self):
self.defaultFormat.setFontPointSize(self.defaultFormat.fontPointSize() self._defaultCharFormat.setFontPointSize(
- 1.0) self._defaultCharFormat.font().pointSize() - 1.0)
self.rehighlight() self.rehighlight()
def setEnableLargeHeadingSizes(self, enable): def setEnableLargeHeadingSizes(self, enable):
@ -577,8 +467,9 @@ class MarkdownHighlighter(QSyntaxHighlighter):
self.rehighlight() self.rehighlight()
def setFont(self, fontFamily, fontSize): def setFont(self, fontFamily, fontSize):
font = QFont(family=fontFamily, pointSize=fontSize, weight=QFont.Normal, italic=False) font = QFont(family=fontFamily, pointSize=fontSize,
self.defaultFormat.setFont(font) weight=QFont.Normal, italic=False)
self._defaultCharFormat.setFont(font)
self.rehighlight() self.rehighlight()
def setSpellCheckEnabled(self, enabled): def setSpellCheckEnabled(self, enabled):

246
manuskript/ui/highlighters/markdownTokenizer.py
View file

@ -6,8 +6,8 @@ from PyQt5.QtCore import *
from PyQt5.QtGui import * from PyQt5.QtGui import *
from PyQt5.QtWidgets import * from PyQt5.QtWidgets import *
from noteflow.ui.views.markdownEnums import MarkdownState as MS from manuskript.ui.highlighters import MarkdownState as MS
from noteflow.ui.views.markdownEnums import MarkdownTokenType as MTT from manuskript.ui.highlighters import MarkdownTokenType as MTT
# This file is simply a python translation of GhostWriter's Tokenizer. # This file is simply a python translation of GhostWriter's Tokenizer.
# http://wereturtle.github.io/ghostwriter/ # http://wereturtle.github.io/ghostwriter/
@ -69,10 +69,10 @@ class HighlightTokenizer:
class MarkdownTokenizer(HighlightTokenizer): class MarkdownTokenizer(HighlightTokenizer):
DUMMY_CHAR = "$" DUMMY_CHAR = "$"
MAX_MARKDOWN_HEADING_LEVEL = 6 MAX_MARKDOWN_HEADING_LEVEL = 6
paragraphBreakRegex = QRegExp("^\\s*$") paragraphBreakRegex = QRegExp("^\\s*$")
heading1SetextRegex = QRegExp("^===+\\s*$") heading1SetextRegex = QRegExp("^===+\\s*$")
heading2SetextRegex = QRegExp("^---+\\s*$") heading2SetextRegex = QRegExp("^---+\\s*$")
@ -112,21 +112,21 @@ class MarkdownTokenizer(HighlightTokenizer):
htmlInlineCommentRegex.setMinimal(True) htmlInlineCommentRegex.setMinimal(True)
mentionRegex = QRegExp("\\B@\\w+(\\-\\w+)*(/\\w+(\\-\\w+)*)?") mentionRegex = QRegExp("\\B@\\w+(\\-\\w+)*(/\\w+(\\-\\w+)*)?")
pipeTableDividerRegex = QRegExp("^ {0,3}(\\|[ :]?)?-{3,}([ :]?\\|[ :]?-{3,}([ :]?\\|)?)+\\s*$") pipeTableDividerRegex = QRegExp("^ {0,3}(\\|[ :]?)?-{3,}([ :]?\\|[ :]?-{3,}([ :]?\\|)?)+\\s*$")
def __init__(self): def __init__(self):
HighlightTokenizer.__init__(self) HighlightTokenizer.__init__(self)
def tokenize(self, text, currentState, previousState, nextState): def tokenize(self, text, currentState, previousState, nextState):
self.currentState = currentState self.currentState = currentState
self.previousState = previousState self.previousState = previousState
self.nextState = nextState self.nextState = nextState
if (self.previousState == MS.MarkdownStateInGithubCodeFence or \ if (self.previousState == MS.MarkdownStateInGithubCodeFence or \
self.previousState == MS.MarkdownStateInPandocCodeFence) and \ self.previousState == MS.MarkdownStateInPandocCodeFence) and \
self.tokenizeCodeBlock(text): self.tokenizeCodeBlock(text):
# No further tokenizing required # No further tokenizing required
pass pass
elif self.previousState != MS.MarkdownStateComment \ elif self.previousState != MS.MarkdownStateComment \
and self.paragraphBreakRegex.exactMatch(text): and self.paragraphBreakRegex.exactMatch(text):
@ -137,7 +137,7 @@ class MarkdownTokenizer(HighlightTokenizer):
elif previousState != MS.MarkdownStateCodeBlock or \ elif previousState != MS.MarkdownStateCodeBlock or \
(text[:1] != "\t" and text[-4:] != " "): (text[:1] != "\t" and text[-4:] != " "):
self.setState(MS.MarkdownStateParagraphBreak) self.setState(MS.MarkdownStateParagraphBreak)
elif self.tokenizeSetextHeadingLine2(text) or \ elif self.tokenizeSetextHeadingLine2(text) or \
self.tokenizeCodeBlock(text) or \ self.tokenizeCodeBlock(text) or \
self.tokenizeMultilineComment(text) or \ self.tokenizeMultilineComment(text) or \
@ -145,7 +145,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.tokenizeTableDivider(text): self.tokenizeTableDivider(text):
# No further tokenizing required # No further tokenizing required
pass pass
elif self.tokenizeSetextHeadingLine1(text) or \ elif self.tokenizeSetextHeadingLine1(text) or \
self.tokenizeAtxHeading(text) or \ self.tokenizeAtxHeading(text) or \
self.tokenizeBlockquote(text) or \ self.tokenizeBlockquote(text) or \
@ -153,7 +153,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.tokenizeBulletPointList(text): self.tokenizeBulletPointList(text):
self.tokenizeLineBreak(text) self.tokenizeLineBreak(text)
self.tokenizeInline(text) self.tokenizeInline(text)
else: else:
if previousState in [MS.MarkdownStateListLineBreak, if previousState in [MS.MarkdownStateListLineBreak,
MS.MarkdownStateNumberedList, MS.MarkdownStateNumberedList,
@ -168,7 +168,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.setState(MS.MarkdownStateParagraph) self.setState(MS.MarkdownStateParagraph)
self.tokenizeLineBreak(text) self.tokenizeLineBreak(text)
self.tokenizeInline(text) self.tokenizeInline(text)
# Make sure that if the second line of a setext heading is removed the # Make sure that if the second line of a setext heading is removed the
# first line is reprocessed. Otherwise, it will still show up in the # first line is reprocessed. Otherwise, it will still show up in the
# document as a heading. # document as a heading.
@ -177,7 +177,7 @@ class MarkdownTokenizer(HighlightTokenizer):
(previousState == MS.MarkdownStateSetextHeading2Line1 and \ (previousState == MS.MarkdownStateSetextHeading2Line1 and \
self.getState() != MS.MarkdownStateSetextHeading2Line2): self.getState() != MS.MarkdownStateSetextHeading2Line2):
self.requestBacktrack() self.requestBacktrack()
def tokenizeSetextHeadingLine1(self, text): def tokenizeSetextHeadingLine1(self, text):
#Check the next line's state to see if this is a setext-style heading. #Check the next line's state to see if this is a setext-style heading.
level = 0 level = 0
@ -188,12 +188,12 @@ class MarkdownTokenizer(HighlightTokenizer):
level = 1 level = 1
self.setState(MS.MarkdownStateSetextHeading1Line1) self.setState(MS.MarkdownStateSetextHeading1Line1)
token.type = MTT.TokenSetextHeading1Line1 token.type = MTT.TokenSetextHeading1Line1
elif MS.MarkdownStateSetextHeading2Line2 == nextState: elif MS.MarkdownStateSetextHeading2Line2 == nextState:
level = 2 level = 2
self.setState(MS.MarkdownStateSetextHeading2Line1) self.setState(MS.MarkdownStateSetextHeading2Line1)
token.type = MTT.TokenSetextHeading2Line1 token.type = MTT.TokenSetextHeading2Line1
if level > 0: if level > 0:
token.length = len(text) token.length = len(text)
token.position = 0 token.position = 0
@ -201,7 +201,7 @@ class MarkdownTokenizer(HighlightTokenizer):
return True return True
return False return False
def tokenizeSetextHeadingLine2(self, text): def tokenizeSetextHeadingLine2(self, text):
level = 0 level = 0
setextMatch = False setextMatch = False
@ -212,13 +212,13 @@ class MarkdownTokenizer(HighlightTokenizer):
setextMatch = self.heading1SetextRegex.exactMatch(text) setextMatch = self.heading1SetextRegex.exactMatch(text)
self.setState(MS.MarkdownStateSetextHeading1Line2) self.setState(MS.MarkdownStateSetextHeading1Line2)
token.type = MTT.TokenSetextHeading1Line2 token.type = MTT.TokenSetextHeading1Line2
elif previousState == MS.MarkdownStateSetextHeading2Line1: elif previousState == MS.MarkdownStateSetextHeading2Line1:
level = 2 level = 2
setextMatch = self.heading2SetextRegex.exactMatch(text) setextMatch = self.heading2SetextRegex.exactMatch(text)
self.setState(MS.MarkdownStateSetextHeading2Line2) self.setState(MS.MarkdownStateSetextHeading2Line2)
token.type = MTT.TokenSetextHeading2Line2 token.type = MTT.TokenSetextHeading2Line2
elif previousState == MS.MarkdownStateParagraph: elif previousState == MS.MarkdownStateParagraph:
h1Line2 = self.heading1SetextRegex.exactMatch(text) h1Line2 = self.heading1SetextRegex.exactMatch(text)
h2Line2 = self.heading2SetextRegex.exactMatch(text) h2Line2 = self.heading2SetextRegex.exactMatch(text)
@ -232,7 +232,7 @@ class MarkdownTokenizer(HighlightTokenizer):
if h1Line2: if h1Line2:
self.setState(MS.MarkdownStateSetextHeading1Line2) self.setState(MS.MarkdownStateSetextHeading1Line2)
token.type = MTT.TokenSetextHeading1Line2 token.type = MTT.TokenSetextHeading1Line2
else: else:
self.setState(MS.MarkdownStateSetextHeading2Line2) self.setState(MS.MarkdownStateSetextHeading2Line2)
token.type = MTT.TokenSetextHeading2Line2 token.type = MTT.TokenSetextHeading2Line2
@ -246,32 +246,32 @@ class MarkdownTokenizer(HighlightTokenizer):
token.position = 0 token.position = 0
self.addToken(token) self.addToken(token)
return True return True
else: else:
# Restart tokenizing on the previous line. # Restart tokenizing on the previous line.
self.requestBacktrack() self.requestBacktrack()
False False
return False return False
def tokenizeAtxHeading(self, text): def tokenizeAtxHeading(self, text):
escapedText = self.dummyOutEscapeCharacters(text) escapedText = self.dummyOutEscapeCharacters(text)
trailingPoundCount = 0 trailingPoundCount = 0
level = 0 level = 0
#Count the number of pound signs at the front of the string, #Count the number of pound signs at the front of the string,
#up to the maximum allowed, to determine the heading level. #up to the maximum allowed, to determine the heading level.
while escapedText[level] == "#": while escapedText[level] == "#":
level += 1 level += 1
if level >= len(escapedText) or level >= self.MAX_MARKDOWN_HEADING_LEVEL: if level >= len(escapedText) or level >= self.MAX_MARKDOWN_HEADING_LEVEL:
break break
if level > 0 and level < len(text): if level > 0 and level < len(text):
# Count how many pound signs are at the end of the text. # Count how many pound signs are at the end of the text.
while escapedText[-trailingPoundCount -1] == "#": while escapedText[-trailingPoundCount -1] == "#":
trailingPoundCount += 1 trailingPoundCount += 1
token = Token() token = Token()
token.position = 0 token.position = 0
token.length = len(text) token.length = len(text)
@ -282,7 +282,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.setState(MS.MarkdownStateAtxHeading1 + level -1) self.setState(MS.MarkdownStateAtxHeading1 + level -1)
return True return True
return False return False
def tokenizeNumberedList(self, text): def tokenizeNumberedList(self, text):
previousState = self.previousState previousState = self.previousState
if (previousState in [MS.MarkdownStateParagraphBreak, if (previousState in [MS.MarkdownStateParagraphBreak,
@ -296,7 +296,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.numberedNestedListRegex.exactMatch(text)): self.numberedNestedListRegex.exactMatch(text)):
periodIndex = text.find(".") periodIndex = text.find(".")
parenthIndex = text.find(")") parenthIndex = text.find(")")
if periodIndex < 0: if periodIndex < 0:
index = parenthIndex index = parenthIndex
elif parenthIndex < 0: elif parenthIndex < 0:
@ -305,7 +305,7 @@ class MarkdownTokenizer(HighlightTokenizer):
index = periodIndex index = periodIndex
else: else:
index = parenthIndex index = parenthIndex
if index > 0: if index > 0:
token = Token() token = Token()
token.type = MTT.TokenNumberedList token.type = MTT.TokenNumberedList
@ -315,11 +315,11 @@ class MarkdownTokenizer(HighlightTokenizer):
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateNumberedList) self.setState(MS.MarkdownStateNumberedList)
return True return True
return False return False
return False return False
def tokenizeBulletPointList(self, text): def tokenizeBulletPointList(self, text):
foundBulletChar = False foundBulletChar = False
bulletCharIndex = -1 bulletCharIndex = -1
@ -338,17 +338,17 @@ class MarkdownTokenizer(HighlightTokenizer):
# Search for the bullet point character, which can # Search for the bullet point character, which can
# be either a '+', '-', or '*'. # be either a '+', '-', or '*'.
for i in range(len(text)): for i in range(len(text)):
if text[i] == " ": if text[i] == " ":
if foundBulletChar: if foundBulletChar:
# We've confirmed it's a bullet point by the whitespace that # We've confirmed it's a bullet point by the whitespace that
# follows the bullet point character, and can now exit the # follows the bullet point character, and can now exit the
# loop. # loop.
whitespaceFoundAfterBulletChar = True whitespaceFoundAfterBulletChar = True
break break
else: else:
spaceCount += 1 spaceCount += 1
@ -356,7 +356,7 @@ class MarkdownTokenizer(HighlightTokenizer):
# number of spaces preceeding the bullet point does not # number of spaces preceeding the bullet point does not
# exceed three, as that would indicate a code block rather # exceed three, as that would indicate a code block rather
# than a bullet point list. # than a bullet point list.
if spaceCount > 3 and previousState not in [ if spaceCount > 3 and previousState not in [
MS.MarkdownStateNumberedList, MS.MarkdownStateNumberedList,
MS.MarkdownStateBulletPointList, MS.MarkdownStateBulletPointList,
@ -367,30 +367,30 @@ class MarkdownTokenizer(HighlightTokenizer):
MS.MarkdownStateCodeBlock, MS.MarkdownStateCodeBlock,
MS.MarkdownStateCodeFenceEnd,]: MS.MarkdownStateCodeFenceEnd,]:
return False return False
elif text[i] == "\t": elif text[i] == "\t":
if foundBulletChar: if foundBulletChar:
# We've confirmed it's a bullet point by the whitespace that # We've confirmed it's a bullet point by the whitespace that
# follows the bullet point character, and can now exit the # follows the bullet point character, and can now exit the
# loop. # loop.
whitespaceFoundAfterBulletChar = True whitespaceFoundAfterBulletChar = True
break break
elif previousState in [ elif previousState in [
MS.MarkdownStateParagraphBreak, MS.MarkdownStateParagraphBreak,
MS.MarkdownStateUnknown]: MS.MarkdownStateUnknown]:
# If this list item is the first in the list, ensure that # If this list item is the first in the list, ensure that
# no tab character preceedes the bullet point, as that would # no tab character preceedes the bullet point, as that would
# indicate a code block rather than a bullet point list. # indicate a code block rather than a bullet point list.
return False return False
elif text[i] in ["+", "-", "*"]: elif text[i] in ["+", "-", "*"]:
foundBulletChar = True foundBulletChar = True
bulletCharIndex = i bulletCharIndex = i
else: else:
return False return False
@ -403,9 +403,9 @@ class MarkdownTokenizer(HighlightTokenizer):
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateBulletPointList) self.setState(MS.MarkdownStateBulletPointList)
return True return True
return False return False
def tokenizeHorizontalRule (self, text): def tokenizeHorizontalRule (self, text):
if self.hruleRegex.exactMatch(text): if self.hruleRegex.exactMatch(text):
token = Token() token = Token()
@ -417,12 +417,12 @@ class MarkdownTokenizer(HighlightTokenizer):
return True return True
return False return False
def tokenizeLineBreak(self, text): def tokenizeLineBreak(self, text):
currentState = self.currentState currentState = self.currentState
previousState = self.previousState previousState = self.previousState
nextState = self.nextState nextState = self.nextState
if currentState in [ if currentState in [
MS.MarkdownStateParagraph, MS.MarkdownStateParagraph,
MS.MarkdownStateBlockquote, MS.MarkdownStateBlockquote,
@ -434,7 +434,7 @@ class MarkdownTokenizer(HighlightTokenizer):
MS.MarkdownStateNumberedList, MS.MarkdownStateNumberedList,
MS.MarkdownStateBulletPointList,]: MS.MarkdownStateBulletPointList,]:
self.requestBacktrack() self.requestBacktrack()
if nextState in [ if nextState in [
MS.MarkdownStateParagraph, MS.MarkdownStateParagraph,
MS.MarkdownStateBlockquote, MS.MarkdownStateBlockquote,
@ -448,17 +448,17 @@ class MarkdownTokenizer(HighlightTokenizer):
token.length = 1 token.length = 1
self.addToken(token) self.addToken(token)
return True return True
return False return False
def tokenizeBlockquote(self, text): def tokenizeBlockquote(self, text):
previousState = self.previousState previousState = self.previousState
if previousState == MS.MarkdownStateBlockquote or \ if previousState == MS.MarkdownStateBlockquote or \
self.blockquoteRegex.exactMatch(text): self.blockquoteRegex.exactMatch(text):
# Find any '>' characters at the front of the line. # Find any '>' characters at the front of the line.
markupLength = 0 markupLength = 0
for i in range(len(text)): for i in range(len(text)):
if text[i] == ">": if text[i] == ">":
markupLength = i + 1 markupLength = i + 1
@ -466,27 +466,27 @@ class MarkdownTokenizer(HighlightTokenizer):
# There are no more '>' characters at the front of the line, # There are no more '>' characters at the front of the line,
# so stop processing. # so stop processing.
break break
token = Token() token = Token()
token.type = MTT.TokenBlockquote token.type = MTT.TokenBlockquote
token.position = 0 token.position = 0
token.length = len(text) token.length = len(text)
if markupLength > 0: if markupLength > 0:
token.openingMarkupLength = markupLength token.openingMarkupLength = markupLength
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateBlockquote) self.setState(MS.MarkdownStateBlockquote)
return True return True
return False return False
def tokenizeCodeBlock(self, text): def tokenizeCodeBlock(self, text):
previousState = self.previousState previousState = self.previousState
if previousState in [ if previousState in [
MS.MarkdownStateInGithubCodeFence, MS.MarkdownStateInGithubCodeFence,
MS.MarkdownStateInPandocCodeFence]: MS.MarkdownStateInPandocCodeFence]:
self.setState(previousState) self.setState(previousState)
if (previousState == MS.MarkdownStateInGithubCodeFence and \ if (previousState == MS.MarkdownStateInGithubCodeFence and \
self.githubCodeFenceEndRegex.exactMatch(text)) or \ self.githubCodeFenceEndRegex.exactMatch(text)) or \
(previousState == MS.MarkdownStateInPandocCodeFence and \ (previousState == MS.MarkdownStateInPandocCodeFence and \
@ -497,16 +497,16 @@ class MarkdownTokenizer(HighlightTokenizer):
token.length = len(text) token.length = len(text)
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateCodeFenceEnd) self.setState(MS.MarkdownStateCodeFenceEnd)
else: else:
token = Token() token = Token()
token.type = MTT.TokenCodeBlock token.type = MTT.TokenCodeBlock
token.position = 0 token.position = 0
token.length = len(text) token.length = len(text)
self.addToken(token) self.addToken(token)
return True return True
elif previousState in [ elif previousState in [
MS.MarkdownStateCodeBlock, MS.MarkdownStateCodeBlock,
MS.MarkdownStateParagraphBreak, MS.MarkdownStateParagraphBreak,
@ -520,7 +520,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateCodeBlock) self.setState(MS.MarkdownStateCodeBlock)
return True return True
elif previousState in [ elif previousState in [
MS.MarkdownStateParagraphBreak, MS.MarkdownStateParagraphBreak,
MS.MarkdownStateParagraph, MS.MarkdownStateParagraph,
@ -532,49 +532,49 @@ class MarkdownTokenizer(HighlightTokenizer):
foundCodeFenceStart = True foundCodeFenceStart = True
token.type = MTT.TokenGithubCodeFence token.type = MTT.TokenGithubCodeFence
self.setState(MS.MarkdownStateInGithubCodeFence) self.setState(MS.MarkdownStateInGithubCodeFence)
elif self.pandocCodeFenceStartRegex.exactMatch(text): elif self.pandocCodeFenceStartRegex.exactMatch(text):
foundCodeFenceStart = True foundCodeFenceStart = True
token.type = MTT.TokenPandocCodeFence token.type = MTT.TokenPandocCodeFence
self.setState(MS.MarkdownStateInPandocCodeFence) self.setState(MS.MarkdownStateInPandocCodeFence)
if foundCodeFenceStart: if foundCodeFenceStart:
token.position = 0 token.position = 0
token.length = len(text) token.length = len(text)
self.addToken(token) self.addToken(token)
return True return True
return False return False
def tokenizeMultilineComment(self, text): def tokenizeMultilineComment(self, text):
previousState = self.previousState previousState = self.previousState
if previousState == MS.MarkdownStateComment: if previousState == MS.MarkdownStateComment:
# Find the end of the comment, if any. # Find the end of the comment, if any.
index = text.find("-->") index = text.find("-->")
token = Token() token = Token()
token.type = MTT.TokenHtmlComment token.type = MTT.TokenHtmlComment
token.position = 0 token.position = 0
if index >= 0: if index >= 0:
token.length = index + 3 token.length = index + 3
self.addToken(token) self.addToken(token)
# Return false so that the rest of the line that isn't within # Return false so that the rest of the line that isn't within
# the commented segment can be highlighted as normal paragraph # the commented segment can be highlighted as normal paragraph
# text. # text.
else: else:
token.length = len(text) token.length = len(text)
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateComment) self.setState(MS.MarkdownStateComment)
return True return True
return False return False
def tokenizeInline(self, text): def tokenizeInline(self, text):
escapedText = self.dummyOutEscapeCharacters(text) escapedText = self.dummyOutEscapeCharacters(text)
# Check if the line is a reference definition. # Check if the line is a reference definition.
if self.referenceDefinitionRegex.exactMatch(text): if self.referenceDefinitionRegex.exactMatch(text):
colonIndex = escapedText.find(":") colonIndex = escapedText.find(":")
@ -583,10 +583,10 @@ class MarkdownTokenizer(HighlightTokenizer):
token.position = 0 token.position = 0
token.length = colonIndex + 1 token.length = colonIndex + 1
self.addToken(token) self.addToken(token)
# Replace the first bracket so that the '[...]:' reference definition # Replace the first bracket so that the '[...]:' reference definition
# start doesn't get highlighted as a reference link. # start doesn't get highlighted as a reference link.
firstBracketIndex = escapedText.find("[") firstBracketIndex = escapedText.find("[")
if firstBracketIndex >= 0: if firstBracketIndex >= 0:
i = firstBracketIndex i = firstBracketIndex
@ -610,21 +610,21 @@ class MarkdownTokenizer(HighlightTokenizer):
escapedText = self.tokenizeMatches(MTT.TokenMention, escapedText, self.mentionRegex, 0, 0, False, True) escapedText = self.tokenizeMatches(MTT.TokenMention, escapedText, self.mentionRegex, 0, 0, False, True)
return True return True
def tokenizeVerbatim(self, text): def tokenizeVerbatim(self, text):
index = self.verbatimRegex.indexIn(text) index = self.verbatimRegex.indexIn(text)
while index >= 0: while index >= 0:
end = "" end = ""
count = self.verbatimRegex.matchedLength() count = self.verbatimRegex.matchedLength()
# Search for the matching end, which should have the same number # Search for the matching end, which should have the same number
# of back ticks as the start. # of back ticks as the start.
for i in range(count): for i in range(count):
end += '`' end += '`'
endIndex = text.find(end, index + count) endIndex = text.find(end, index + count)
# If the end was found, add the verbatim token. # If the end was found, add the verbatim token.
if endIndex >= 0: if endIndex >= 0:
token = Token() token = Token()
@ -634,26 +634,26 @@ class MarkdownTokenizer(HighlightTokenizer):
token.openingMarkupLength = count token.openingMarkupLength = count
token.closingMarkupLength = count token.closingMarkupLength = count
self.addToken(token) self.addToken(token)
# Fill out the token match in the string with the dummy # Fill out the token match in the string with the dummy
# character so that searches for other Markdown elements # character so that searches for other Markdown elements
# don't find anything within this token's range in the string. # don't find anything within this token's range in the string.
for i in range(index, index + token.length): for i in range(index, index + token.length):
text = text[:i] + self.DUMMY_CHAR + text[i+1:] text = text[:i] + self.DUMMY_CHAR + text[i+1:]
index += token.length index += token.length
# Else start searching again at the very next character. # Else start searching again at the very next character.
else: else:
index += 1 index += 1
index = self.verbatimRegex.indexIn(text, index) index = self.verbatimRegex.indexIn(text, index)
return text return text
def tokenizeHtmlComments(self, text): def tokenizeHtmlComments(self, text):
previousState = self.previousState previousState = self.previousState
# Check for the end of a multiline comment so that it doesn't get further # Check for the end of a multiline comment so that it doesn't get further
# tokenized. Don't bother formatting the comment itself, however, because # tokenized. Don't bother formatting the comment itself, however, because
# it should have already been tokenized in tokenizeMultilineComment(). # it should have already been tokenized in tokenizeMultilineComment().
@ -661,10 +661,10 @@ class MarkdownTokenizer(HighlightTokenizer):
commentEnd = text.find("-->") commentEnd = text.find("-->")
for i in range(commentEnd + 3): for i in range(commentEnd + 3):
text = text[:i] + self.DUMMY_CHAR + text[i+1:] text = text[:i] + self.DUMMY_CHAR + text[i+1:]
# Now check for inline comments (non-multiline). # Now check for inline comments (non-multiline).
commentStart = self.htmlInlineCommentRegex.indexIn(text) commentStart = self.htmlInlineCommentRegex.indexIn(text)
while commentStart >= 0: while commentStart >= 0:
commentLength = self.htmlInlineCommentRegex.matchedLength() commentLength = self.htmlInlineCommentRegex.matchedLength()
token = Token() token = Token()
@ -672,15 +672,15 @@ class MarkdownTokenizer(HighlightTokenizer):
token.position = commentStart token.position = commentStart
token.length = commentLength token.length = commentLength
self.addToken(token) self.addToken(token)
# Replace comment segment with dummy characters so that it doesn't # Replace comment segment with dummy characters so that it doesn't
# get tokenized again. # get tokenized again.
for i in range(commentStart, commentStart + commentLength): for i in range(commentStart, commentStart + commentLength):
text = text[:i] + self.DUMMY_CHAR + text[i+1:] text = text[:i] + self.DUMMY_CHAR + text[i+1:]
commentStart = self.htmlInlineCommentRegex.indexIn(text, commentStart + commentLength) commentStart = self.htmlInlineCommentRegex.indexIn(text, commentStart + commentLength)
# Find multiline comment start, if any. # Find multiline comment start, if any.
commentStart = text.find("<!--") commentStart = text.find("<!--")
if commentStart >= 0: if commentStart >= 0:
@ -690,18 +690,18 @@ class MarkdownTokenizer(HighlightTokenizer):
token.length = len(text) - commentStart token.length = len(text) - commentStart
self.addToken(token) self.addToken(token)
self.setState(MS.MarkdownStateComment) self.setState(MS.MarkdownStateComment)
# Replace comment segment with dummy characters so that it doesn't # Replace comment segment with dummy characters so that it doesn't
# get tokenized again. # get tokenized again.
for i in range(commentStart, len(text)): for i in range(commentStart, len(text)):
text = text[:i] + self.DUMMY_CHAR + text[i+1:] text = text[:i] + self.DUMMY_CHAR + text[i+1:]
return text return text
def tokenizeTableHeaderRow(self, text): def tokenizeTableHeaderRow(self, text):
previousState = self.previousState previousState = self.previousState
nextState = self.nextState nextState = self.nextState
if previousState in [ if previousState in [
MS.MarkdownStateParagraphBreak, MS.MarkdownStateParagraphBreak,
MS.MarkdownStateListLineBreak, MS.MarkdownStateListLineBreak,
@ -721,7 +721,7 @@ class MarkdownTokenizer(HighlightTokenizer):
MS.MarkdownStateUnknown] and \ MS.MarkdownStateUnknown] and \
nextState == MS.MarkdownStatePipeTableDivider: nextState == MS.MarkdownStatePipeTableDivider:
self.setState(MS.MarkdownStatePipeTableHeader) self.setState(MS.MarkdownStatePipeTableHeader)
headerStart = 0 headerStart = 0
for i in range(len(text)): for i in range(len(text)):
if text[i] == "|": if text[i] == "|":
@ -731,21 +731,21 @@ class MarkdownTokenizer(HighlightTokenizer):
# to prevent formatting such as strong and emphasis from # to prevent formatting such as strong and emphasis from
# picking it up. # picking it up.
text = text[:i] + " " + text[i+1:] text = text[:i] + " " + text[i+1:]
token = Token() token = Token()
if i > 0: if i > 0:
token.type = MTT.TokenTableHeader token.type = MTT.TokenTableHeader
token.position = headerStart token.position = headerStart
token.length = i - headerStart token.length = i - headerStart
self.addToken(token) self.addToken(token)
token.type = MTT.TokenTablePipe token.type = MTT.TokenTablePipe
token.position = i token.position = i
token.length = 1 token.length = 1
self.addToken(token) self.addToken(token)
headerStart = i + 1 headerStart = i + 1
if headerStart < len(text): if headerStart < len(text):
token = Token() token = Token()
token.type = MTT.TokenTableHeader token.type = MTT.TokenTableHeader
@ -754,7 +754,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.addToken(token) self.addToken(token)
return text return text
def tokenizeTableDivider(self, text): def tokenizeTableDivider(self, text):
previousState = self.previousState previousState = self.previousState
if previousState == MS.MarkdownStatePipeTableHeader: if previousState == MS.MarkdownStatePipeTableHeader:
@ -765,9 +765,9 @@ class MarkdownTokenizer(HighlightTokenizer):
token.length = len(text) token.length = len(text)
token.position = 0 token.position = 0
self.addToken(token) self.addToken(token)
return True return True
else: else:
# Restart tokenizing on the previous line. # Restart tokenizing on the previous line.
self.requestBacktrack() self.requestBacktrack()
@ -776,24 +776,24 @@ class MarkdownTokenizer(HighlightTokenizer):
# Restart tokenizing on the previous line. # Restart tokenizing on the previous line.
self.requestBacktrack() self.requestBacktrack()
self.setState(MS.MarkdownStatePipeTableDivider) self.setState(MS.MarkdownStatePipeTableDivider)
token = Token() token = Token()
token.length = len(text) token.length = len(text)
token.position = 0 token.position = 0
token.type = MTT.TokenTableDivider token.type = MTT.TokenTableDivider
self.addToken(token) self.addToken(token)
return True return True
return False return False
def tokenizeTableRow(self, text): def tokenizeTableRow(self, text):
previousState = self.previousState previousState = self.previousState
if previousState in [ if previousState in [
MS.MarkdownStatePipeTableDivider, MS.MarkdownStatePipeTableDivider,
MS.MarkdownStatePipeTableRow]: MS.MarkdownStatePipeTableRow]:
self.setState(MS.MarkdownStatePipeTableRow) self.setState(MS.MarkdownStatePipeTableRow)
for i in range(len(text)): for i in range(len(text)):
if text[i] == "|": if text[i] == "|":
# Replace pipe with space so that it doesn't get formatted # Replace pipe with space so that it doesn't get formatted
@ -801,7 +801,7 @@ class MarkdownTokenizer(HighlightTokenizer):
# Note that we use a space rather than DUMMY_CHAR for this, # Note that we use a space rather than DUMMY_CHAR for this,
# to prevent formatting such as strong and emphasis from # to prevent formatting such as strong and emphasis from
# picking it up. # picking it up.
text = text[:i] + " " + text[i+1:] text = text[:i] + " " + text[i+1:]
token = Token() token = Token()
@ -811,7 +811,7 @@ class MarkdownTokenizer(HighlightTokenizer):
self.addToken(token) self.addToken(token)
return text return text
def tokenizeMatches(self, tokenType, text, regex, def tokenizeMatches(self, tokenType, text, regex,
markupStartCount=0, markupEndCount=0, markupStartCount=0, markupEndCount=0,
replaceMarkupChars=False, replaceAllChars=False): replaceMarkupChars=False, replaceAllChars=False):
@ -821,40 +821,40 @@ class MarkdownTokenizer(HighlightTokenizer):
tokens. The markupStartCount and markupEndCount values are used to tokens. The markupStartCount and markupEndCount values are used to
indicate how many markup special characters preceed and follow the indicate how many markup special characters preceed and follow the
main text, respectively. main text, respectively.
For example, if the matched string is "**bold**", and For example, if the matched string is "**bold**", and
markupStartCount = 2 and markupEndCount = 2, then the asterisks markupStartCount = 2 and markupEndCount = 2, then the asterisks
preceeding and following the word "bold" will be set as opening and preceeding and following the word "bold" will be set as opening and
closing markup in the token. closing markup in the token.
If replaceMarkupChars is true, then the markupStartCount and If replaceMarkupChars is true, then the markupStartCount and
markupEndCount characters will be replaced with a dummy character in markupEndCount characters will be replaced with a dummy character in
the text QString so that subsequent parsings of the same line do not the text QString so that subsequent parsings of the same line do not
pick up the original characters. pick up the original characters.
If replaceAllChars is true instead, then the entire matched text will If replaceAllChars is true instead, then the entire matched text will
be replaced with dummy characters--again, for ease in parsing the be replaced with dummy characters--again, for ease in parsing the
same line for other regular expression matches. same line for other regular expression matches.
""" """
index = regex.indexIn(text) index = regex.indexIn(text)
while index >= 0: while index >= 0:
length = regex.matchedLength() length = regex.matchedLength()
token = Token() token = Token()
token.type = tokenType token.type = tokenType
token.position = index token.position = index
token.length = length token.length = length
if markupStartCount > 0: if markupStartCount > 0:
token.openingMarkupLength = markupStartCount token.openingMarkupLength = markupStartCount
if markupEndCount > 0: if markupEndCount > 0:
token.closingMarkupLength = markupEndCount token.closingMarkupLength = markupEndCount
if replaceAllChars: if replaceAllChars:
for i in range(index, index + length): for i in range(index, index + length):
text = text[:i] + self.DUMMY_CHAR + text[i+1:] text = text[:i] + self.DUMMY_CHAR + text[i+1:]
elif replaceMarkupChars: elif replaceMarkupChars:
for i in range(index, index + markupStartCount): for i in range(index, index + markupStartCount):
text = text[:i] + self.DUMMY_CHAR + text[i+1:] text = text[:i] + self.DUMMY_CHAR + text[i+1:]
@ -865,19 +865,19 @@ class MarkdownTokenizer(HighlightTokenizer):
index = regex.indexIn(text, index + length) index = regex.indexIn(text, index + length)
return text return text
def dummyOutEscapeCharacters(self, text): def dummyOutEscapeCharacters(self, text):
""" """
Replaces escaped characters in text so they aren't picked up Replaces escaped characters in text so they aren't picked up
during parsing. Returns a copy of the input text string during parsing. Returns a copy of the input text string
with the escaped characters replaced with a dummy character. with the escaped characters replaced with a dummy character.
""" """
return re.sub("\\\\.", "\$", text) return re.sub("\\\\.", "\$", text)
#escape = False #escape = False
#escapedText = text #escapedText = text
#for i in range(len(text)): #for i in range(len(text)):
#if escape: #if escape:
#escapedText = escapedText[:i] + self.DUMMY_CHAR + escapedText[i+1:] #escapedText = escapedText[:i] + self.DUMMY_CHAR + escapedText[i+1:]

2
manuskript/ui/style.py
View file

@ -24,6 +24,8 @@ button = p.color(QPalette.Button).name() # Button background
buttonText = p.color(QPalette.ButtonText).name() # Button Text buttonText = p.color(QPalette.ButtonText).name() # Button Text
highlight = p.color(QPalette.Highlight).name() # Other background highlight = p.color(QPalette.Highlight).name() # Other background
highlightedText = p.color(QPalette.HighlightedText).name() # Base Text highlightedText = p.color(QPalette.HighlightedText).name() # Base Text
link = p.color(QPalette.Link).name() # Link
linkVisited = p.color(QPalette.LinkVisited).name() # Link visited
light = p.color(QPalette.Light).name() # Lighter than Button color light = p.color(QPalette.Light).name() # Lighter than Button color
midlight = p.color(QPalette.Midlight).name() # Between Button and Light midlight = p.color(QPalette.Midlight).name() # Between Button and Light

20
manuskript/ui/views/textEditView.py
View file

@ -11,7 +11,8 @@ from manuskript.enums import Outline
from manuskript import functions as F from manuskript import functions as F
from manuskript.models.outlineModel import outlineModel from manuskript.models.outlineModel import outlineModel
from manuskript.ui.editors.MDFunctions import MDFormatSelection from manuskript.ui.editors.MDFunctions import MDFormatSelection
from manuskript.ui.highlighters import MMDHighlighter, BasicHighlighter from manuskript.ui.highlighters import BasicHighlighter, MarkdownHighlighter
from manuskript.ui.highlighters import MMDHighlighter
from manuskript.ui.editors.textFormat import textFormat from manuskript.ui.editors.textFormat import textFormat
from manuskript.ui import style as S from manuskript.ui import style as S
@ -22,8 +23,8 @@ except ImportError:
class textEditView(QTextEdit): class textEditView(QTextEdit):
def __init__(self, parent=None, index=None, html=None, spellcheck=True, highlighting=False, dict="", def __init__(self, parent=None, index=None, html=None, spellcheck=True,
autoResize=False): highlighting=False, dict="", autoResize=False):
QTextEdit.__init__(self, parent) QTextEdit.__init__(self, parent)
self._column = Outline.text.value self._column = Outline.text.value
self._index = None self._index = None
@ -75,7 +76,8 @@ class textEditView(QTextEdit):
# Spellchecking # Spellchecking
if enchant and self.spellcheck: if enchant and self.spellcheck:
try: try:
self._dict = enchant.Dict(self.currentDict if self.currentDict else self.getDefaultLocale()) self._dict = enchant.Dict(self.currentDict if self.currentDict
else self.getDefaultLocale())
except enchant.errors.DictNotFoundError: except enchant.errors.DictNotFoundError:
self.spellcheck = False self.spellcheck = False
@ -188,11 +190,12 @@ class textEditView(QTextEdit):
if self._highlighting: if self._highlighting:
item = index.internalPointer() item = index.internalPointer()
if self._column in [Outline.text.value, Outline.notes.value]: if self._column in [Outline.text.value, Outline.notes.value]:
self.highlighter = MMDHighlighter(self) self.highlighter = MarkdownHighlighter(self)
else: else:
self.highlighter = BasicHighlighter(self) self.highlighter = BasicHighlighter(self)
self.highlighter.setDefaultBlockFormat(self._defaultBlockFormat) self.highlighter.setDefaultBlockFormat(self._defaultBlockFormat)
self.highlighter.updateColorScheme()
def loadFontSettings(self): def loadFontSettings(self):
if self._fromTheme or \ if self._fromTheme or \
@ -204,8 +207,10 @@ class textEditView(QTextEdit):
opt = settings.textEditor opt = settings.textEditor
f = QFont() f = QFont()
f.fromString(opt["font"]) f.fromString(opt["font"])
background = opt["background"] if not opt["backgroundTransparent"] else "transparent" background = (opt["background"] if not opt["backgroundTransparent"]
foreground = opt["fontColor"] if not opt["backgroundTransparent"] else S.text else "transparent")
foreground = opt["fontColor"] # if not opt["backgroundTransparent"]
# else S.text
# self.setFont(f) # self.setFont(f)
self.setStyleSheet("""QTextEdit{{ self.setStyleSheet("""QTextEdit{{
background: {bg}; background: {bg};
@ -261,6 +266,7 @@ class textEditView(QTextEdit):
self._defaultBlockFormat = bf self._defaultBlockFormat = bf
if self.highlighter: if self.highlighter:
self.highlighter.updateColorScheme()
self.highlighter.setMisspelledColor(QColor(opt["misspelled"])) self.highlighter.setMisspelledColor(QColor(opt["misspelled"]))
self.highlighter.setDefaultCharFormat(self._defaultCharFormat) self.highlighter.setDefaultCharFormat(self._defaultCharFormat)
self.highlighter.setDefaultBlockFormat(self._defaultBlockFormat) self.highlighter.setDefaultBlockFormat(self._defaultBlockFormat)