Adds: frequency analyzer

This commit is contained in:
Olivier Keshavjee 2016-02-08 13:50:35 +01:00
parent eb1e6c7fcc
commit 308fe5e623
8 changed files with 479 additions and 15 deletions

View file

@ -23,6 +23,7 @@ from manuskript.ui.collapsibleDockWidgets import collapsibleDockWidgets
from manuskript.ui.compileDialog import compileDialog
from manuskript.ui.helpLabel import helpLabel
from manuskript.ui.mainWindow import Ui_MainWindow
from manuskript.ui.tools.frequencyAnalyzer import frequencyAnalyzer
from manuskript.ui.views.outlineDelegates import outlinePersoDelegate
from manuskript.ui.views.plotDelegate import plotDelegate
@ -108,6 +109,7 @@ class MainWindow(QMainWindow, Ui_MainWindow):
self.actSettings.triggered.connect(self.settingsWindow)
self.actCloseProject.triggered.connect(self.closeProject)
self.actQuit.triggered.connect(self.close)
self.actToolFrequency.triggered.connect(self.frequencyAnalyzer)
self.generateViewMenu()
self.makeUIConnections()
@ -969,6 +971,14 @@ class MainWindow(QMainWindow, Ui_MainWindow):
self.sw.setTab(tab)
self.sw.show()
###############################################################################
# TOOLS
###############################################################################
def frequencyAnalyzer(self):
self.fw = frequencyAnalyzer(self)
self.fw.show()
###############################################################################
# VIEW MENU
###############################################################################

View file

@ -453,6 +453,12 @@ class outlineItem():
def childCount(self):
return len(self.childItems)
def childCountRecursive(self):
n = self.childCount()
for c in self.children():
n += c.childCountRecursive()
return n
def children(self):
return self.childItems

View file

@ -73,12 +73,19 @@ revisions = {
None: 60 * 60 * 24 * 7, # One per week for eternity
})
}
frequencyAnalyzer = {
"wordMin": 1,
"wordExclude": "a, and, or",
"phraseMin": 2,
"phraseMax": 5
}
def save(filename=None):
global spellcheck, dict, corkSliderFactor, viewSettings, corkSizeFactor, folderView, lastTab, openIndexes, \
autoSave, autoSaveDelay, saveOnQuit, autoSaveNoChanges, autoSaveNoChangesDelay, outlineViewColumns, \
corkBackground, fullScreenTheme, defaultTextType, textEditor, revisions
corkBackground, fullScreenTheme, defaultTextType, textEditor, revisions, frequencyAnalyzer
allSettings = {
"viewSettings": viewSettings,
@ -99,6 +106,7 @@ def save(filename=None):
"defaultTextType":defaultTextType,
"textEditor":textEditor,
"revisions":revisions,
"frequencyAnalyzer": frequencyAnalyzer
}
#pp=pprint.PrettyPrinter(indent=4, compact=False)
@ -201,4 +209,9 @@ def load(string, fromString=False):
if "revisions" in allSettings:
global revisions
revisions = allSettings["revisions"]
revisions = allSettings["revisions"]
if "frequencyAnalyzer" in allSettings:
global frequencyAnalyzer
frequencyAnalyzer = allSettings["frequencyAnalyzer"]

View file

@ -2,7 +2,7 @@
# Form implementation generated from reading ui file 'manuskript/ui/mainWindow.ui'
#
# Created: Mon Feb 8 09:02:57 2016
# Created: Mon Feb 8 11:20:10 2016
# by: PyQt5 UI code generator 5.2.1
#
# WARNING! All changes made in this file will be lost!
@ -1130,6 +1130,8 @@ class Ui_MainWindow(object):
self.actCloseProject.setObjectName("actCloseProject")
self.actCompile = QtWidgets.QAction(MainWindow)
self.actCompile.setObjectName("actCompile")
self.actToolFrequency = QtWidgets.QAction(MainWindow)
self.actToolFrequency.setObjectName("actToolFrequency")
self.menuFile.addAction(self.actOpen)
self.menuFile.addAction(self.menuRecents.menuAction())
self.menuFile.addAction(self.actSave)
@ -1144,6 +1146,7 @@ class Ui_MainWindow(object):
self.menuMode.addAction(self.actModeFractal)
self.menuHelp.addAction(self.actShowHelp)
self.menuTools.addAction(self.actSpellcheck)
self.menuTools.addAction(self.actToolFrequency)
self.menuEdit.addAction(self.actLabels)
self.menuEdit.addAction(self.actStatus)
self.menuEdit.addAction(self.actSettings)
@ -1291,18 +1294,19 @@ class Ui_MainWindow(object):
self.actCloseProject.setText(_translate("MainWindow", "&Close project"))
self.actCompile.setText(_translate("MainWindow", "Co&mpile"))
self.actCompile.setShortcut(_translate("MainWindow", "F6"))
self.actToolFrequency.setText(_translate("MainWindow", "Frequency Analyzer"))
from manuskript.ui.views.textEditView import textEditView
from manuskript.ui.views.persoTreeView import persoTreeView
from manuskript.ui.views.basicItemView import basicItemView
from manuskript.ui.cheatSheet import cheatSheet
from manuskript.ui.views.lineEditView import lineEditView
from manuskript.ui.welcome import welcome
from manuskript.ui.sldImportance import sldImportance
from manuskript.ui.views.outlineView import outlineView
from manuskript.ui.search import search
from manuskript.ui.views.textEditCompleter import textEditCompleter
from manuskript.ui.views.plotTreeView import plotTreeView
from manuskript.ui.views.treeView import treeView
from manuskript.ui.editors.mainEditor import mainEditor
from manuskript.ui.views.metadataView import metadataView
from manuskript.ui.views.basicItemView import basicItemView
from manuskript.ui.editors.mainEditor import mainEditor
from manuskript.ui.views.plotTreeView import plotTreeView
from manuskript.ui.sldImportance import sldImportance
from manuskript.ui.views.textEditCompleter import textEditCompleter
from manuskript.ui.views.treeView import treeView
from manuskript.ui.views.persoTreeView import persoTreeView
from manuskript.ui.search import search
from manuskript.ui.cheatSheet import cheatSheet
from manuskript.ui.views.outlineView import outlineView
from manuskript.ui.views.textEditView import textEditView
from manuskript.ui.views.lineEditView import lineEditView

View file

@ -2013,6 +2013,7 @@
<string>&amp;Tools</string>
</property>
<addaction name="actSpellcheck"/>
<addaction name="actToolFrequency"/>
</widget>
<widget class="QMenu" name="menuEdit">
<property name="title">
@ -2320,6 +2321,11 @@ QListView::item:hover {
<string>F6</string>
</property>
</action>
<action name="actToolFrequency">
<property name="text">
<string>Frequency Analyzer</string>
</property>
</action>
</widget>
<customwidgets>
<customwidget>

View file

@ -0,0 +1,133 @@
#!/usr/bin/env python
# --!-- coding: utf8 --!--
from PyQt5.QtCore import QVariant, Qt
from PyQt5.QtGui import QStandardItemModel, QStandardItem
from PyQt5.QtWidgets import QWidget, QTextEdit, QTableWidgetItem, QHeaderView
from manuskript import settings
from manuskript.enums import Outline
from manuskript.ui.tools.frequency_ui import Ui_FrequencyAnalyzer
import re
from collections import Counter
class frequencyAnalyzer(QWidget, Ui_FrequencyAnalyzer):
def __init__(self, mainWindow):
QWidget.__init__(self)
self.setupUi(self)
self.mw = mainWindow
self.splitter.setSizes([10, 100])
self.splitter.setStretchFactor(1, 10)
self.progressBarWord.hide()
self.tblWord.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
self.tblPhrase.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeToContents)
self.btnAnalyzeWord.clicked.connect(self.analyzeWord)
self.btnAnalyzePhrase.clicked.connect(self.analyzePhrase)
# Settings
self.spnWordMin.setValue(settings.frequencyAnalyzer["wordMin"])
self.txtWordExclude.setPlainText(settings.frequencyAnalyzer["wordExclude"])
self.spnPhraseMin.setValue(settings.frequencyAnalyzer["phraseMin"])
self.spnPhraseMax.setValue(settings.frequencyAnalyzer["phraseMax"])
self.spnWordMin.valueChanged.connect(self.updateSettings)
self.txtWordExclude.textChanged.connect(self.updateSettings)
self.spnPhraseMin.valueChanged.connect(self.updateSettings)
self.spnPhraseMax.valueChanged.connect(self.updateSettings)
def analyzePhrase(self):
root = self.mw.mdlOutline.rootItem
nMin = self.spnPhraseMin.value()
nMax = self.spnPhraseMax.value()
def listPhrases(item, nMin, nMax):
txt = item.text()
# Convert to plain text
if item.isHTML():
e = QTextEdit()
e.setHtml(item.text())
txt = e.toPlainText()
# Split into words
lst = re.findall(r"[\w']+", txt) # Ignores punctuation
# lst = re.findall(r"[\w']+|[.,!?;]", txt) # Includes punctuation
phrases = []
# Make tuples of n-length
for n in range(nMin, nMax + 1):
for l in range(len(lst) - n + 1):
phrases.append(tuple(lst[l:l+n]))
for c in item.children():
phrases += listPhrases(c, nMin, nMax)
return phrases
lst = listPhrases(root, nMin, nMax)
# Count
count = Counter(lst)
# Showing
mdl = QStandardItemModel()
mdl.setHorizontalHeaderLabels([self.tr("Phrases"), self.tr("Frequency")])
for i in count.most_common():
word = QStandardItem(" ".join(i[0]))
number = QStandardItem()
number.setData(i[1], Qt.DisplayRole)
if i[1] > 1:
mdl.appendRow([word, number])
self.tblPhrase.setModel(mdl)
def analyzeWord(self):
root = self.mw.mdlOutline.rootItem
exclude = self.txtWordExclude.toPlainText().split(",")
exclude = [e.strip().lower() for e in exclude]
def listWords(item):
txt = item.text()
# Convert to plain text
if item.isHTML():
e = QTextEdit()
e.setHtml(item.text())
txt = e.toPlainText()
lst = re.findall(r"[\w']+", txt)
for c in item.children():
lst += listWords(c)
return lst
lst = listWords(root)
lst2 = []
# Cleaning
for i in lst:
if len(i) >= self.spnWordMin.value() and not i.lower() in exclude:
lst2.append(i.lower())
# Count
count = Counter(lst2)
# Showing
mdl = QStandardItemModel()
mdl.setHorizontalHeaderLabels([self.tr("Word"), self.tr("Frequency")])
for i in count.most_common():
word = QStandardItem(i[0])
number = QStandardItem()
number.setData(i[1], Qt.DisplayRole)
mdl.appendRow([word, number])
self.tblWord.setModel(mdl)
def updateSettings(self):
settings.frequencyAnalyzer["wordMin"] = self.spnWordMin.value()
settings.frequencyAnalyzer["wordExclude"] = self.txtWordExclude.toPlainText()
settings.frequencyAnalyzer["phraseMin"] = self.spnPhraseMin.value()
settings.frequencyAnalyzer["phraseMax"] = self.spnPhraseMax.value()

View file

@ -0,0 +1,115 @@
# -*- coding: utf-8 -*-
# Form implementation generated from reading ui file 'manuskript/ui/tools/frequency_ui.ui'
#
# Created: Mon Feb 8 13:54:01 2016
# by: PyQt5 UI code generator 5.2.1
#
# WARNING! All changes made in this file will be lost!
from PyQt5 import QtCore, QtGui, QtWidgets
class Ui_FrequencyAnalyzer(object):
def setupUi(self, FrequencyAnalyzer):
FrequencyAnalyzer.setObjectName("FrequencyAnalyzer")
FrequencyAnalyzer.resize(733, 453)
self.horizontalLayout = QtWidgets.QHBoxLayout(FrequencyAnalyzer)
self.horizontalLayout.setObjectName("horizontalLayout")
self.tabWidget = QtWidgets.QTabWidget(FrequencyAnalyzer)
self.tabWidget.setObjectName("tabWidget")
self.tab = QtWidgets.QWidget()
self.tab.setObjectName("tab")
self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.tab)
self.verticalLayout_2.setObjectName("verticalLayout_2")
self.splitter = QtWidgets.QSplitter(self.tab)
self.splitter.setOrientation(QtCore.Qt.Horizontal)
self.splitter.setObjectName("splitter")
self.widget = QtWidgets.QWidget(self.splitter)
self.widget.setObjectName("widget")
self.verticalLayout = QtWidgets.QVBoxLayout(self.widget)
self.verticalLayout.setContentsMargins(0, 0, 0, 0)
self.verticalLayout.setObjectName("verticalLayout")
self.groupBox = QtWidgets.QGroupBox(self.widget)
self.groupBox.setObjectName("groupBox")
self.formLayout = QtWidgets.QFormLayout(self.groupBox)
self.formLayout.setFieldGrowthPolicy(QtWidgets.QFormLayout.AllNonFixedFieldsGrow)
self.formLayout.setRowWrapPolicy(QtWidgets.QFormLayout.WrapLongRows)
self.formLayout.setObjectName("formLayout")
self.label = QtWidgets.QLabel(self.groupBox)
self.label.setObjectName("label")
self.formLayout.setWidget(0, QtWidgets.QFormLayout.LabelRole, self.label)
self.spnWordMin = QtWidgets.QSpinBox(self.groupBox)
self.spnWordMin.setProperty("value", 1)
self.spnWordMin.setObjectName("spnWordMin")
self.formLayout.setWidget(0, QtWidgets.QFormLayout.FieldRole, self.spnWordMin)
self.label_2 = QtWidgets.QLabel(self.groupBox)
self.label_2.setObjectName("label_2")
self.formLayout.setWidget(1, QtWidgets.QFormLayout.LabelRole, self.label_2)
self.txtWordExclude = QtWidgets.QPlainTextEdit(self.groupBox)
self.txtWordExclude.setPlainText("")
self.txtWordExclude.setObjectName("txtWordExclude")
self.formLayout.setWidget(1, QtWidgets.QFormLayout.FieldRole, self.txtWordExclude)
self.verticalLayout.addWidget(self.groupBox)
self.progressBarWord = QtWidgets.QProgressBar(self.widget)
self.progressBarWord.setProperty("value", 0)
self.progressBarWord.setObjectName("progressBarWord")
self.verticalLayout.addWidget(self.progressBarWord)
self.btnAnalyzeWord = QtWidgets.QPushButton(self.widget)
self.btnAnalyzeWord.setObjectName("btnAnalyzeWord")
self.verticalLayout.addWidget(self.btnAnalyzeWord)
self.tblWord = QtWidgets.QTableView(self.splitter)
self.tblWord.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)
self.tblWord.setSortingEnabled(True)
self.tblWord.setObjectName("tblWord")
self.verticalLayout_2.addWidget(self.splitter)
self.tabWidget.addTab(self.tab, "")
self.tab_2 = QtWidgets.QWidget()
self.tab_2.setObjectName("tab_2")
self.verticalLayout_3 = QtWidgets.QVBoxLayout(self.tab_2)
self.verticalLayout_3.setObjectName("verticalLayout_3")
self.horizontalLayout_2 = QtWidgets.QHBoxLayout()
self.horizontalLayout_2.setObjectName("horizontalLayout_2")
self.label_3 = QtWidgets.QLabel(self.tab_2)
self.label_3.setObjectName("label_3")
self.horizontalLayout_2.addWidget(self.label_3)
self.spnPhraseMin = QtWidgets.QSpinBox(self.tab_2)
self.spnPhraseMin.setProperty("value", 2)
self.spnPhraseMin.setObjectName("spnPhraseMin")
self.horizontalLayout_2.addWidget(self.spnPhraseMin)
self.label_4 = QtWidgets.QLabel(self.tab_2)
self.label_4.setObjectName("label_4")
self.horizontalLayout_2.addWidget(self.label_4)
self.spnPhraseMax = QtWidgets.QSpinBox(self.tab_2)
self.spnPhraseMax.setProperty("value", 5)
self.spnPhraseMax.setObjectName("spnPhraseMax")
self.horizontalLayout_2.addWidget(self.spnPhraseMax)
spacerItem = QtWidgets.QSpacerItem(40, 20, QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Minimum)
self.horizontalLayout_2.addItem(spacerItem)
self.btnAnalyzePhrase = QtWidgets.QPushButton(self.tab_2)
self.btnAnalyzePhrase.setObjectName("btnAnalyzePhrase")
self.horizontalLayout_2.addWidget(self.btnAnalyzePhrase)
self.verticalLayout_3.addLayout(self.horizontalLayout_2)
self.tblPhrase = QtWidgets.QTableView(self.tab_2)
self.tblPhrase.setSortingEnabled(True)
self.tblPhrase.setObjectName("tblPhrase")
self.verticalLayout_3.addWidget(self.tblPhrase)
self.tabWidget.addTab(self.tab_2, "")
self.horizontalLayout.addWidget(self.tabWidget)
self.retranslateUi(FrequencyAnalyzer)
self.tabWidget.setCurrentIndex(1)
QtCore.QMetaObject.connectSlotsByName(FrequencyAnalyzer)
def retranslateUi(self, FrequencyAnalyzer):
_translate = QtCore.QCoreApplication.translate
FrequencyAnalyzer.setWindowTitle(_translate("FrequencyAnalyzer", "Frequency Analyzer"))
self.groupBox.setTitle(_translate("FrequencyAnalyzer", "Settings"))
self.label.setText(_translate("FrequencyAnalyzer", "Minimum size:"))
self.label_2.setText(_translate("FrequencyAnalyzer", "Exclude words (coma seperated):"))
self.btnAnalyzeWord.setText(_translate("FrequencyAnalyzer", "Analyze"))
self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab), _translate("FrequencyAnalyzer", "Word frequency"))
self.label_3.setText(_translate("FrequencyAnalyzer", "Number of words: from"))
self.label_4.setText(_translate("FrequencyAnalyzer", "to"))
self.btnAnalyzePhrase.setText(_translate("FrequencyAnalyzer", "Analyze"))
self.tabWidget.setTabText(self.tabWidget.indexOf(self.tab_2), _translate("FrequencyAnalyzer", "Phrase frequency"))

View file

@ -0,0 +1,177 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>FrequencyAnalyzer</class>
<widget class="QWidget" name="FrequencyAnalyzer">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>733</width>
<height>453</height>
</rect>
</property>
<property name="windowTitle">
<string>Frequency Analyzer</string>
</property>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QTabWidget" name="tabWidget">
<property name="currentIndex">
<number>1</number>
</property>
<widget class="QWidget" name="tab">
<attribute name="title">
<string>Word frequency</string>
</attribute>
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<widget class="QSplitter" name="splitter">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<widget class="QWidget" name="">
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Settings</string>
</property>
<layout class="QFormLayout" name="formLayout">
<property name="fieldGrowthPolicy">
<enum>QFormLayout::AllNonFixedFieldsGrow</enum>
</property>
<property name="rowWrapPolicy">
<enum>QFormLayout::WrapLongRows</enum>
</property>
<item row="0" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>Minimum size:</string>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="QSpinBox" name="spnWordMin">
<property name="value">
<number>1</number>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>Exclude words (coma seperated):</string>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QPlainTextEdit" name="txtWordExclude">
<property name="plainText">
<string/>
</property>
</widget>
</item>
</layout>
</widget>
</item>
<item>
<widget class="QProgressBar" name="progressBarWord">
<property name="value">
<number>0</number>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="btnAnalyzeWord">
<property name="text">
<string>Analyze</string>
</property>
</widget>
</item>
</layout>
</widget>
<widget class="QTableView" name="tblWord">
<property name="editTriggers">
<set>QAbstractItemView::NoEditTriggers</set>
</property>
<property name="sortingEnabled">
<bool>true</bool>
</property>
</widget>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab_2">
<attribute name="title">
<string>Phrase frequency</string>
</attribute>
<layout class="QVBoxLayout" name="verticalLayout_3">
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
<widget class="QLabel" name="label_3">
<property name="text">
<string>Number of words: from</string>
</property>
</widget>
</item>
<item>
<widget class="QSpinBox" name="spnPhraseMin">
<property name="value">
<number>2</number>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="label_4">
<property name="text">
<string>to</string>
</property>
</widget>
</item>
<item>
<widget class="QSpinBox" name="spnPhraseMax">
<property name="value">
<number>5</number>
</property>
</widget>
</item>
<item>
<spacer name="horizontalSpacer">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>40</width>
<height>20</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QPushButton" name="btnAnalyzePhrase">
<property name="text">
<string>Analyze</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QTableView" name="tblPhrase">
<property name="sortingEnabled">
<bool>true</bool>
</property>
</widget>
</item>
</layout>
</widget>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections/>
</ui>