diff --git a/package.json b/package.json index 50bddcf..b9577ac 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "mithril-node-render": "^2.2.0", "node-png": "^0.4.3", "pretty-data": "^0.40.0", + "reading-level": "0.0.7", "request": "^2.85.0", "sanitize-filename": "^1.6.0", "twemoji": "^2.5.0", diff --git a/src/FimFic2Epub.js b/src/FimFic2Epub.js index 558a357..feb908c 100644 --- a/src/FimFic2Epub.js +++ b/src/FimFic2Epub.js @@ -14,7 +14,6 @@ import sizeOf from 'image-size' import Emitter from 'es6-event-emitter' import { cleanMarkup } from './cleanMarkup' -import htmlWordCount from './html-wordcount' import fetch from './fetch' import fetchRemote from './fetchRemote' import * as template from './templates' @@ -98,7 +97,8 @@ class FimFic2Epub extends Emitter { addChapterHeadings: true, includeExternal: true, paragraphStyle: 'spaced', - joinSubjects: false + joinSubjects: false, + calculateReadingEase: false } this.options = Object.assign(this.defaultOptions, options) @@ -151,8 +151,8 @@ class FimFic2Epub extends Emitter { this.pcache.fetchAll = this.fetchMetadata() .then(this.fetchChapters.bind(this)) .then(this.fetchCoverImage.bind(this)) - .then(this.buildPages.bind(this)) .then(this.buildChapters.bind(this)) + .then(this.buildPages.bind(this)) .then(this.findIcons.bind(this)) .then(this.fetchRemoteFiles.bind(this)) .then(() => { @@ -253,9 +253,7 @@ class FimFic2Epub extends Emitter { this.chaptersWithNotes.push(i) } this.chapters[i] = chapter - let ch = this.storyInfo.chapters[i] - ch.realWordCount = htmlWordCount(chapter.content) - }) + }).then(() => new Promise((resolve, reject) => setTimeout(resolve, 20))) } return p }).then(() => { @@ -385,6 +383,15 @@ class FimFic2Epub extends Emitter { this.notesHtml[i] = html }) } + chain = chain.then(() => { + if (!ch.realWordCount) { + ch.realWordCount = utils.htmlWordCount(chapter.content) + } + if (this.options.calculateReadingEase && !ch.readingEase) { + ch.readingEase = utils.readingEase(utils.htmlToText(chapter.content)) + } + this.progress(0, (i + 1) / this.chapters.length, 'Processed chapter ' + (i + 1) + ' / ' + this.chapters.length) + }).then(() => new Promise((resolve) => setTimeout(resolve, 20))) } return chain diff --git a/src/html-wordcount.js b/src/html-wordcount.js deleted file mode 100644 index 2f28c49..0000000 --- a/src/html-wordcount.js +++ /dev/null @@ -1,16 +0,0 @@ -import htmlToText from 'html-to-text' -import matchWords from 'match-words' - -export default function htmlWordCount (html) { - let text = htmlToText.fromString(html, { - wordwrap: false, - ignoreImage: true, - ignoreHref: true - }) - - let count = 0 - try { - count = matchWords(text).length - } catch (err) { count = 0 } - return count -} diff --git a/src/main.js b/src/main.js index 86fd875..7e3ae6a 100644 --- a/src/main.js +++ b/src/main.js @@ -6,7 +6,7 @@ import m from 'mithril' import prop from 'mithril/stream' import { saveAs } from 'file-saver' import autosize from 'autosize' -import htmlToText from 'html-to-text' +import { htmlToText } from './utils' function blobToDataURL (blob) { return new Promise((resolve, reject) => { @@ -136,6 +136,7 @@ let dialog = { this.includeExternal = prop(ffc.options.includeExternal) this.joinSubjects = prop(ffc.options.joinSubjects) this.paragraphStyle = prop(ffc.options.paragraphStyle) + this.calculateReadingEase = prop(ffc.options.calculateReadingEase) this.onOpen = (vnode) => { this.el(vnode.dom) @@ -146,11 +147,7 @@ let dialog = { ffcProgress(-1) this.title(ffc.storyInfo.title) this.author(ffc.storyInfo.author.name) - this.description(htmlToText.fromString(ffc.storyInfo.description, { - wordwrap: false, - ignoreImage: true, - ignoreHref: true - }) || ffc.storyInfo.short_description) + this.description(htmlToText(ffc.storyInfo.description) || ffc.storyInfo.short_description) this.subjects(ffc.subjects.slice(0)) redraw(true) this.center() @@ -257,6 +254,7 @@ let dialog = { m(checkbox, {checked: ctrl.addCommentsLink(), onchange: m.withAttr('checked', ctrl.addCommentsLink)}, 'Add link to online comments (at the end of chapters)'), m(checkbox, {checked: ctrl.includeAuthorNotes(), onchange: m.withAttr('checked', ctrl.includeAuthorNotes)}, 'Include author\'s notes'), m(checkbox, {checked: ctrl.useAuthorNotesIndex(), onchange: m.withAttr('checked', ctrl.useAuthorNotesIndex), disabled: !ctrl.includeAuthorNotes()}, 'Put all notes at the end of the ebook'), + m(checkbox, {checked: ctrl.calculateReadingEase(), onchange: m.withAttr('checked', ctrl.calculateReadingEase)}, 'Calculate Flesch reading ease'), m(checkbox, {checked: ctrl.includeExternal(), onchange: m.withAttr('checked', ctrl.includeExternal)}, 'Download & include remote content (embed images)'), m('div', {style: 'font-size: 0.9em; line-height: 1em; margin-top: 4px; margin-bottom: 6px; color: #777;'}, 'Note: Disabling this creates invalid EPUBs and requires internet access to see remote content. Only cover image will be embedded.') )), @@ -321,6 +319,7 @@ function createEpub (model) { ffc.options.paragraphStyle = model.paragraphStyle() ffc.subjects = model.subjects() ffc.options.joinSubjects = model.joinSubjects() + ffc.options.calculateReadingEase = model.calculateReadingEase() redraw() chain diff --git a/src/templates.js b/src/templates.js index 83f64ff..2d56c54 100644 --- a/src/templates.js +++ b/src/templates.js @@ -346,6 +346,15 @@ function calcWordCount (chapters) { } return count } +function calcReadingEase (chapters) { + let avg = 0 + for (let i = 0; i < chapters.length; i++) { + let ch = chapters[i] + avg += ch.readingEase.ease + } + avg = avg / chapters.length + return Math.round(avg * 100) / 100 +} export function createTitlePage (ffc) { const tokenContent = '%%HTML_CONTENT_' + Math.random() + '%%' @@ -392,7 +401,8 @@ export function createTitlePage (ffc) { ]), ffc.storyInfo.publishDate && infoBox('First Published', prettyDate(new Date(ffc.storyInfo.publishDate * 1000))), infoBox('Last Modified', prettyDate(new Date(ffc.storyInfo.date_modified * 1000))), - infoBox('Word Count', calcWordCount(ffc.storyInfo.chapters).toLocaleString('en-GB')) + infoBox('Word Count', calcWordCount(ffc.storyInfo.chapters).toLocaleString('en-GB')), + ffc.options.calculateReadingEase ? infoBox('Reading Ease', calcReadingEase(ffc.storyInfo.chapters).toLocaleString('en-GB')) : null ]), // m('hr'), m('.tags', [ diff --git a/src/utils.js b/src/utils.js index cb0214f..2c9afe5 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,4 +1,8 @@ +import htmlToTextModule from 'html-to-text' +import matchWords from 'match-words' +import { readingLevel } from 'reading-level' + export function replaceAsync (str, re, callback) { // http://es5.github.io/#x15.5.4.11 str = String(str) @@ -59,3 +63,27 @@ export function webp2png (data) { png.pack(decodedData, width[0], height[0]) }) } + +export function htmlToText (html) { + return htmlToTextModule.fromString(html, { + wordwrap: false, + ignoreImage: true, + ignoreHref: true + }) +} + +export function htmlWordCount (html) { + let text = htmlToText(html) + + let count = 0 + try { + count = matchWords(text).length + } catch (err) { count = 0 } + return count +} + +export function readingEase (text) { + const result = readingLevel(text, 'full') + const ease = 206.835 - 1.015 * (result.words / result.sentences) - 84.6 * (result.syllables / result.words) + return {ease, gradeLevel: result.unrounded} +}