Add reading ease calculation option

package.json

@@ -35,6 +35,7 @@
     "mithril-node-render": "^2.2.0",
     "node-png": "^0.4.3",
     "pretty-data": "^0.40.0",
+    "reading-level": "0.0.7",
     "request": "^2.85.0",
     "sanitize-filename": "^1.6.0",
     "twemoji": "^2.5.0",

src/FimFic2Epub.js

@@ -14,7 +14,6 @@ import sizeOf from 'image-size'
 import Emitter from 'es6-event-emitter'
 
 import { cleanMarkup } from './cleanMarkup'
-import htmlWordCount from './html-wordcount'
 import fetch from './fetch'
 import fetchRemote from './fetchRemote'
 import * as template from './templates'
@@ -98,7 +97,8 @@ class FimFic2Epub extends Emitter {
       addChapterHeadings: true,
       includeExternal: true,
       paragraphStyle: 'spaced',
-      joinSubjects: false
+      joinSubjects: false,
+      calculateReadingEase: false
     }
 
     this.options = Object.assign(this.defaultOptions, options)
@@ -151,8 +151,8 @@ class FimFic2Epub extends Emitter {
     this.pcache.fetchAll = this.fetchMetadata()
       .then(this.fetchChapters.bind(this))
       .then(this.fetchCoverImage.bind(this))
-      .then(this.buildPages.bind(this))
       .then(this.buildChapters.bind(this))
+      .then(this.buildPages.bind(this))
       .then(this.findIcons.bind(this))
       .then(this.fetchRemoteFiles.bind(this))
       .then(() => {
@@ -253,9 +253,7 @@ class FimFic2Epub extends Emitter {
             this.chaptersWithNotes.push(i)
           }
           this.chapters[i] = chapter
-          let ch = this.storyInfo.chapters[i]
-          ch.realWordCount = htmlWordCount(chapter.content)
-        })
+        }).then(() => new Promise((resolve, reject) => setTimeout(resolve, 20)))
       }
       return p
     }).then(() => {
@@ -385,6 +383,15 @@ class FimFic2Epub extends Emitter {
           this.notesHtml[i] = html
         })
       }
+      chain = chain.then(() => {
+        if (!ch.realWordCount) {
+          ch.realWordCount = utils.htmlWordCount(chapter.content)
+        }
+        if (this.options.calculateReadingEase && !ch.readingEase) {
+          ch.readingEase = utils.readingEase(utils.htmlToText(chapter.content))
+        }
+        this.progress(0, (i + 1) / this.chapters.length, 'Processed chapter ' + (i + 1) + ' / ' + this.chapters.length)
+      }).then(() => new Promise((resolve) => setTimeout(resolve, 20)))
     }
 
     return chain

src/html-wordcount.js

@@ -1,16 +0,0 @@
-import htmlToText from 'html-to-text'
-import matchWords from 'match-words'
-
-export default function htmlWordCount (html) {
-  let text = htmlToText.fromString(html, {
-    wordwrap: false,
-    ignoreImage: true,
-    ignoreHref: true
-  })
-
-  let count = 0
-  try {
-    count = matchWords(text).length
-  } catch (err) { count = 0 }
-  return count
-}

src/main.js

@@ -6,7 +6,7 @@ import m from 'mithril'
 import prop from 'mithril/stream'
 import { saveAs } from 'file-saver'
 import autosize from 'autosize'
-import htmlToText from 'html-to-text'
+import { htmlToText } from './utils'
 
 function blobToDataURL (blob) {
   return new Promise((resolve, reject) => {
@@ -136,6 +136,7 @@ let dialog = {
     this.includeExternal = prop(ffc.options.includeExternal)
     this.joinSubjects = prop(ffc.options.joinSubjects)
     this.paragraphStyle = prop(ffc.options.paragraphStyle)
+    this.calculateReadingEase = prop(ffc.options.calculateReadingEase)
 
     this.onOpen = (vnode) => {
       this.el(vnode.dom)
@@ -146,11 +147,7 @@ let dialog = {
         ffcProgress(-1)
         this.title(ffc.storyInfo.title)
         this.author(ffc.storyInfo.author.name)
-        this.description(htmlToText.fromString(ffc.storyInfo.description, {
-          wordwrap: false,
-          ignoreImage: true,
-          ignoreHref: true
-        }) || ffc.storyInfo.short_description)
+        this.description(htmlToText(ffc.storyInfo.description) || ffc.storyInfo.short_description)
         this.subjects(ffc.subjects.slice(0))
         redraw(true)
         this.center()
@@ -257,6 +254,7 @@ let dialog = {
             m(checkbox, {checked: ctrl.addCommentsLink(), onchange: m.withAttr('checked', ctrl.addCommentsLink)}, 'Add link to online comments (at the end of chapters)'),
             m(checkbox, {checked: ctrl.includeAuthorNotes(), onchange: m.withAttr('checked', ctrl.includeAuthorNotes)}, 'Include author\'s notes'),
             m(checkbox, {checked: ctrl.useAuthorNotesIndex(), onchange: m.withAttr('checked', ctrl.useAuthorNotesIndex), disabled: !ctrl.includeAuthorNotes()}, 'Put all notes at the end of the ebook'),
+            m(checkbox, {checked: ctrl.calculateReadingEase(), onchange: m.withAttr('checked', ctrl.calculateReadingEase)}, 'Calculate Flesch reading ease'),
             m(checkbox, {checked: ctrl.includeExternal(), onchange: m.withAttr('checked', ctrl.includeExternal)}, 'Download & include remote content (embed images)'),
             m('div', {style: 'font-size: 0.9em; line-height: 1em; margin-top: 4px; margin-bottom: 6px; color: #777;'}, 'Note: Disabling this creates invalid EPUBs and requires internet access to see remote content. Only cover image will be embedded.')
           )),
@@ -321,6 +319,7 @@ function createEpub (model) {
   ffc.options.paragraphStyle = model.paragraphStyle()
   ffc.subjects = model.subjects()
   ffc.options.joinSubjects = model.joinSubjects()
+  ffc.options.calculateReadingEase = model.calculateReadingEase()
   redraw()
 
   chain

src/templates.js

@@ -346,6 +346,15 @@ function calcWordCount (chapters) {
   }
   return count
 }
+function calcReadingEase (chapters) {
+  let avg = 0
+  for (let i = 0; i < chapters.length; i++) {
+    let ch = chapters[i]
+    avg += ch.readingEase.ease
+  }
+  avg = avg / chapters.length
+  return Math.round(avg * 100) / 100
+}
 
 export function createTitlePage (ffc) {
   const tokenContent = '%%HTML_CONTENT_' + Math.random() + '%%'
@@ -392,7 +401,8 @@ export function createTitlePage (ffc) {
           ]),
           ffc.storyInfo.publishDate && infoBox('First Published', prettyDate(new Date(ffc.storyInfo.publishDate * 1000))),
           infoBox('Last Modified', prettyDate(new Date(ffc.storyInfo.date_modified * 1000))),
-          infoBox('Word Count', calcWordCount(ffc.storyInfo.chapters).toLocaleString('en-GB'))
+          infoBox('Word Count', calcWordCount(ffc.storyInfo.chapters).toLocaleString('en-GB')),
+          ffc.options.calculateReadingEase ? infoBox('Reading Ease', calcReadingEase(ffc.storyInfo.chapters).toLocaleString('en-GB')) : null
         ]),
         // m('hr'),
         m('.tags', [

src/utils.js

@@ -1,4 +1,8 @@
 
+import htmlToTextModule from 'html-to-text'
+import matchWords from 'match-words'
+import { readingLevel } from 'reading-level'
+
 export function replaceAsync (str, re, callback) {
   // http://es5.github.io/#x15.5.4.11
   str = String(str)
@@ -59,3 +63,27 @@ export function webp2png (data) {
     png.pack(decodedData, width[0], height[0])
   })
 }
+
+export function htmlToText (html) {
+  return htmlToTextModule.fromString(html, {
+    wordwrap: false,
+    ignoreImage: true,
+    ignoreHref: true
+  })
+}
+
+export function htmlWordCount (html) {
+  let text = htmlToText(html)
+
+  let count = 0
+  try {
+    count = matchWords(text).length
+  } catch (err) { count = 0 }
+  return count
+}
+
+export function readingEase (text) {
+  const result = readingLevel(text, 'full')
+  const ease = 206.835 - 1.015 * (result.words / result.sentences) - 84.6 * (result.syllables / result.words)
+  return {ease, gradeLevel: result.unrounded}
+}