Fixes for Fimfiction 4.0 update. Drop Tidy, add paragraph customiztion

extension/manifest.json

@@ -4,7 +4,7 @@
 	"name": "fimfic2epub",
 	"short_name": "fimfic2epub",
 	"description": "Improved EPUB exporter for Fimfiction",
-	"version": "1.5.0",
+	"version": "1.6.0",
 
 	"icons": {
 		"128": "icon-128.png"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fimfic2epub",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Tool to generate improved EPUB ebooks from Fimfiction stories",
   "author": "djazz",
   "repository": {
@@ -32,7 +32,6 @@
     "pretty-data": "^0.40.0",
     "request": "^2.74.0",
     "sanitize-filename": "^1.6.0",
-    "tidy-html5": "^0.1.1",
     "zero-fill": "^2.2.3"
   },
   "devDependencies": {

src/FimFic2Epub.js

@@ -10,7 +10,7 @@ import fileType from 'file-type'
 import sizeOf from 'image-size'
 import Emitter from 'es6-event-emitter'
 
-import { styleCss, coverstyleCss, titlestyleCss } from './styles'
+import { styleCss, coverstyleCss, titlestyleCss, paragraphsCss } from './styles'
 
 import { cleanMarkup } from './cleanMarkup'
 import htmlWordCount from './html-wordcount'
@@ -91,7 +91,7 @@ class FimFic2Epub extends Emitter {
       useAuthorNotesIndex: false,
       addChapterHeadings: true,
       includeExternal: true,
-
+      paragraphStyle: 'spaced',
       joinSubjects: false
     }
 
@@ -386,7 +386,7 @@ class FimFic2Epub extends Emitter {
       }
     }
 
-    this.zip.file('OEBPS/Styles/style.css', styleCss)
+    this.zip.file('OEBPS/Styles/style.css', styleCss + '\n\n' + (paragraphsCss[this.options.paragraphStyle] || ''))
 
     this.remoteResources.forEach((r) => {
       if (r.dest) {
@@ -532,36 +532,49 @@ class FimFic2Epub extends Emitter {
   }
 
   extractTitlePageInfo (html) {
-    let descPos = html.indexOf('<div class="description" id="description')
+    let startTagsPos = html.indexOf('<div class="story_content_box"')
-    descPos = descPos + html.substring(descPos).indexOf('">') + 2
+    startTagsPos += html.substring(startTagsPos).indexOf('<ul class="story-tags">') + 23
-    html = html.substring(descPos)
+    let tagsHtml = html.substring(startTagsPos)
-    let ma = html.match(/<a href="(.*?)" class="source">Source<\/a>/)
+
-    this.storyInfo.source_image = null
+    let endTagsPos = tagsHtml.indexOf('</ul>')
-    if (ma) {
+    tagsHtml = tagsHtml.substring(0, endTagsPos)
-      this.storyInfo.source_image = ma[1]
-    }
-    let endCatsPos = html.indexOf('<hr />')
-    let startCatsPos = html.substring(0, endCatsPos).lastIndexOf('</div>')
-    let catsHtml = html.substring(startCatsPos, endCatsPos)
-    html = html.substring(endCatsPos + 6)
 
     let categories = []
+    let tags = []
+    tags.byImage = {}
     this.subjects.length = 0
     this.subjects.push('Fimfiction')
     this.subjects.push(this.storyInfo.content_rating_text)
-    let matchCategory = /<a href="(.*?)" class="(.*?)">(.*?)<\/a>/g
+    let matchCategory = /<a href="(.*?)" class="(.*?)".*? data-tag="(.*?)">(.*?)<\/a>/g
-    for (let c; (c = matchCategory.exec(catsHtml));) {
+    for (let c; (c = matchCategory.exec(tagsHtml));) {
-      let cat = {
+      if (c[2] === 'tag-genre') {
-        url: 'http://www.fimfiction.net' + c[1],
+        let cat = {
-        className: c[2],
+          url: 'http://www.fimfiction.net' + c[1],
-        name: entities.decode(c[3])
+          className: 'story_category story_category_' + c[3],
+          name: entities.decode(c[4])
+        }
+        categories.push(cat)
+        this.subjects.push(cat.name)
+      } else if (c[2] === 'tag-character') {
+        let t = {
+          url: 'http://www.fimfiction.net' + c[1],
+          // filename: 'tag-' + c[3],
+          name: entities.decode(c[4])
+          // image: 'https://static.fimfiction.net/images/characters/' + entities.decode(c[3]).replace(/-/g, '_') + '.png'
+        }
+        tags.push(t)
+        // tags.byImage[t.image] = t
+        // this.remoteResources.set(t.image, {filename: t.filename, originalUrl: t.image, where: ['tags']})
+      } else {
+        console.log(c)
       }
-      categories.push(cat)
-      this.subjects.push(cat.name)
     }
     this.categories = categories
 
-    ma = html.match(/This story is a sequel to <a href="([^"]*)">(.*?)<\/a>/)
+    html = html.substring(endTagsPos + 5)
+    html = html.substring(html.indexOf('<span class="description-text bbcode">') + 38)
+
+    let ma = html.match(/This story is a sequel to <a href="([^"]*)">(.*?)<\/a>/)
     if (ma) {
       this.storyInfo.prequel = {
         url: 'http://www.fimfiction.net' + ma[1],
@@ -569,7 +582,8 @@ class FimFic2Epub extends Emitter {
       }
       html = html.substring(html.indexOf('<hr />') + 6)
     }
-    let endDescPos = html.indexOf('</div>\n')
+
+    let endDescPos = html.indexOf('</span>\n')
     let description = html.substring(0, endDescPos).trim()
     this.description = description
 
@@ -586,8 +600,6 @@ class FimFic2Epub extends Emitter {
 
     html = html.substring(0, html.indexOf('<div class="button-group"'))
 
-    let tags = []
-    tags.byImage = {}
     let matchTag = /<a href="\/tag\/(.*?)" class="character_icon" title="(.*?)" style=".*?"><img src="(.*?)" class="character_icon" \/><\/a>/g
     for (let tag; (tag = matchTag.exec(html));) {
       let t = {
@@ -616,10 +628,10 @@ class FimFic2Epub extends Emitter {
       authorNotes = authorNotes.replace(trimWhitespace, '')
     }
 
-    let chapterPos = html.indexOf('<div id="chapter_container">')
+    let chapterPos = html.indexOf('<div class="bbcode">')
-    let chapter = html.substring(chapterPos + 29)
+    let chapter = html.substring(chapterPos + 20)
 
-    let pos = chapter.indexOf('\t</div>\t\t\n\t')
+    let pos = chapter.indexOf('\t\t</div>\n')
 
     chapter = chapter.substring(0, pos).trim()
 

src/cleanMarkup.js

@@ -3,16 +3,7 @@ import m from 'mithril'
 import render from './lib/mithril-node-render'
 
 import fetch from './fetch'
-import { tidyOptions, youtubeKey } from './constants'
+import { youtubeKey } from './constants'
-
-import isNode from 'detect-node'
-
-let tidy
-if (!isNode) {
-  tidy = require('exports-loader?tidy_html5!tidy-html5')
-} else {
-  tidy = require('tidy-html5').tidy_html5
-}
 
 export function cleanMarkup (html) {
   if (!html) {
@@ -20,11 +11,23 @@ export function cleanMarkup (html) {
   }
 
   return new Promise((resolve, reject) => {
+    // replace HTML non-breaking spaces with normal spaces
+    html = html.replace(/ /g, ' ')
+    html = html.replace(/ /g, ' ')
+
     html = fixParagraphIndent(html)
 
+    html = fixDoubleSpacing(html)
+
     // fix center tags
-    html = html.replace(/<center>/g, '<p style="text-align: center;">')
+    // html = html.replace(/<center>/g, '<p style="text-align: center;">')
-    html = html.replace(/<\/center>/g, '</p>')
+    // html = html.replace(/<\/center>/g, '</p>')
+
+    html = html.replace(/<p>\s*/g, '<p>')
+    html = html.replace(/\s*<\/p>/g, '</p>')
+
+    html = html.replace(/<p><p>/g, '<p>')
+    html = html.replace(/<\/div><\/p>/g, '</div>')
 
     // fix floating blockquote tags
     html = html.replace('<blockquote style="margin: 10px 0px; box-sizing:border-box; -moz-box-sizing:border-box;margin-right:25px; padding: 15px;background-color: #F7F7F7;border: 1px solid #AAA;width: 50%;float:left;box-shadow: 5px 5px 0px #EEE;">', '<blockquote class="left_insert">')
@@ -101,13 +104,7 @@ export function cleanMarkup (html) {
     }
 
     function continueParsing () {
-      html = tidy(html, tidyOptions).trim()
+      // html = tidy(html, tidyOptions).trim()
-
-      // replace HTML non-breaking spaces with normal spaces
-      html = html.replace(/&nbsp;/g, ' ')
-      html = html.replace(/&#160;/g, ' ')
-
-      html = fixDoubleSpacing(html)
 
       resolve(html)
     }

src/fetch.js

@@ -37,7 +37,7 @@ export default function fetch (url, responseType) {
       window.fetch(url, {
         method: 'GET',
         mode: 'cors',
-        credentials: 'omit',
+        credentials: 'include',
         cache: 'default',
         redirect: 'follow'
       }).then((response) => {

src/main.js

@@ -33,11 +33,11 @@ let logoUrl = chrome.extension.getURL('fimfic2epub-logo.png')
 
 let ffc
 
-let stories = document.querySelectorAll('.story_container .story_content_box')
+let stories = document.querySelectorAll('.story_container')
 
 stories.forEach((story) => {
-  let id = story.id.substring(6)
+  let id = story.dataset.story
-  let epubButton = story.querySelector('ul.chapters li.bottom a[title="Download Story (.epub)"]')
+  let epubButton = story.querySelector('.story-top-toolbar .button-group .drop-down ul li a[title="Download Story (.epub)"]')
   if (!epubButton) return
   epubButton.addEventListener('click', function (e) {
     e.preventDefault()
@@ -47,7 +47,7 @@ stories.forEach((story) => {
   logo.className = 'fimfic2epub-logo'
   logo.title = 'Download EPUB with fimfic2epub'
   logo.src = logoUrl
-  story.querySelector('.title').appendChild(logo)
+  story.querySelector('.story_content_box .title').appendChild(logo)
   logo.addEventListener('click', function (e) {
     e.preventDefault()
     openStory(id)
@@ -89,6 +89,15 @@ let checkbox = {
   }
 }
 
+function selectOptions (list, selected = '') {
+  return list.map((item) => {
+    return m('option', {
+      value: item[0],
+      selected: selected === item[0]
+    }, item[1])
+  })
+}
+
 let ffcProgress = m.prop(0)
 let ffcStatus = m.prop('')
 
@@ -117,6 +126,7 @@ let dialog = {
     this.addChapterHeadings = m.prop(ffc.options.addChapterHeadings)
     this.includeExternal = m.prop(ffc.options.includeExternal)
     this.joinSubjects = m.prop(ffc.options.joinSubjects)
+    this.paragraphStyle = m.prop(ffc.options.paragraphStyle)
 
     this.onOpen = function (el, isInitialized) {
       if (!isInitialized) {
@@ -219,6 +229,7 @@ let dialog = {
       ffc.options.useAuthorNotesIndex = this.useAuthorNotesIndex()
       ffc.options.addChapterHeadings = this.addChapterHeadings()
       ffc.options.includeExternal = this.includeExternal()
+      ffc.options.paragraphStyle = this.paragraphStyle()
       ffc.subjects = this.subjects()
       ffc.options.joinSubjects = this.joinSubjects()
       m.redraw()
@@ -240,7 +251,7 @@ let dialog = {
   },
 
   view (ctrl, args, extras) {
-    return m('.drop-down-pop-up-container', {config: ctrl.onOpen.bind(ctrl)}, m('.drop-down-pop-up', [
+    return m('.drop-down-pop-up-container', {config: ctrl.onOpen.bind(ctrl)}, m('.drop-down-pop-up', {style: {'min-width': '700px'}}, [
       m('h1', {onmousedown: ctrl.ondown}, m('i.fa.fa-book'), 'Export to EPUB', m('a.close_button', {onclick: closeDialog})),
       m('.drop-down-pop-up-content', [
         ctrl.isLoading() ? m('div', {style: 'text-align:center;'}, m('i.fa.fa-spin.fa-spinner', {style: 'font-size:50px; margin:20px; color:#777;'})) : m('table.properties', [
@@ -253,6 +264,14 @@ let dialog = {
             ),
             m('td', {style: 'width: 1px'}, m(checkbox, {checked: ctrl.checkboxCoverUrl(), onchange: m.withAttr('checked', ctrl.checkboxCoverUrl)}, 'Use image URL'))
           ),
+          m('tr', m('td.label', 'Paragraph style'), m('td', {colspan: 2},
+            m('select', {onchange: m.withAttr('value', ctrl.paragraphStyle)}, selectOptions([
+              ['indented', 'Indent first line in all paragraphs except the first (Traditional Paperback)'],
+              ['spaced', 'Separate each paragraph with double space (Traditional Web)'],
+              ['both', 'Double space and indent all paragraphs except first (Fusion)'],
+              ['indentedall', 'Indent all paragraphs including the first (Modified Traditional)']
+            ], ctrl.paragraphStyle()))
+          )),
           m('tr', m('td.label', ''), m('td', {colspan: 2},
             m(checkbox, {checked: ctrl.addChapterHeadings(), onchange: m.withAttr('checked', ctrl.addChapterHeadings)}, 'Add chapter headings'),
             m(checkbox, {checked: ctrl.addCommentsLink(), onchange: m.withAttr('checked', ctrl.addCommentsLink)}, 'Add link to online comments (at the end of chapters)'),

src/style/paragraphs-indented.styl

@@ -0,0 +1,8 @@
+
+p + p {
+  text-indent: 1em;
+}
+
+.chapter-title + p.indented {
+  text-indent: 0;
+}

src/style/paragraphs-indentedall.styl

@@ -0,0 +1,4 @@
+
+p {
+  text-indent: 1em;
+}

src/style/paragraphs-spaced.styl

@@ -0,0 +1,12 @@
+
+p {
+  margin-top: 1em;
+}
+
+.chapter-title + p {
+  margin-top: 0;
+}
+
+.bbcode-center, .bbcode-right {
+  margin-top: 1em;
+}

src/style/style.styl

@@ -8,26 +8,39 @@ body {
 }
 
 p {
-  margin-top: 0.0em;
+  margin-top: 0;
-  margin-bottom: 0.0em;
+  margin-bottom: 0;
-  text-indent: 0.0em;
+  text-indent: 0;
 
   &.double {
-    margin-top: 1.0em;
+    margin-top: 1em;
   }
   &.double2 {
-    margin-top: 1.0em;
+    margin-top: 1em;
-    margin-bottom: 1.0em;
+    margin-bottom: 1em;
   }
   &.indented {
-    text-indent: 1.0em;
+    text-indent: 1em;
   }
+  &:first-child {
+    margin-top: 0;
+  }
+}
+div p {
+  text-indent: 0;
 }
 
 h1, h2, h3, h4, h5, h6 {
   text-align: left;
 }
 
+em {
+  font-style: italic;
+}
+em em {
+  font-style: normal;
+}
+
 .chapter-title {
   margin-top: 1.5em;
   margin-bottom: 1.5em;
@@ -37,9 +50,6 @@ h1, h2, h3, h4, h5, h6 {
     font-weight: normal;
     margin: 0;
   }
-  & + p.indented {
-    text-indent: 0;
-  }
 }
 
 img {

src/style/titlestyle.styl

@@ -52,6 +52,7 @@
 .characters {
   margin-top: 0.5em;
   text-align: left;
+  font-size: .875em;
 }
 
 // Last Modified and First Published
@@ -172,6 +173,18 @@ span.character_icon {
   font-family: sans-serif;
 }
 
+.story_character {
+  display: inline-block;
+  padding: 0.3em 0.5em;
+  line-height: 1.0em;
+  bgcolor(#23b974);
+  textcolor(#fff);
+  text-decoration: none;
+  border-radius: 0.3em;
+  border: 1px solid #1e9d63;
+  font-family: sans-serif;
+}
+
 .story_category_sex {
   bgcolor(#992584);
   box-shadow: 0px 1px 0px #b82c9e inset;

src/styles.js

@@ -1,8 +1,15 @@
 
 let styleCss, coverstyleCss, titlestyleCss
 
-styleCss = require('./style')
+styleCss = require('./style/style')
-coverstyleCss = require('./coverstyle')
+coverstyleCss = require('./style/coverstyle')
-titlestyleCss = require('./titlestyle')
+titlestyleCss = require('./style/titlestyle')
+let paragraphsCss = {
+  spaced: require('./style/paragraphs-spaced'),
+  indented: require('./style/paragraphs-indented'),
+  indentedall: require('./style/paragraphs-indentedall')
+}
 
-export { styleCss, coverstyleCss, titlestyleCss }
+paragraphsCss.both = paragraphsCss.indented + '\n' + paragraphsCss.spaced
+
+export { styleCss, coverstyleCss, titlestyleCss, paragraphsCss }

src/templates.js

@@ -351,7 +351,8 @@ export function createTitlePage (ffc) {
         // m('hr'),
         m('.characters', [
           ffc.tags.map((t) =>
-            m('span', {className: 'character_icon', title: t.name}, m('img', {src: t.image, className: 'character_icon'}))
+            // m('span', {className: 'character_icon', title: t.name}, m('img', {src: t.image, className: 'character_icon'}))
+            m('span.story_character', t.name)
           )
         ])
       ])

webpack.config.babel.js

@@ -30,9 +30,6 @@ const bundleExtensionConfig = {
         test: /\.styl$/,
         use: ['raw-loader', 'stylus-loader']
       }
-    ],
-    noParse: [
-      /[/\\]node_modules[/\\]tidy-html5[/\\]tidy\.js$/
     ]
   },
 
@@ -47,7 +44,7 @@ const bundleExtensionConfig = {
     }
   },
 
-  externals: ['request', 'tidy-html5'],
+  externals: ['request'],
 
   plugins: [],
   devtool: 'source-map'
@@ -79,9 +76,6 @@ const bundleNpmModuleConfig = {
         test: /\.styl$/,
         use: ['raw-loader', 'stylus-loader']
       }
-    ],
-    noParse: [
-      /[/\\]node_modules[/\\]tidy-html5[/\\]tidy\.js$/
     ]
   },
 
@@ -93,7 +87,7 @@ const bundleNpmModuleConfig = {
     ]
   },
 
-  externals: [nodeExternals({whitelist: ['es6-event-emitter', /^babel-runtime/]}), 'exports?tidy_html5!tidy-html5'],
+  externals: [nodeExternals({whitelist: ['es6-event-emitter', /^babel-runtime/]})],
 
   plugins: [],
   devtool: 'source-map'