CodeMirror/fimfic2epub
1
0
Fork 0
mirror of https://github.com/daniel-j/fimfic2epub.git synced 2024-06-28 11:00:30 +12:00
Code Issues Projects Releases Wiki Activity

promisify! wip

Browse source
This commit is contained in:
Daniel J 2016-08-23 09:19:01 +02:00
parent d1d78bc777
commit 1bbd6316a5
7 changed files with 349 additions and 330 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

200
src/FimFic2Epub.js
View file

@ -11,6 +11,7 @@ import fileType from 'file-type'
import { styleCss, coverstyleCss, titlestyleCss } from './styles'
import { cleanMarkup } from './cleanMarkup'
import fetch from './fetch'
import fetchRemote from './fetchRemote'
import * as template from './templates'
@ -40,8 +41,8 @@ module.exports = class FimFic2Epub {
static fetchStoryInfo (storyId, raw = false) {
return new Promise((resolve, reject) => {
storyId = FimFic2Epub.getStoryId(storyId)
let url = 'https://www.fimfiction.net/api/story.php?story=' + storyId
fetchRemote(url, (content, type) => {
let url = '/api/story.php?story=' + storyId
fetch(url).then((content) => {
let data
try {
data = JSON.parse(content)
@ -76,32 +77,61 @@ module.exports = class FimFic2Epub {
})
}
static parseChapterPage (html) {
let trimWhitespace = /^\s*(<br\s*\/?\s*>)+|(<br\s*\/?\s*>)+\s*$/ig
let authorNotesPos = html.indexOf('<div class="authors-note"')
let authorNotes = ''
if (authorNotesPos !== -1) {
authorNotesPos = authorNotesPos + html.substring(authorNotesPos).indexOf('<b>Author\'s Note:</b>')
authorNotes = html.substring(authorNotesPos + 22)
authorNotes = authorNotes.substring(0, authorNotes.indexOf('\t\n\t</div>'))
authorNotes = authorNotes.trim()
authorNotes = authorNotes.replace(trimWhitespace, '')
}
let chapterPos = html.indexOf('<div id="chapter_container">')
let chapter = html.substring(chapterPos + 29)
let pos = chapter.indexOf('\t</div>\t\t\n\t')
chapter = chapter.substring(0, pos).trim()
// remove leading and trailing <br /> tags and whitespace
chapter = chapter.replace(trimWhitespace, '')
return {content: chapter, notes: authorNotes, notesFirst: authorNotesPos < chapterPos}
}
constructor (storyId) {
this.storyId = FimFic2Epub.getStoryId(storyId)
this.hasDownloaded = false
this.downloadPromise = null
this.fetchPromise = null
this.storyInfo = null
this.titlePage = null
this.chapters = []
this.chapterContent = []
this.remoteResources = new Map()
this.cachedFile = null
this.hasCoverImage = false
this.coverImageDimensions = {width: 0, height: 0}
this.includeTitlePage = true
this.categories = []
this.tags = []
// this.includeTitlePage = true
// this.categories = []
// this.tags = []
this.zip = new JSZip()
}
download () {
if (this.downloadPromise) {
return this.downloadPromise
fetch () {
if (this.fetchPromise) {
return this.fetchPromise
}
this.storyInfo = null
this.chapters.length = 0
this.remoteResources.clear()
console.log('Fetching story metadata...')
let p = FimFic2Epub.fetchStoryInfo(this.storyId)
@ -112,31 +142,35 @@ module.exports = class FimFic2Epub {
this.filename = FimFic2Epub.getFilename(this.storyInfo)
})
.then(this.fetchTitlePage.bind(this))
.then(this.fetchChapters.bind(this))
.then(() => {
this.downloadPromise = null
console.log('Fetch complete')
console.log(this)
this.fetchPromise = null
})
this.downloadPromise = p
this.fetchPromise = p
return p
}
build () {
return this.extractTitlePageInfo(this.titlePage)
.then(this.checkCoverImage.bind(this))
this.chapterContent.length = 0
return this.checkCoverImage()
.then(this.processChapters.bind(this))
.then(this.fetchRemoteFiles.bind(this))
.then(this.processStory.bind(this))
.then(() => {
console.log('Build complete')
})
}
fetchTitlePage () {
console.log('Fetching index page...')
return new Promise((resolve, reject) => {
fetchRemote(this.storyInfo.url, (raw, type) => {
this.titlePage = raw
resolve(raw)
})
})
console.log('Fetching title page...')
let url = this.storyInfo.url.replace('http://www.fimfiction.net', '')
return fetch(url).then(this.extractTitlePageInfo.bind(this))
}
extractTitlePageInfo (html) {
return new Promise((resolve, reject) => {
let descPos = html.indexOf('<div class="description" id="description')
descPos = descPos + html.substring(descPos).indexOf('">') + 2
html = html.substring(descPos)
@ -202,11 +236,49 @@ module.exports = class FimFic2Epub {
}
this.tags = tags
cleanMarkup(description).then((html) => {
return cleanMarkup(description).then((html) => {
this.storyInfo.description = html
this.findRemoteResources('description', 'description', html)
resolve()
})
}
fetchChapters () {
console.log('Fetching chapters...')
return new Promise((resolve, reject) => {
let chapters = this.storyInfo.chapters
let chapterCount = this.storyInfo.chapters.length
let currentChapter = 0
let completeCount = 0
if (chapterCount === 0) {
resolve()
return
}
let recursive = () => {
let index = currentChapter++
let ch = chapters[index]
if (!ch) {
return
}
console.log('Fetching chapter ' + (index + 1) + ' of ' + chapters.length + ': ' + ch.title)
let url = ch.link.replace('http://www.fimfiction.net', '')
fetch(url).then((html) => {
this.chapters[index] = FimFic2Epub.parseChapterPage(html)
completeCount++
if (completeCount < chapterCount) {
recursive()
} else {
resolve()
}
})
}
// concurrent downloads!
recursive()
recursive()
recursive()
recursive()
})
}
@ -224,29 +296,35 @@ module.exports = class FimFic2Epub {
coverImage.addEventListener('load', () => {
this.coverImageDimensions.width = coverImage.width
this.coverImageDimensions.height = coverImage.height
this.processStory(resolve, reject)
resolve()
}, false)
coverImage.addEventListener('error', () => {
console.warn('Unable to fetch cover image, skipping...')
this.hasCoverImage = false
this.processStory(resolve, reject)
resolve()
})
} else {
this.processStory(resolve, reject)
resolve()
}
} else {
this.processStory(resolve, reject)
resolve()
}
})
}
processStory (resolve, reject) {
console.log('Fetching chapters...')
processChapters () {
let p = []
for (let i = 0; i < this.storyInfo.chapters.length; i++) {
let ch = this.storyInfo.chapters[i]
p.push(template.createChapter(ch, this.chapters[i]).then((html) => {
this.findRemoteResources('ch_' + zeroFill(3, i + 1), i, html)
this.chapterContent[i] = html
}))
}
return Promise.all(p)
}
this.fetchChapters().then(() => {
console.log('Fetching remote files...')
this.fetchRemoteFiles(() => {
processStory () {
console.log('Finishing build...')
this.zip.file('mimetype', 'application/epub+zip')
@ -303,12 +381,11 @@ module.exports = class FimFic2Epub {
}
this.hasDownloaded = true
resolve()
})
})
}
fetchRemoteFiles (cb) {
fetchRemoteFiles () {
return new Promise((resolve, reject) => {
console.log('Fetching remote files...')
let iter = this.remoteResources.entries()
let count = 0
let completeCount = 0
@ -317,7 +394,7 @@ module.exports = class FimFic2Epub {
let r = iter.next().value
if (!r) {
if (completeCount === this.remoteResources.size) {
cb()
resolve()
}
return
}
@ -327,11 +404,11 @@ module.exports = class FimFic2Epub {
console.log('Fetching remote file ' + (count + 1) + ' of ' + this.remoteResources.size + ': ' + r.filename, url)
count++
fetchRemote(url, (data, type) => {
fetchRemote(url).then((data) => {
r.dest = null
let info = fileType(isNode ? data : new Uint8Array(data))
if (info) {
type = info.mime
let type = info.mime
r.type = type
let isImage = type.indexOf('image/') === 0
let folder = isImage ? 'Images' : 'Misc'
@ -348,47 +425,6 @@ module.exports = class FimFic2Epub {
}, 'arraybuffer')
}
// concurrent downloads!
recursive()
recursive()
recursive()
recursive()
}
fetchChapters () {
return new Promise((resolve, reject) => {
let chapters = this.storyInfo.chapters
let chapterCount = this.storyInfo.chapters.length
let currentChapter = 0
let completeCount = 0
if (chapterCount === 0) {
resolve()
return
}
let recursive = () => {
let index = currentChapter++
let ch = chapters[index]
if (!ch) {
return
}
console.log('Fetching chapter ' + (index + 1) + ' of ' + chapters.length + ': ' + ch.title)
let url = ch.link.replace('http', 'https')
fetchRemote(url, (html) => {
template.createChapter(ch, html, (html) => {
this.findRemoteResources('ch_' + zeroFill(3, index + 1), index, html)
this.chapterContent[index] = html
completeCount++
if (completeCount < chapterCount) {
recursive()
} else {
resolve()
}
})
})
}
// concurrent downloads!
recursive()
recursive()

2
src/cleanMarkup.js
View file

@ -50,7 +50,7 @@ export function cleanMarkup (html) {
let completeCount = 0
function getYoutubeInfo (ids) {
fetch('https://www.googleapis.com/youtube/v3/videos?id=' + ids + '&part=snippet&maxResults=50&key=' + youtubeKey, (raw, type) => {
fetch('https://www.googleapis.com/youtube/v3/videos?id=' + ids + '&part=snippet&maxResults=50&key=' + youtubeKey).then((raw) => {
let data
try {
data = JSON.parse(raw).items

13
src/eventPage.js
View file

@ -5,12 +5,11 @@ import fetch from './fetch'
if (typeof safari !== 'undefined') {
safari.application.addEventListener('message', function (ev) {
let url = ev.message
fetch(url, (buffer, type) => {
console.log('Fetched ' + url + ' (' + type + ')')
fetch(url).then((buffer) => {
console.log('Fetched ' + url)
ev.target.page.dispatchMessage('remote', {
input: url,
output: buffer,
type: type
output: buffer
})
}, 'arraybuffer')
}, false)
@ -19,9 +18,9 @@ if (typeof safari !== 'undefined') {
onMessage.addListener(function (request, sender, sendResponse) {
if (typeof request === 'string') {
fetch(request, (blob, type) => {
sendResponse(URL.createObjectURL(blob), type)
}, 'blob')
fetch(request, 'blob').then((blob) => {
sendResponse(URL.createObjectURL(blob))
})
// required for async
return true
} else if (request.showPageAction) {

27
src/fetch.js
View file

@ -1,8 +1,12 @@
import isNode from 'detect-node'
function fetchNode (url, cb, responseType) {
function fetchNode (url, responseType) {
const request = require('request')
if (url.indexOf('/') === 0) {
url = 'http://www.fimfiction.net' + url
}
return new Promise((resolve, reject) => {
request({
url: url,
encoding: responseType ? null : 'utf8',
@ -12,36 +16,37 @@ function fetchNode (url, cb, responseType) {
}
}, (error, response, body) => {
if (error) {
console.error(error)
cb()
reject(error)
return
}
let type = response.headers['content-type']
cb(body, type)
resolve(body)
})
})
}
export default function fetch (url, cb, responseType) {
export default function fetch (url, responseType) {
if (url.indexOf('//') === 0) {
url = 'http:' + url
}
if (isNode) {
fetchNode(url, cb, responseType)
return
return fetchNode(url, responseType)
}
return new Promise((resolve, reject) => {
let x = new XMLHttpRequest()
x.open('get', url, true)
if (responseType) {
console.log(responseType)
x.responseType = responseType
}
x.onload = function () {
cb(x.response, x.getResponseHeader('content-type'))
// x.getResponseHeader('content-type')
resolve(x.response)
}
x.onerror = function () {
console.error('error')
cb(null)
reject('Error fetching ' + url)
}
x.send()
})
}

23
src/fetchRemote.js
View file

@ -39,33 +39,34 @@ if (typeof safari !== 'undefined') {
safari.self.addEventListener('message', safariHandler, false)
}
function fetchBackground (url, cb, responseType) {
function fetchBackground (url, responseType) {
return new Promise((resolve, reject) => {
if (typeof chrome !== 'undefined' && chrome.runtime.sendMessage) {
chrome.runtime.sendMessage(url, function (objurl) {
fetch(objurl, cb, responseType)
resolve(fetch(objurl, responseType))
URL.revokeObjectURL(objurl)
})
} else if (typeof safari !== 'undefined') {
safariQueue[url] = {cb: cb, responseType: responseType}
safariQueue[url] = {cb: resolve, responseType: responseType}
safari.self.tab.dispatchMessage('remote', url)
} else {
cb(null)
resolve(null)
}
})
}
export default function fetchRemote (url, cb, responseType) {
export default function fetchRemote (url, responseType) {
if (url.indexOf('//') === 0) {
url = 'http:' + url
}
if (!isNode && document.location.protocol === 'https:' && url.indexOf('http:') === 0) {
fetchBackground(url, cb, responseType)
return
return fetchBackground(url, responseType)
}
fetch(url, (data, type) => {
return fetch(url, responseType).then((data) => {
if (!data) {
fetchBackground(url, cb, responseType)
return fetchBackground(url, responseType)
} else {
cb(data, type)
return Promise.resolve(data)
}
}, responseType)
})
}

2
src/main.js
View file

@ -75,7 +75,7 @@ let dialog = {
}
this.createEpub = (e) => {
e.target.disabled = true
ffc.download()
ffc.fetch()
.then(ffc.build.bind(ffc))
.then(ffc.getFile.bind(ffc)).then((file) => {
console.log('Saving file...')

32
src/templates.js
View file

@ -24,43 +24,21 @@ function prettyDate (d) {
return d.getDate() + nth(d) + ' ' + months[d.getMonth()].substring(0, 3) + ' ' + d.getFullYear()
}
export function createChapter (ch, html, callback) {
let trimWhitespace = /^\s*(<br\s*\/?\s*>)+|(<br\s*\/?\s*>)+\s*$/ig
let authorNotesPos = html.indexOf('<div class="authors-note"')
let authorNotes = ''
if (authorNotesPos !== -1) {
authorNotesPos = authorNotesPos + html.substring(authorNotesPos).indexOf('<b>Author\'s Note:</b>')
authorNotes = html.substring(authorNotesPos + 22)
authorNotes = authorNotes.substring(0, authorNotes.indexOf('\t\n\t</div>'))
authorNotes = authorNotes.trim()
authorNotes = authorNotes.replace(trimWhitespace, '')
}
let chapterPos = html.indexOf('<div id="chapter_container">')
let chapter = html.substring(chapterPos + 29)
let pos = chapter.indexOf('\t</div>\t\t\n\t')
chapter = chapter.substring(0, pos).trim()
// remove leading and trailing <br /> tags and whitespace
chapter = chapter.replace(trimWhitespace, '')
Promise.all([cleanMarkup(chapter), cleanMarkup(authorNotes)]).then((values) => {
export function createChapter (ch, chapter) {
return Promise.all([cleanMarkup(chapter.content), cleanMarkup(chapter.notes)]).then((values) => {
let [cleanChapter, cleanAuthorNotes] = values
ch.realWordCount = htmlWordCount(cleanChapter)
let content = [
m.trust(cleanChapter),
cleanAuthorNotes ? m('div#author_notes', {className: authorNotesPos < chapterPos ? 'top' : 'bottom'}, [
cleanAuthorNotes ? m('div#author_notes', {className: chapter.notesFirst ? 'top' : 'bottom'}, [
m('p', m('b', 'Author\'s Note:')),
m.trust(cleanAuthorNotes)]) : null
]
// if author notes are a the beginning of the chapter
if (cleanAuthorNotes && authorNotesPos < chapterPos) {
if (cleanAuthorNotes && chapter.notesFirst) {
content.reverse()
}
@ -84,7 +62,7 @@ export function createChapter (ch, html, callback) {
])
))
callback(chapterPage)
return Promise.resolve(chapterPage)
})
}