diff --git a/package-lock.json b/package-lock.json index 34dfc16..f69f18a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1604,7 +1604,8 @@ }, "acorn": { "version": "5.7.1", - "resolved": "" + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.1.tgz", + "integrity": "sha512-d+nbxBUGKg7Arpsvbnlq61mc12ek3EY8EQldM3GPAhWJ1UVxC6TDGbIvUMNU6obBX3i1+ptCIzV4vq0gFPEGVQ==" }, "acorn-globals": { "version": "4.3.4", @@ -1652,7 +1653,6 @@ "version": "6.12.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", - "dev": true, "requires": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", @@ -2377,8 +2377,7 @@ "chardet": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", - "dev": true + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, "cheerio": { "version": "0.22.0", @@ -4177,8 +4176,7 @@ "fast-deep-equal": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", - "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==", - "dev": true + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==" }, "fast-json-stable-stringify": { "version": "2.0.0", @@ -6881,6 +6879,114 @@ "mimic-fn": "^2.1.0" } }, + "open-graph-scraper": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/open-graph-scraper/-/open-graph-scraper-3.6.2.tgz", + "integrity": "sha512-4II493mZ5vzfDTuHc7OpVfmH58FtrmpTUo5KTyWN6XuProcD5TkXaZPyduOronZ5TPU5Q+eWKp2EXdX5U7ApxQ==", + "requires": { + "chardet": "0.7.0", + "cheerio": "1.0.0-rc.2", + "iconv-lite": "0.4.24", + "lodash": "4.17.14", + "request": "2.88.0" + }, + "dependencies": { + "aws4": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.9.1.tgz", + "integrity": "sha512-wMHVg2EOHaMRxbzgFJ9gtjOOCrI80OHLG14rxi28XwOW8ux6IiEbRCGGGqCtdAIg4FQCbW20k9RsT4y3gJlFug==" + }, + "cheerio": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", + "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, + "har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "requires": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "lodash": { + "version": "4.17.14", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz", + "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==" + }, + "oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" + }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + }, + "request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "requires": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } + } + } + }, "opencollective-postinstall": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz", diff --git a/package.json b/package.json index 63bfc0a..706567c 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,18 @@ { "name": "@frontender-magazine/builder", - "version": "1.0.0", + "version": "2.0.0", "description": "build an article from url", - "main": "index.js", + "main": "source/index.js", "scripts": { "precommit": "lint-staged", - "lint": "eslint --color -f stylish --fix ./source/linttest/*.jsx" + "lint": "eslint --color -f stylish --fix ./source/linttest/*.jsx", + "postversion": "git push && git push --tags", + "major": "npm version major && npm publish --tag latest --access public", + "minor": "npm version minor && npm publish --tag latest --access public", + "patch": "npm version patch && npm publish --tag latest --access public", + "dopreminor": "npm version preminor && npm publish --tag next --access public", + "dopremajor": "npm version premajor && npm publish --tag next --access public", + "doprepatch": "npm version prepatch && npm publish --tag next --access public" }, "repository": { "type": "git", @@ -73,6 +80,7 @@ "joi": "^14.3.1", "jsdom": "^16.2.2", "keyword-extractor": "0.0.19", + "open-graph-scraper": "^3.6.2", "path": "^0.12.7", "prettier": "^2.0.5", "pretty": "^2.0.0", diff --git a/source/index.js b/source/index.js index 4c19399..c7f4be8 100644 --- a/source/index.js +++ b/source/index.js @@ -3,12 +3,14 @@ const path = require('path'); const fs = require('fs'); const { flatten } = require('array-flatten'); -// console.log(flatten); - -// process.exit(0); - dotenv.config(); +// eslint-disable-next-line no-unused-vars +const logger = (name) => { + console.log(name); + return name; +}; + /** * ArticleBuilder * @class @@ -66,20 +68,20 @@ class ArticleBuilder { }; let plugins = this.pluginCollector(path.resolve('./source/plugins')); // eslint-disable-next-line import/no-dynamic-require, global-require - plugins = plugins.map(uri => (require(uri))); + plugins = plugins.map((uri) => (require(uri))); - await flatten(this.stages + return flatten(this.stages // remove stages we should skip - .filter(stage => (!this.skip.stages.includes(stage))) + .filter((stage) => (!this.skip.stages.includes(stage))) // map stages to plugins array - .map(stage => plugins + .map((stage) => plugins // filter plugin that have no functions for this stage - .filter(plugin => ((plugin[stage] !== undefined) && (typeof plugin[stage] === 'function'))) + .filter((plugin) => ((plugin[stage] !== undefined) && (typeof plugin[stage] === 'function'))) // filter plugin we need to skip .filter((plugin) => { const { meta: { name } } = plugin; return (this.skip.plugins.find( - skippedPlugin => ( + (skippedPlugin) => ( ( skippedPlugin.name === name && skippedPlugin.stages === undefined @@ -99,8 +101,9 @@ class ArticleBuilder { .sort((pluginA, pluginB) => ( pluginA.meta.dependency.includes(pluginB.meta.name) ? 1 : -1)) + // .map(logger) // map plugins to functions - .map(plugin => (plugin[stage])))) + .map((plugin) => (plugin[stage])))) .reduce(async (state, plugin) => { const resolvedState = await state; return plugin(resolvedState); @@ -109,9 +112,32 @@ class ArticleBuilder { } // (async () => { +// const builder = new ArticleBuilder(); +// builder.skip.stages = [ +// 'github:before', +// 'github', +// 'github:after', +// ]; +// builder.skip.plugins = [ +// { name: 'codepenTransform' }, +// { name: 'codepenTransformIFrame' }, +// { name: 'createREADME' }, +// { name: 'downloadImages' }, +// { name: 'writeMarkdown' }, +// { name: 'TMPDir' }, +// { name: 'initGithub' }, +// { name: 'uploadToRepo' }, +// { name: 'createRepo' }, +// { name: 'createREADME' }, +// { name: 'createCard' }, +// ]; + // try { -// const builder = new ArticleBuilder(); -// await builder.create('https://www.smashingmagazine.com/2020/05/convince-others-against-dark-patterns/'); +// const result = await builder.create('https://increment.com/frontend/a-users-guide-to-css-variables/'); +// // console.log(result); +// console.log(result.tags); +// console.log(result.mercury[0].author); +// console.log(result.openGraph); // } catch (error) { // console.log(error); // } diff --git a/source/libs/PluginBase.js b/source/libs/PluginBase.js index 1523f55..4277a8b 100644 --- a/source/libs/PluginBase.js +++ b/source/libs/PluginBase.js @@ -25,7 +25,7 @@ module.exports = { * @throw {Error} - if some dependencies not met */ dependencyCheck: (stack = [], dependency = [], name = null) => { - const error = dependency.find(plugin => (!stack.includes(plugin))); + const error = dependency.find((plugin) => (!stack.includes(plugin))); if (error !== undefined) throw new Error(`Dependencies ${name ? `of ${name}` : ''} not met: ${error}`); }, @@ -36,6 +36,6 @@ module.exports = { */ domainCheck: (url, domain) => { const currentDomain = /https?:\/\/(?[^/\\]+)/ig.exec(url); - return currentDomain && (domain === currentDomain[1]); + return (domain === null) || (currentDomain && (currentDomain[1].includes(domain))); }, }; diff --git a/source/plugins/TMPDir/TMPDir.js b/source/plugins/TMPDir/TMPDir.js index defcbd8..34e07a5 100644 --- a/source/plugins/TMPDir/TMPDir.js +++ b/source/plugins/TMPDir/TMPDir.js @@ -30,7 +30,6 @@ module.exports = deepmerge(pluginBase, { * @return {object} - modified article state */ after: (unmodified) => { - return unmodified; const { meta: { name, @@ -52,7 +51,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, [...dependency, `${name}:before`]); const articleDIR = path.resolve(TMP_DIR_NAME, slug); rimraf.sync(articleDIR); @@ -88,8 +87,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const articleDIR = path.resolve(TMP_DIR_NAME, slug); diff --git a/source/plugins/alistapart.com/alistapart.com.js b/source/plugins/alistapart.com/alistapart.com.js new file mode 100644 index 0000000..2d7d075 --- /dev/null +++ b/source/plugins/alistapart.com/alistapart.com.js @@ -0,0 +1,64 @@ +const deepmerge = require('deepmerge'); +const pluginBase = require('../../libs/PluginBase'); +const TagExtractor = require('../../libs/TagExtractor'); + +/** + * @typedef {object} PluginMeta + * @property {string} name - plugin name + * @property {string[]} dependency - array of plugins that we need to run first + * @property {boolean} async - function return Promise? + */ + +/** + * @namespace + * @typedef {object} Plugin + * @property {PluginMeta} meta - plugins mata data + * @property {function} before - plugin function + */ +module.exports = deepmerge(pluginBase, { + meta: { + name: 'alistapart.com', + dependency: ['createMarkdown', 'domain', 'getTags'], + domain: 'alistapart.com', + }, + + /** + * create README.md file + * @param {object} unmodified - current article sate + * @return {object} - modified article state + */ + [['mutation:after']]: async (unmodified) => { + const { + meta: { + name, + dependency, + domain, + }, + dependencyCheck, + domainCheck, + } = module.exports; + const { + url, + stack, + domain: domainName, + dom: { original }, + mercury: [page], + } = unmodified; + const modified = { + tags: [], + stack: [], + ...unmodified, + }; + const { + tags, + } = modified; + + if (!domainCheck(url, domain)) return unmodified; + dependencyCheck(stack, dependency, name); + const extractedTags = [...original.window.document.querySelectorAll('.cat-links a')].map((element) => element.innerHTML); + modified.tags = [...extractedTags, domainName]; + page.author = original.window.document.querySelector('.author.vcard[itemprop="author"] [itemprop="name"]').innerHTML; + modified.stack.push(name); + return modified; + }, +}); diff --git a/source/plugins/cleanDisqus/cleanDisqus.js b/source/plugins/cleanDisqus/cleanDisqus.js index e7b82f5..a3c0f54 100644 --- a/source/plugins/cleanDisqus/cleanDisqus.js +++ b/source/plugins/cleanDisqus/cleanDisqus.js @@ -49,7 +49,7 @@ module.exports = deepmerge(pluginBase, { mercury, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); Array.from(mercury.window.document.querySelectorAll('[class*="disqus"],[class*="dsq-"],[id*="disqus"]')) diff --git a/source/plugins/cleanHidden/cleanHidden.js b/source/plugins/cleanHidden/cleanHidden.js index 690c19a..3afb1ad 100644 --- a/source/plugins/cleanHidden/cleanHidden.js +++ b/source/plugins/cleanHidden/cleanHidden.js @@ -49,7 +49,7 @@ module.exports = deepmerge(pluginBase, { mercury, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); Array.from(mercury.window.document.querySelectorAll('[hidden],[style*="display:none"],[style*="display: none"]')) diff --git a/source/plugins/codepenTransform/codepenTransform.js b/source/plugins/codepenTransform/codepenTransform.js index 35b15f7..45e2391 100644 --- a/source/plugins/codepenTransform/codepenTransform.js +++ b/source/plugins/codepenTransform/codepenTransform.js @@ -68,7 +68,7 @@ module.exports = deepmerge(pluginBase, { mercury, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); modified.stack.push(name); diff --git a/source/plugins/codepenTransformIFrame/codepenTransformIFrame.js b/source/plugins/codepenTransformIFrame/codepenTransformIFrame.js index d3aab6d..caf6fe3 100644 --- a/source/plugins/codepenTransformIFrame/codepenTransformIFrame.js +++ b/source/plugins/codepenTransformIFrame/codepenTransformIFrame.js @@ -70,7 +70,7 @@ module.exports = deepmerge(pluginBase, { mercury, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); modified.stack.push(name); diff --git a/source/plugins/createCard/createCard.js b/source/plugins/createCard/createCard.js index d6b1766..a5984bf 100644 --- a/source/plugins/createCard/createCard.js +++ b/source/plugins/createCard/createCard.js @@ -52,7 +52,7 @@ module.exports = deepmerge(pluginBase, { assignees, tags, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const [{ title }] = mercury; await gitHubUtils.createCard(url, title, tags, assignees); diff --git a/source/plugins/createMarkdown/createMarkdown.js b/source/plugins/createMarkdown/createMarkdown.js index 3c03176..2220995 100644 --- a/source/plugins/createMarkdown/createMarkdown.js +++ b/source/plugins/createMarkdown/createMarkdown.js @@ -1,5 +1,3 @@ -const fs = require('fs'); -const path = require('path'); const prettier = require('prettier'); const query = require('query-string'); const deepmerge = require('deepmerge'); @@ -179,8 +177,6 @@ module.exports = deepmerge(pluginBase, { const { url, stack, - slug, - TMP_DIR_NAME, } = unmodified; const modified = { dom: {}, @@ -192,7 +188,7 @@ module.exports = deepmerge(pluginBase, { mercury, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); let markdown = convertToMD(mercury); @@ -204,18 +200,6 @@ module.exports = deepmerge(pluginBase, { useTabs: false, }); - fs.writeFileSync(path.resolve( - TMP_DIR_NAME, - slug, - 'eng.md', - ), markdown); - - fs.writeFileSync(path.resolve( - TMP_DIR_NAME, - slug, - 'rus.md', - ), markdown); - modified.markdown = markdown; modified.stack.push(name); return modified; diff --git a/source/plugins/createREADME/createREADME.js b/source/plugins/createREADME/createREADME.js index 104b035..9976a1e 100644 --- a/source/plugins/createREADME/createREADME.js +++ b/source/plugins/createREADME/createREADME.js @@ -53,7 +53,7 @@ module.exports = deepmerge(pluginBase, { excerpt, }], } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); fs.writeFileSync(path.resolve( diff --git a/source/plugins/createRepo/createRepo.js b/source/plugins/createRepo/createRepo.js index 5fd9273..d8974c0 100644 --- a/source/plugins/createRepo/createRepo.js +++ b/source/plugins/createRepo/createRepo.js @@ -47,7 +47,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const [{ title }] = mercury; await gitHubUtils.createRepo(slug, title); diff --git a/source/plugins/css-tricks.com/css-tricks.com.js b/source/plugins/css-tricks.com/css-tricks.com.js new file mode 100644 index 0000000..34079e1 --- /dev/null +++ b/source/plugins/css-tricks.com/css-tricks.com.js @@ -0,0 +1,64 @@ +const deepmerge = require('deepmerge'); +const pluginBase = require('../../libs/PluginBase'); +const TagExtractor = require('../../libs/TagExtractor'); + +/** + * @typedef {object} PluginMeta + * @property {string} name - plugin name + * @property {string[]} dependency - array of plugins that we need to run first + * @property {boolean} async - function return Promise? + */ + +/** + * @namespace + * @typedef {object} Plugin + * @property {PluginMeta} meta - plugins mata data + * @property {function} before - plugin function + */ +module.exports = deepmerge(pluginBase, { + meta: { + name: 'css-tricks.com', + dependency: ['createMarkdown', 'domain', 'getTags'], + domain: 'css-tricks.com', + }, + + /** + * create README.md file + * @param {object} unmodified - current article sate + * @return {object} - modified article state + */ + [['mutation:after']]: async (unmodified) => { + const { + meta: { + name, + dependency, + domain, + }, + dependencyCheck, + domainCheck, + } = module.exports; + const { + url, + stack, + domain: domainName, + dom: { original }, + mercury: [page], + } = unmodified; + const modified = { + tags: [], + stack: [], + ...unmodified, + }; + const { + tags, + } = modified; + + if (!domainCheck(url, domain)) return unmodified; + dependencyCheck(stack, dependency, name); + const extractedTags = [...original.window.document.querySelectorAll('.tags a')].map((element) => element.innerHTML); + modified.tags = [...extractedTags, domainName]; + page.author = original.window.document.querySelector('.author-name-area .author-name').innerHTML; + modified.stack.push(name); + return modified; + }, +}); diff --git a/source/plugins/detectLanguage/detectLanguage.js b/source/plugins/detectLanguage/detectLanguage.js index e016947..4a4ad87 100644 --- a/source/plugins/detectLanguage/detectLanguage.js +++ b/source/plugins/detectLanguage/detectLanguage.js @@ -51,7 +51,7 @@ module.exports = deepmerge(pluginBase, { mercury, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const detectLanguage = new DetectLanguage({ diff --git a/source/plugins/dev.to/dev.to.js b/source/plugins/dev.to/dev.to.js new file mode 100644 index 0000000..1616f7c --- /dev/null +++ b/source/plugins/dev.to/dev.to.js @@ -0,0 +1,60 @@ +const deepmerge = require('deepmerge'); +const pluginBase = require('../../libs/PluginBase'); + +/** + * @typedef {object} PluginMeta + * @property {string} name - plugin name + * @property {string[]} dependency - array of plugins that we need to run first + * @property {boolean} async - function return Promise? + */ + +/** + * @namespace + * @typedef {object} Plugin + * @property {PluginMeta} meta - plugins mata data + * @property {function} before - plugin function + */ +module.exports = deepmerge(pluginBase, { + meta: { + name: 'dev.to', + dependency: ['createMarkdown', 'domain', 'getTags'], + domain: 'dev.to', + }, + + /** + * create README.md file + * @param {object} unmodified - current article sate + * @return {object} - modified article state + */ + [['mutation:after']]: async (unmodified) => { + const { + meta: { + name, + dependency, + domain, + }, + dependencyCheck, + domainCheck, + } = module.exports; + const { + url, + stack, + domain: domainName, + dom: { original }, + } = unmodified; + const modified = { + tags: [], + stack: [], + ...unmodified, + }; + + if (!domainCheck(url, domain)) return unmodified; + dependencyCheck(stack, dependency, name); + + const extractedTags = [...original.window.document.querySelectorAll('.tags .tag')].map((element) => element.innerHTML).map((element) => element.slice(1)); + + modified.tags = [...extractedTags, domainName]; + modified.stack.push(name); + return modified; + }, +}); diff --git a/source/plugins/domain/domain.js b/source/plugins/domain/domain.js index 9f486da..e8e8cdc 100644 --- a/source/plugins/domain/domain.js +++ b/source/plugins/domain/domain.js @@ -41,7 +41,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(unmodified.url, targetDomain)) return unmodified; + if (!domainCheck(unmodified.url, targetDomain)) return unmodified; dependencyCheck(unmodified.stack, dependency, name); const parsed = path.parse(unmodified.url); if (parsed.ext !== '') { diff --git a/source/plugins/downloadImages/downloadImages.js b/source/plugins/downloadImages/downloadImages.js index 4dfd842..b228f88 100644 --- a/source/plugins/downloadImages/downloadImages.js +++ b/source/plugins/downloadImages/downloadImages.js @@ -148,7 +148,7 @@ module.exports = deepmerge(pluginBase, { }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const linked = mercury.window.document.querySelectorAll('a img'); @@ -184,7 +184,7 @@ module.exports = deepmerge(pluginBase, { }); const list = [...new Set(flatten(downloadsList))]; - const downloads = list.map(async uri => download(uri, base, DIR)); + const downloads = list.map(async (uri) => download(uri, base, DIR)); const names = await Promise.all(downloads); let html = mercury.window.document.documentElement.outerHTML; diff --git a/source/plugins/fetch/fetch.js b/source/plugins/fetch/fetch.js index c7b72bc..5a7507f 100644 --- a/source/plugins/fetch/fetch.js +++ b/source/plugins/fetch/fetch.js @@ -43,7 +43,7 @@ module.exports = deepmerge(pluginBase, { domainCheck, dependencyCheck, } = module.exports; - if (domainCheck(unmodified.url, domain)) return unmodified; + if (!domainCheck(unmodified.url, domain)) return unmodified; dependencyCheck(unmodified.stack, dependency, name); const response = await fetch(unmodified.url); if (!response.ok) throw new Error(`${name}: can't fetch resource.`); diff --git a/source/plugins/getTags/getTags.js b/source/plugins/getTags/getTags.js index de7b3ed..2f22a83 100644 --- a/source/plugins/getTags/getTags.js +++ b/source/plugins/getTags/getTags.js @@ -50,7 +50,7 @@ module.exports = deepmerge(pluginBase, { const { tags, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const extractedTags = await new TagExtractor(markdown); modified.tags = [...tags, domainName, ...extractedTags]; diff --git a/source/plugins/initGithub/initGithub.js b/source/plugins/initGithub/initGithub.js index fde161f..da393fc 100644 --- a/source/plugins/initGithub/initGithub.js +++ b/source/plugins/initGithub/initGithub.js @@ -44,7 +44,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); modified.gitHubUtils = new GitHubUtils(); modified.stack.push(name); diff --git a/source/plugins/matchContainer/matchContainer.js b/source/plugins/matchContainer/matchContainer.js index 041ca85..bbfee29 100644 --- a/source/plugins/matchContainer/matchContainer.js +++ b/source/plugins/matchContainer/matchContainer.js @@ -1,6 +1,17 @@ const deepmerge = require('deepmerge'); const pluginBase = require('../../libs/PluginBase'); +const Node = { + ELEMENT_NODE: 1, // An Element node like

or

. + TEXT_NODE: 3, // The actual Text inside an Element or Attr. + CDATA_SECTION_NODE: 4, // A CDATASection, such as . + PROCESSING_INSTRUCTION_NODE: 7, // A ProcessingInstruction of an XML document, such as . + COMMENT_NODE: 8, // A Comment node, such as . + DOCUMENT_NODE: 9, // A Document node. + DOCUMENT_TYPE_NODE: 10, // A DocumentType node, such as . + DOCUMENT_FRAGMENT_NODE: 11, // A DocumentFragment node. +}; + /** * @typedef {object} PluginMeta * @property {string} name - plugin name @@ -29,7 +40,8 @@ module.exports = deepmerge(pluginBase, { if (element === null) throw new Error('element should be provided'); const tag = element.tagName.toLowerCase(); const id = element.getAttribute('id'); - const classname = element.getAttribute('class').split(' ').join('.'); + let classname = element.getAttribute('class'); + if (classname && classname.indexOf(' ') > -1) classname = classname.split(' ').join('.'); let attributesSelector = ''; const skipAttr = ['class', 'id']; for (let i = element.attributes.length - 1; i >= 0; i -= 1) { @@ -40,6 +52,7 @@ module.exports = deepmerge(pluginBase, { /** * match mercury and fetch dom containers + * @todo find out why do I need this. * @param {object} unmodified - current article sate * @return {object} - modified article state */ @@ -67,7 +80,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const element = mercury.window.document.querySelector('body').firstChild; @@ -78,13 +91,13 @@ module.exports = deepmerge(pluginBase, { } else if (container.length > 0) { container = Array.from(container); const selectors = Array.from(element.childNodes) - .filter(node => (node.nodeType === 1)) + .filter((node) => (node.nodeType === 1)) .map(getNodeSelector); - container.filter(node => ( + container.filter((node) => ( Array.from(node.childNodes) - .filter(child => (child.nodeType !== Node.TEXT_NODE)) + .filter((child) => (child.nodeType !== Node.TEXT_NODE)) .find( - kid => (!selectors.include(getNodeSelector(kid))), + (kid) => (!selectors.includes(getNodeSelector(kid))), ) === undefined)); if (container.length === 1) { [modified.dom.matched] = container; diff --git a/source/plugins/mercury/mercury.js b/source/plugins/mercury/mercury.js index ed373ea..87979d3 100644 --- a/source/plugins/mercury/mercury.js +++ b/source/plugins/mercury/mercury.js @@ -44,13 +44,13 @@ module.exports = deepmerge(pluginBase, { domainCheck, dependencyCheck, } = module.exports; - if (domainCheck(unmodified.url, domain)) return unmodified; + if (!domainCheck(unmodified.url, domain)) return unmodified; dependencyCheck(unmodified.stack, dependency, name); const { JSDOM } = jsdom; const parser = new Mercury(); modified.mercury = await parser.getAll(unmodified.url); modified.html.mercury = modified.mercury - .map(page => (page.content)) + .map((page) => (page.content)) .reduce((accumulator, page) => (`${accumulator}${page}`)); modified.dom.mercury = new JSDOM(modified.html.mercury); modified.stack.push(name); diff --git a/source/plugins/openGraph/openGraph.js b/source/plugins/openGraph/openGraph.js new file mode 100644 index 0000000..c0a1363 --- /dev/null +++ b/source/plugins/openGraph/openGraph.js @@ -0,0 +1,104 @@ +const deepmerge = require('deepmerge'); +const ogs = require('open-graph-scraper'); +const pluginBase = require('../../libs/PluginBase'); + +const getOpenGraph = (html) => new Promise((resolve, reject) => ogs({ html }, (error, results) => { + if (error) reject(results); + resolve(results.data); +})); + +/** + * @typedef {object} PluginMeta + * @property {string} name - plugin name + * @property {string[]} dependency - array of plugins that we need to run first + * @property {boolean} async - function return Promise? + */ + +/** + * @namespace + * @typedef {object} Plugin + * @property {PluginMeta} meta - plugins mata data + * @property {function} before - plugin function + */ +module.exports = deepmerge(pluginBase, { + meta: { + name: 'open-graph', + dependency: ['fetch'], + }, + + /** + * Aquire open-graph data + * @param {object} unmodified - current article sate + * @return {object} - modified article state + */ + [['metadata']]: async (unmodified) => { + const { + meta: { + name, + dependency, + }, + dependencyCheck, + } = module.exports; + const { + stack, + html, + dom: { original }, + } = unmodified; + const modified = { + openGraph: {}, + ...unmodified, + }; + dependencyCheck(stack, dependency, name); + + const article = ['publisher', 'author', 'tag', 'section', 'published_time', 'modified_time']; + const book = ['author', 'isbn', 'release_date', 'tag']; + const profile = ['first_name', 'last_name', 'username', 'gender']; + const noVertical = { + article, + book, + profile, + }; + + const extended = Object.entries(noVertical).reduce((collector, [propertyName, list]) => { + const noVerticalInstance = list.reduce((block, content) => { + const element = original.window.document.querySelector(`meta[property="${propertyName}:${content}"]`); + if (element === null) return block; + return { + [[content]]: element.getAttribute('content'), + ...block, + }; + }, {}); + if (Object.keys(noVerticalInstance).length === 0) return collector; + return { ...collector, [[propertyName]]: noVerticalInstance }; + }, {}); + + let openGraph = { ...extended }; + + const ogupdatedTimeElement = original.window.document.querySelector('meta[property="og:updated_time"]'); + if (ogupdatedTimeElement !== null) { + const ogUpdatedTime = ogupdatedTimeElement.getAttribute('content'); + if (ogUpdatedTime) { + openGraph = { + ...openGraph, + ogUpdatedTime, + }; + } + } + + try { + const data = await getOpenGraph(html.original); + openGraph = { + ...openGraph, + ...data, + }; + // eslint-disable-next-line no-empty + } catch (error) {} + + modified.openGraph = { + ...modified.openGraph, + ...openGraph, + }; + modified.stack.push(name); + return modified; + }, +}); diff --git a/source/plugins/restoreCodeSnippets/restoreCodeSnippets.js b/source/plugins/restoreCodeSnippets/restoreCodeSnippets.js index e08296b..bc1a96c 100644 --- a/source/plugins/restoreCodeSnippets/restoreCodeSnippets.js +++ b/source/plugins/restoreCodeSnippets/restoreCodeSnippets.js @@ -50,7 +50,7 @@ module.exports = deepmerge(pluginBase, { matched, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); modified.stack.push(name); if (!matched) return unmodified; diff --git a/source/plugins/restoreImages/restoreImages.js b/source/plugins/restoreImages/restoreImages.js index cf78c1d..6c699d9 100644 --- a/source/plugins/restoreImages/restoreImages.js +++ b/source/plugins/restoreImages/restoreImages.js @@ -50,7 +50,7 @@ module.exports = deepmerge(pluginBase, { matched, }, } = modified; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); modified.stack.push(name); if (!matched) return unmodified; diff --git a/source/plugins/slug/slug.js b/source/plugins/slug/slug.js index 6c3e93b..f7c435f 100644 --- a/source/plugins/slug/slug.js +++ b/source/plugins/slug/slug.js @@ -41,7 +41,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(unmodified.url, domain)) return unmodified; + if (!domainCheck(unmodified.url, domain)) return unmodified; dependencyCheck(unmodified.stack, dependency, name); const { slug, diff --git a/source/plugins/smashingmagazine.com/smashingmagazine.com.js b/source/plugins/smashingmagazine.com/smashingmagazine.com.js new file mode 100644 index 0000000..95e0063 --- /dev/null +++ b/source/plugins/smashingmagazine.com/smashingmagazine.com.js @@ -0,0 +1,121 @@ +const deepmerge = require('deepmerge'); +const pluginBase = require('../../libs/PluginBase'); + +/** + * @typedef {object} PluginMeta + * @property {string} name - plugin name + * @property {string[]} dependency - array of plugins that we need to run first + * @property {boolean} async - function return Promise? + */ + +/** + * @namespace + * @typedef {object} Plugin + * @property {PluginMeta} meta - plugins mata data + * @property {function} before - plugin function + */ +module.exports = deepmerge(pluginBase, { + meta: { + name: 'smashingmagazine.com', + dependency: ['fetch', 'domain', 'getTags'], + domain: 'smashingmagazine.com', + }, + + /** + * Cleaning open-graph title + */ + [['metadata:after']]: async (unmodified) => { + const { + meta: { + name, + domain, + }, + dependencyCheck, + domainCheck, + } = module.exports; + const { + url, + stack, + dom: { original }, + } = unmodified; + const modified = { + openGraph: {}, + ...unmodified, + }; + const { + openGraph: { + ogTitle, + twitterTitle, + }, + } = modified; + + if (!domainCheck(url, domain)) return unmodified; + dependencyCheck(stack, ['open-graph'], name); + + const publisher = original.window.document.querySelector('meta[property="article:publisher"]').getAttribute('content'); + const author = original.window.document.querySelector('meta[property="article:author"]').getAttribute('content'); + const tag = original.window.document.querySelector('meta[property="article:tag"]').getAttribute('content'); + const section = original.window.document.querySelector('meta[property="article:section"]').getAttribute('content'); + const publishedTime = original.window.document.querySelector('meta[property="article:published_time"]').getAttribute('content'); + const modifiedTime = original.window.document.querySelector('meta[property="article:modified_time"]').getAttribute('content'); + const ogupdatedTime = original.window.document.querySelector('meta[property="og:updated_time"]').getAttribute('content'); + + const openGraph = { + article: { + publisher, + author, + tag, + section, + publishedTime, + modifiedTime, + }, + ogupdatedTime, + }; + + modified.openGraph = { + ...modified.openGraph, + ...openGraph, + }; + modified.openGraph.ogTitle = ogTitle.replace(' — Smashing Magazine', ''); + modified.openGraph.twitterTitle = twitterTitle.replace(' — Smashing Magazine', ''); + modified.stack.push(name); + return modified; + }, + + /** + * create README.md file + * @param {object} unmodified - current article sate + * @return {object} - modified article state + */ + [['mutation:after']]: async (unmodified) => { + const { + meta: { + name, + dependency, + domain, + }, + dependencyCheck, + domainCheck, + } = module.exports; + const { + url, + stack, + domain: domainName, + dom: { original }, + mercury: [page], + } = unmodified; + const modified = { + tags: [], + stack: [], + ...unmodified, + }; + + if (!domainCheck(url, domain)) return unmodified; + dependencyCheck(stack, dependency, name); + const extractedTags = [...original.window.document.querySelectorAll('.meta-box--tags a')].map((element) => element.innerHTML); + modified.tags = [...extractedTags, domainName]; + page.author = original.window.document.querySelector('.bio-image-image').getAttribute('data-alt'); + modified.stack.push(name); + return modified; + }, +}); diff --git a/source/plugins/uploadToRepo/uploadToRepo.js b/source/plugins/uploadToRepo/uploadToRepo.js index 23c5ee3..d8f7fed 100644 --- a/source/plugins/uploadToRepo/uploadToRepo.js +++ b/source/plugins/uploadToRepo/uploadToRepo.js @@ -48,7 +48,7 @@ module.exports = deepmerge(pluginBase, { stack: [], ...unmodified, }; - if (domainCheck(url, domain)) return unmodified; + if (!domainCheck(url, domain)) return unmodified; dependencyCheck(stack, dependency, name); const articleDIR = path.resolve(TMP_DIR_NAME, slug); await gitHubUtils.uploadDir(articleDIR); diff --git a/source/plugins/writeMarkdown.js/writeMarkdown.js b/source/plugins/writeMarkdown.js/writeMarkdown.js new file mode 100644 index 0000000..eba84f5 --- /dev/null +++ b/source/plugins/writeMarkdown.js/writeMarkdown.js @@ -0,0 +1,67 @@ +const fs = require('fs'); +const path = require('path'); +const deepmerge = require('deepmerge'); +const pluginBase = require('../../libs/PluginBase'); + +/** + * @typedef {object} PluginMeta + * @property {string} name - plugin name + * @property {string[]} dependency - array of plugins that we need to run first + * @property {boolean} async - function return Promise? + */ + +/** + * @namespace + * @typedef {object} Plugin + * @property {PluginMeta} meta - plugins mata data + * @property {function} before - plugin function + */ +module.exports = deepmerge(pluginBase, { + meta: { + name: 'writeMarkdown', + dependency: ['createMarkdown', 'slug'], + }, + + /** + * match mercury and fetch dom containers + * @param {object} unmodified - current article sate + * @return {object} - modified article state + */ + [['mutation:after']]: (unmodified) => { + const { + meta: { + name, + dependency, + }, + dependencyCheck, + } = module.exports; + const { + stack, + slug, + TMP_DIR_NAME, + } = unmodified; + const modified = { + stack: [], + ...unmodified, + }; + const { + markdown, + } = modified; + dependencyCheck(stack, dependency, name); + + fs.writeFileSync(path.resolve( + TMP_DIR_NAME, + slug, + 'eng.md', + ), markdown); + + fs.writeFileSync(path.resolve( + TMP_DIR_NAME, + slug, + 'rus.md', + ), markdown); + + modified.stack.push(name); + return modified; + }, +});