diff --git a/.gitignore b/.gitignore index 8314e37..4d693eb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ coverage # Compiled binary addons (http://nodejs.org/api/addons.html) build/Release +# Bikeshed clone +bikeshed + # Dependency directory # Commenting this out is preferred by some people, see # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..cf905b0 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +prefix = ${HOME}/node_modules diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..dd1381b --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,135 @@ +# Run Publican in Docker + +If you have [Docker installed](https://docs.docker.com/installation/) and running, you can run it in a few commands. + +It can summarize as; + +```bash +docker pull webspecs/publican:latest +docker run -it --rm -p 7002:7002 -v "$(pwd)/data":/srv/webplatform/specs/data -v "$(pwd)/spec-data":/opt/bikeshed/bikeshed/spec-data webspecs/publican:latest bin/run.sh +``` + +If you want to run with your own customizations, you’d have to do also have [**Docker Compose** installed](https://docs.docker.com/compose/install/) and do the following. + +Follow the instructions in **Run using _Docker Compose_** in [the registry description instructions](https://registry.hub.docker.com/u/webspecs/publican/) + + +## Other possible usage + +### Build Docker container + +When we call `docker pull webspecs/publican:latest`, its either calling a last successful Docker build command that has the `-t webspecs/publican` +or its asking to get the latest version from the Docker hub. + +In order to deploy, we need to have an image to run, that’s why you have to build one. + +To build a **webspecs/publican** container, you need to use the publican repository. + +Once a build is successful you can push it to [Push Docker container to Docker Hub registry](#Push Docker container to Docker Hub registry). + +Once you have a clone of publican, get the dependencies; + + make clone-bikeshed + +Which will clone bikeshed and create required directories for you. + +Build the container + + make docker-build + +Run the container + + make docker-run + +If you review what’s in the [Makefile](./Makefile), you’ll see that it basicall expands to something like this; + + docker run -it --rm -p 7002:7002 \ + -v /Users/renoirb/workspaces/webplatform/service-publican/repo/data:/srv/webapps/publican/data \ + -v /Users/renoirb/workspaces/webplatform/service-publican/repo/spec-data:/opt/bikeshed/bikeshed/spec-data \ + webspecs/publican:latest bin/run.sh + +Notice that the `-v /full/path/host:/full/path/container` option requires FULL path in both sides of the column + +The last part of the `docker run` is what’s going to run, in this case `bin/run.sh`. +Which means you could get into a running container shell if you need to. + +That’s what the `make docker-bash` do. + +Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) and run this project from a Docker container. + + +### Run an instance locally + +Run a container, and send a hook call + + make docker-run + +On another terminal tab, send a hook; + + curl -H "Content-Type: application/json" -XPOST localhost:7002/hook -d '{"base_ref":"master","repository":{"name":"assets","owner":{"name":"webspecs"}}}' + + +you should see something like; + + {"ok":true,"details":"Queued 1431380611165-28770219 for processing."} + +And in the container, you’d see; + +![publican-run-hook](https://cloud.githubusercontent.com/assets/296940/7575805/f18cb1d2-f805-11e4-8ec3-dba68dae7785.png) + + + +### Enter the container shell + +You can initialize an empty workspace using `publican.js init`, it’ll write in `data/` that should already be mounted through the `Makefile`. + +Its also useful to understand what’s happening behind the scenes. + + make docker-bash + +![publican-init](https://cloud.githubusercontent.com/assets/296940/7575777/b5a76176-f805-11e4-99e7-3a7c58dd304a.png) + + + +### Push Docker container to Docker Hub registry + +The Docker registry is basically a storage service of docker container states from which we can pull and run from it. +It saves us to rebuild the container on every server that would run the container. + +Docker uses a semantic similar to git, a container can be commited and pushed to a repository. That’s what we’ll do here. + +Refer to [Docker command line "commit" documentation](http://docs.docker.com/reference/commandline/cli/#commit-a-container) for further details. + +This project Docker registry entry is at [registry.hub.docker.com/u/**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/). + +To commit a container, we need a running one. Once you have one, you can commit. + +Get current running container; that’s what we commit. + +```bash +docker ps + +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +3cce48d3b30c servicepublican_publican:latest "/bin/bash /srv/webp 11 minutes ago Up 11 minutes 0.0.0.0:8002->7002/tcp sandboxpublican_web_1" +``` + +If you have write access as a *collaborator* to the *webspecs* organization within Docker hub; Commit and push + +```bash +docker commit 3cce48d3b30c +docker push webspecs/publican +``` + +Done. + + + +### Run from Docker composer + +TODO, see notes in [**webspecs/publican**](https://registry.hub.docker.com/u/webspecs/publican/) Docker hub description. + + +## Reference + +Refer to the appropriate documentation if you want to [install](https://docs.docker.com/installation/#installation) +and run this project from a Docker container locally. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..feacebb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,67 @@ +# +# Publican Docker runner +# +# See also: +# * https://github.com/nodesource/docker-node/blob/master/ubuntu/trusty/node/0.10.36/Dockerfile + +FROM nodesource/trusty:0.10.36 + +MAINTAINER Renoir Boulanger + +ENV DEBIAN_FRONTEND=noninteractive + +# Dependencies: Bikeshed, PhantomJS, Bikshed’s lxml +RUN apt-get update && apt-get -y upgrade && \ + apt-get install -yqq git python2.7 python-dev python-pip libxslt1-dev libxml2-dev zlib1g-dev && \ + apt-get install -yqq libfontconfig1 libfreetype6 curl && \ + apt-get autoremove -yqq --purge && \ + pip install --upgrade lxml + +# Copy everything we have locally into the container +# REMINDER: Make sure you run `make clone-bikeshed`, we prefer to keep a copy locally outside +# of the data volume. Otherwise it would make problems saying that bikeshed clone is not in the +# same filesystem. +COPY . /srv/webapps/publican/ + +# Make sure we have a "non root" user and +# delete any local workbench data/ directory +RUN /usr/sbin/groupadd --system --gid 990 webapps && \ + /usr/sbin/useradd --system --gid 990 --uid 990 -G sudo --home-dir /srv/webapps --shell /bin/bash webapps && \ + sed -i '/^%sudo/d' /etc/sudoers && \ + echo '%sudo ALL=NOPASSWD: ALL' >> /etc/sudoers && \ + mv /srv/webapps/publican/bikeshed /opt && \ + rm -rf data && \ + mkdir -p data/temp && \ + rm -rf Dockerfile Makefile .git .gitignore DOCKER.md && \ + chown -R webapps:webapps /srv/webapps/publican && \ + chown -R webapps:webapps /opt/bikeshed + +# Switch from root to webapps system user +# It **HAS to be** the SAME uid/gid as the owner on the host from which we’ll use as volume +USER webapps + +# Where the session will start from +WORKDIR /srv/webapps/publican + +# Environment variables +ENV PATH /srv/webapps/publican/node_modules/.bin:/srv/webapps/publican/bin:/srv/webapps/publican/.local/bin:$PATH +ENV HOME /srv/webapps/publican +ENV TMPDIR /srv/webapps/publican/data/temp +ENV NODE_ENV production +ENV GIT_DISCOVERY_ACROSS_FILESYSTEM true + +# Run what `make deps` would do +RUN pip install --upgrade --user --editable /opt/bikeshed && \ + mkdir -p node_modules && npm install + +# Declare which port we expect to expose +EXPOSE 7002 + +# Allow cli entry for debug, but make sure docker-compose.yml uses "command: bin/run.sh" +ENTRYPOINT ["/bin/bash"] + +# Note leftover: Ideally, it should exclusively run +#ENTRYPOINT ["/bin/bash", "/srv/webapps/publican/bin/run.sh"] + +# Note leftover: What it ends up doing +#CMD ["node_modules/forever/bin/forever", "--fifo", "logs", "0"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a30370e --- /dev/null +++ b/Makefile @@ -0,0 +1,25 @@ +SHELL := bash + +default: deps + +clone-bikeshed: datadirs + git clone -b webspecs https://github.com/webspecs/bikeshed.git bikeshed + +datadirs: + mkdir -p $(CURDIR)/data/{queue,gits,logs,temp} + mkdir -p $(CURDIR)/spec-data + +# Make sure Dockerfile has equivalent at note about `make deps` +deps: + pip install --user --editable bikeshed + npm install + +docker-build: + docker build -t webspecs/publican . + +docker-bash: + docker run -it --rm -v "$(CURDIR)/data":/srv/webapps/publican/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest + +docker-run: + docker run -it --rm -v "$(CURDIR)/data":/srv/webapps/publican/data -v "$(CURDIR)/spec-data":/opt/bikeshed/bikeshed/spec-data -p 7002:7002 webspecs/publican:latest bin/run.sh + diff --git a/README.md b/README.md index 6e62926..2812b95 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -publican -======== +# Publican The set of tools used to automate publishing in the WebSpecs project + diff --git a/bin/run.sh b/bin/run.sh new file mode 100644 index 0000000..ae6f52d --- /dev/null +++ b/bin/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +export RUNDIR="/srv/webapps/publican" + +cd $RUNDIR + +node_modules/forever/bin/forever start $RUNDIR/bin/server.js +node_modules/forever/bin/forever --fifo logs 0 diff --git a/data/config.json.dist b/data/config.json.dist new file mode 100644 index 0000000..8afba2f --- /dev/null +++ b/data/config.json.dist @@ -0,0 +1,18 @@ +{ + "bikeshed": "/opt/bikeshed/bikeshed.py" +, "rsyncPath": "/srv/webapps/publican/" +, "python": "python2" +, "logFile": "logs/all.log" +, "email": { + "to": "team-webplatform-systems@w3.org" + , "from": "publican-noreply@webplatform.org" + , "host": "localhost" + , "password": "XXXX not the real thing, will not work. Should be replaced by a login that can email from here." + , "ssl": true + , "tls": false + , "level": "error" + , "handleExceptions": true + } +, "purgeAllURL": "https://api.fastly.com/service/not_the_real_thing/purge_all" +, "purgeAllKey": "not_the_real_thing" +} diff --git a/docker-compose.yml.sample b/docker-compose.yml.sample new file mode 100644 index 0000000..7e05beb --- /dev/null +++ b/docker-compose.yml.sample @@ -0,0 +1,29 @@ +main: + image: webspecs/publican:latest + + # Default in this container is bash, using command below runs + # publican server for us while allowing us to enter the original + # container image in case of need to understand what’s available. + command: bin/run.sh + + # Make sure you have a data folder in the same + # directory as this file. Container expects the following + # files and folders. + # - data/{queue,gits,logs,temp} + # - data/config.json + # - spec-data/read-only + volumes: + - data/:/srv/webapps/publican/data + - spec-data/:/opt/bikeshed/bikeshed/spec-data + + restart: always + + ports: + - "7002:7002" + + dns: + - 10.10.10.41 + - 8.8.8.8 + +# vim: et ts=2 sw=2 ft=yaml: + diff --git a/lib/lock.js b/lib/lock.js index e6e6f16..b543582 100644 --- a/lib/lock.js +++ b/lib/lock.js @@ -3,19 +3,19 @@ var lock = require("lock")() , timeout = 1000 * 60 * 10 // 10 minutes ; -exports.createLock = function (man) { - return function (key, action, done) { +exports.createLock = function createLockHandler (man) { + return function createLockReturner (key, action, done) { man.log.info("Locking " + key); - lock(key, function (release) { + lock(key, function createLockLocker (release) { man.log.log("silly", "Processing locked key " + key); var tid = setTimeout(function () { man.log.error("Timeout on lock expired, releasing " + key + " with risk of conflict."); release(done)(); }, timeout); - action(function (err) { + action(function createLockLockerAction (err) { man.log.info("Releasing lock " + key); clearTimeout(tid); - release(function () { done(err); })(); + release(function createLockRelease () { done(err); })(); }); }); }; diff --git a/lib/manager.js b/lib/manager.js index 0b2f33a..e60e929 100644 --- a/lib/manager.js +++ b/lib/manager.js @@ -11,6 +11,7 @@ var fs = require("fs") , processStatic = require("./tasks/process-static") , processBikeshed = require("./tasks/init-bikeshed") , processSpec = require("./tasks/process-spec") +, purge = require("./tasks/purge") ; // loads the configuration itself @@ -25,19 +26,19 @@ function Manager (opt) { } Manager.prototype = { // run this with extreme caution - initSetup: function (cb) { + initSetup: function initSetupHandler (cb) { this.log.info("Initialising setup"); - this.runTask(require("./tasks/init-all"), this.getContext(), function (err) { + this.runTask(require("./tasks/init-all"), this.getContext(), function runTaskErrorHandler (err) { if (err) return this.log.error(err); this.log.info("OK!"); if (cb) cb(); }.bind(this)); } -, updateContextForRepo: function (ctx, opt) { +, updateContextForRepo: function updateContextForRepoHandler (ctx, opt) { if (opt.repository) { // git details - ctx.gitRepoURL = "git@github.com:" + opt.repository + ".git"; + ctx.gitRepoURL = "https://github.com/" + opt.repository + ".git"; ctx.gitDir = jn(dataDir, "gits", opt.repository); ctx.repository = opt.repository; } @@ -48,15 +49,15 @@ Manager.prototype = { } } -, getContext: function (opt) { +, getContext: function getContextHandler (opt) { opt = opt || {}; var ctx = { // common facilities log: this.log , lock: lock.createLock(this) - , rfs: function (file) { return fs.readFileSync(file, "utf8"); } - , wfs: function (file, content) { fs.writeFileSync(file, content, { encoding: "utf8" }); } - , buildSpecDir: function (repo, branch) { + , rfs: function readFileSyncHandler (file) { return fs.readFileSync(file, "utf8"); } + , wfs: function writeFileSyncHandler (file, content) { fs.writeFileSync(file, content, { encoding: "utf8" }); } + , buildSpecDir: function buildSpecDirHandler (repo, branch) { var parts = repo.split("/", 2); return [parts[1], parts[0], branch].join("/"); } @@ -65,7 +66,7 @@ Manager.prototype = { , publishDir: jn(dataDir, "publish") , theIndexPath: jn(dataDir, "publish/index.html") // bikeshed details (conf has python) - , bikeshed: jn(dataDir, "bikeshed/bikeshed.py") + , bikeshed: "bikeshed" } ; this.updateContextForRepo(ctx, opt); @@ -74,15 +75,15 @@ Manager.prototype = { return ctx; } -, runTask: function (tasks, ctx, cb) { +, runTask: function runTaskHandler (tasks, ctx, cb) { if (!ctx) ctx = this.getContext(); if (!util.isArray(tasks)) tasks = [tasks]; tasks = tasks.concat(); - var done = function (err) { + var done = function runTaskDoneHandler (err) { if (err) return cb(err); iterate(); } - , iterate = function () { + , iterate = function runTaskIterateHandler() { if (!tasks.length) return cb(); var task = tasks.shift(); // console.log("####### tasks: " + tasks.length); @@ -98,7 +99,7 @@ Manager.prototype = { // this is the entry point for the manager, called by the server // it looks at repoDesc (which has a repository and branch) and based on that it will // pick a task to run, having prepared the correct context for it -, runAppropriateTask: function (repoDesc, cb) { // replaces processRepository +, runAppropriateTask: function funAppropriateTaskHandler (repoDesc, cb) { // replaces processRepository var repo = repoDesc.repository , ctx = this.getContext(repoDesc) ; @@ -119,7 +120,9 @@ Manager.prototype = { else { ctx.subDirOnly = ctx.specSubDir; this.runTask(processSpec, ctx, cb); + this.runTask(purge, ctx, cb); } } + }; module.exports = Manager; diff --git a/lib/queue.js b/lib/queue.js index 7051ab3..45ab56d 100644 --- a/lib/queue.js +++ b/lib/queue.js @@ -5,24 +5,24 @@ var fs = require("fs") , queueDir = jn(dataDir, "queue") ; -exports.enqueue = function (repo, branch, cb) { +exports.enqueue = function enqueueHandler (repo, branch, cb) { var stamp = Date.now() + "-" + process.hrtime()[1]; fs.writeFile( jn(queueDir, stamp + ".json") , JSON.stringify({ repository: repo, branch: branch }, null, 4) , { encoding: "utf8" } - , function (err) { + , function enqueueCallbackHandler (err) { if (err) throw err; cb(null, stamp); } ); }; -exports.next = function (cb) { - fs.readdir(queueDir, function (err, queue) { +exports.next = function enqueueNextHandler (cb) { + fs.readdir(queueDir, function enqueueNextReadDirHandler (err, queue) { if (err) throw err; queue = queue - .sort(function (a, b) { // we have to do this to guarantee numeric comparison + .sort(function enqueueNextReadDirSortHandler (a, b) { // we have to do this to guarantee numeric comparison if (a < b) return -1; if (a > b) return 1; return 0; @@ -31,10 +31,10 @@ exports.next = function (cb) { var top = queue[0] , file = jn(queueDir, top) ; - fs.readFile(file, { encoding: "utf8" }, function (err, content) { + fs.readFile(file, { encoding: "utf8" }, function enqueueNextReadFileHandler (err, content) { if (err) throw err; var content = JSON.parse(content); - fs.unlink(file, function (err) { + fs.unlink(file, function enqueueNextReadFileUnlinkHandler (err) { if (err) throw err; cb(null, content); }); diff --git a/lib/tasks/extract-the-index.js b/lib/tasks/extract-the-index.js index 139cb0c..f05dd55 100644 --- a/lib/tasks/extract-the-index.js +++ b/lib/tasks/extract-the-index.js @@ -8,13 +8,13 @@ var whacko = require("whacko") // sets // theIndexRepositories: an array of repositories extracted from the-index // it has: branch, repository, baseFileName -module.exports = function (ctx, cb) { +module.exports = function extractTheIndex (ctx, cb) { ctx.theIndexRepositories = []; if (!fs.existsSync(ctx.theIndexPath)) return cb(); var $ = whacko.load(ctx.rfs(ctx.theIndexPath)); ctx.log.info("Extracting data from the-index"); $("script[type='application/webspec+json']") - .each(function () { + .each(function extractTheIndexSeekLoop () { try { var data = JSON.parse($(this).text()); for (var k in data) { diff --git a/lib/tasks/generate-bikeshed.js b/lib/tasks/generate-bikeshed.js index 885e7e7..f8bda43 100644 --- a/lib/tasks/generate-bikeshed.js +++ b/lib/tasks/generate-bikeshed.js @@ -8,19 +8,19 @@ var jn = require("path").join // bikeshed: the path to bikeshed // specDir: the path to the directory in which the spec lives // baseFileName: the base file name, without extension, to use (defaults to index) -module.exports = function (ctx, cb) { +module.exports = function generateBikeshed (ctx, cb) { var fn = ctx.baseFileName; if (fn.indexOf("/") === 0 || fn.indexOf("..") > -1) return cb("Invalid file name: " + fn); ctx.lock( "bikeshed " + ctx.specDir - , function (release) { + , function generateBikeshedLockHandler (release) { var cmd = ctx.python + " " + ctx.bikeshed + " -f spec " + jn(ctx.specDir, fn + ".bs") + " " + jn(ctx.specDir, "index.html") ; ctx.log.info(cmd); - exec(cmd, { cwd: ctx.specDir }, function (err, stdout, stderr) { + exec(cmd, { cwd: ctx.specDir }, function generateBikeshedExecHandler (err, stdout, stderr) { if (err) { ctx.log.error(stderr); ctx.log.error(stdout); diff --git a/lib/tasks/generate-respec.js b/lib/tasks/generate-respec.js index def3324..8f7d7fb 100644 --- a/lib/tasks/generate-respec.js +++ b/lib/tasks/generate-respec.js @@ -3,7 +3,7 @@ var jn = require("path").join , phantom = require("phantomjs") , execFile = require("child_process").execFile , querystring = require("querystring") -, num2 = function (num) { +, num2 = function generateRespecNum (num) { var str = num + ""; if (str.length >= 2) return str; return "0" + str; @@ -14,14 +14,14 @@ var jn = require("path").join // specDir: the path to the directory in which the spec lives // baseFileName: the base file name, without extension, to use (defaults to index) // repository: the repository in user/repo form -module.exports = function (ctx, cb) { +module.exports = function generateRespec (ctx, cb) { var fn = ctx.baseFileName , respec2html = jn(ctx.dataDir, "../node_modules/respec/tools/respec2html.js") ; if (fn.indexOf("/") === 0 || fn.indexOf("..") > -1) return cb("Invalid file name: " + fn); ctx.lock( "respec " + ctx.specDir - , function (release) { + , function generateRespecLockHandler (release) { var d = new Date() , enforce = { specStatus: "webspec" @@ -37,10 +37,10 @@ module.exports = function (ctx, cb) { ] ; ctx.log.info(phantom.path + " " + params.join(" ")); - + var timedOut = false , tickingBomb = setTimeout( - function () { + function generateRespecReleaseHandler () { timedOut = true; release(new Error("ReSpec timed out.")); } @@ -50,7 +50,7 @@ module.exports = function (ctx, cb) { execFile( phantom.path , params - , function (err, stdout, stderr) { + , function generateRespecExecHandler (err, stdout, stderr) { if (timedOut) return; clearTimeout(tickingBomb); if (err) { diff --git a/lib/tasks/generate-the-index.js b/lib/tasks/generate-the-index.js index 2e87bff..eb04516 100644 --- a/lib/tasks/generate-the-index.js +++ b/lib/tasks/generate-the-index.js @@ -6,20 +6,20 @@ var whacko = require("whacko") // expects // theIndexPath: the path to the-index file // publishDir: the directory to publish to -module.exports = function (ctx, cb) { +module.exports = function generateTheIndex (ctx, cb) { var $ = whacko.load(ctx.rfs(ctx.theIndexPath)); // make the tables ctx.log.info("Generating specification linking tables in the-index"); $("script[type='application/webspec+json']") - .each(function () { + .each(function generateTheIndexSeekHandler () { var $script = $(this) , ws = JSON.parse($script.text()) , $dl = $("
") - , simpleDD = function (desc, $dl) { + , simpleDD = function generateTheIndexDdHonk (desc, $dl) { $dl.append($("
").html(desc || "n/a")); } - , linkDT = function (repo, $dl) { + , linkDT = function generateTheIndexDtHonk (repo, $dl) { var parts = repo.split("/", 2) , user = parts[0] , $dt = $("
"); @@ -41,12 +41,12 @@ module.exports = function (ctx, cb) { $script.after($dl); }) ; - + // build the ToC var toc = []; ctx.log.info("Building ToC for the-index"); $("section") - .each(function () { + .each(function generateTheIndexSectionHonk () { var $section = $(this) , topLevel = !!$section.find("h2").length , $h = $section.find(topLevel ? "h2" : "h3").first() @@ -79,7 +79,7 @@ module.exports = function (ctx, cb) { if (!$h.hasClass("no-ref")) $h.append($("").attr("href", "#" + id)); }) ; - var tocline = function (details, $parent) { + var tocline = function generateTheIndexTocLineHonk (details, $parent) { var $li = $("
  • ") , $a = $li.find("a") ; diff --git a/lib/tasks/git-clone-or-fetch.js b/lib/tasks/git-clone-or-fetch.js index 266e224..17a8e21 100644 --- a/lib/tasks/git-clone-or-fetch.js +++ b/lib/tasks/git-clone-or-fetch.js @@ -6,8 +6,8 @@ var fs = require("fs-extra") // expects: // gitDir: the directory in which the repo is stored -module.exports = function (ctx, cb) { - fs.exists(ctx.gitDir, function (exists) { +module.exports = function gitCloneOrFetchExports (ctx, cb) { + fs.exists(ctx.gitDir, function gitCloneOrFetchFileExistsHandler (exists) { this.runTask(exists ? fetch : clone, ctx, cb); }.bind(this)); }; diff --git a/lib/tasks/git-clone.js b/lib/tasks/git-clone.js index f6946f2..89a3b29 100644 --- a/lib/tasks/git-clone.js +++ b/lib/tasks/git-clone.js @@ -6,11 +6,11 @@ var fs = require("fs-extra") // expects: // gitDir: the directory in which the repo is stored // gitRepoURL: the repository -module.exports = function (ctx, cb) { +module.exports = function gitClone (ctx, cb) { ctx.lock( "clone " + ctx.gitRepoURL - , function (release) { - fs.mkdirp(ctx.gitDir, function (err) { + , function gitCloneLockHandler (release) { + fs.mkdirp(ctx.gitDir, function gitCloneMkdirHandler (err) { if (err) return release(err); exec("git clone --mirror " + ctx.gitRepoURL + " " + ctx.gitDir, release); }); diff --git a/lib/tasks/git-fetch.js b/lib/tasks/git-fetch.js index e5b9d6a..f5f870b 100644 --- a/lib/tasks/git-fetch.js +++ b/lib/tasks/git-fetch.js @@ -3,10 +3,10 @@ var exec = require("child_process").exec; // expects: // gitDir: the directory in which the repo is stored -module.exports = function (ctx, cb) { +module.exports = function gitFetch (ctx, cb) { ctx.lock( "fetch " + ctx.gitDir - , function (release) { + , function gitFetchHandler (release) { ctx.log.info("git fetch in " + ctx.gitDir); exec("git fetch", { cwd: ctx.gitDir }, release); } diff --git a/lib/tasks/git-publish.js b/lib/tasks/git-publish.js index 79a506c..a66f866 100644 --- a/lib/tasks/git-publish.js +++ b/lib/tasks/git-publish.js @@ -12,7 +12,7 @@ var fs = require("fs-extra") // copyFile: if defined, the file name to copy over instead of the whole directory module.exports = [ // make a temp directory - function (ctx, cb) { + function gitPublish (ctx, cb) { tmp.dir({ mode: 0755 }, function (err, tmpDir) { if (err) return cb(err); ctx.tmpDir = tmpDir; @@ -20,16 +20,16 @@ module.exports = [ }); } // extract the content - , function (ctx, cb) { + , function gitPublishArchiveHandler (ctx, cb) { var cmd = "git archive " + ctx.branch + " | tar -x -C " + ctx.tmpDir; ctx.log.info("Publish: " + cmd + ", in " + ctx.gitDir); exec(cmd, { cwd: ctx.gitDir }, cb); } // move over the content - , function (ctx, cb) { + , function gitPublishArchiveMover (ctx, cb) { ctx.lock( "move over content to " + ctx.specDir - , function (release) { + , function gitPublishFileLockHandler (release) { var tmp = ctx.tmpDir; delete ctx.tmpDir; if (ctx.copyFile) { @@ -41,7 +41,7 @@ module.exports = [ } , cb ); - + } ] ; diff --git a/lib/tasks/init-all.js b/lib/tasks/init-all.js index 3854aa8..4101720 100644 --- a/lib/tasks/init-all.js +++ b/lib/tasks/init-all.js @@ -5,7 +5,7 @@ var fs = require("fs-extra") ; function makeStaticTask (which) { - return function (ctx, cb) { + return function initAllMakeStatic (ctx, cb) { ctx = this.getContext({ repository: "webspecs/" + which, branch: "master" }); ctx.subDirOnly = which; ctx.specDir = jn(ctx.publishDir, ctx.subDirOnly); @@ -14,10 +14,10 @@ function makeStaticTask (which) { } module.exports = [ - function (ctx, cb) { - "gits publish bikeshed queue logs" + function initAll (ctx, cb) { + "gits publish bikeshed queue logs temp" .split(" ") - .forEach(function (dir) { + .forEach(function initAllDirMaker (dir) { dir = jn(ctx.dataDir, dir); ctx.log.info("Ensuring the existence of " + dir); if (!fs.existsSync(dir)) fs.mkdirSync(dir); @@ -30,7 +30,7 @@ module.exports = [ , require("./init-bikeshed") , require("./init-the-index") , require("./extract-the-index") - , function (ctx, cb) { + , function initAllCallbackHandler (ctx, cb) { ctx.specsToProcess = ctx.theIndexRepositories; cb(); } diff --git a/lib/tasks/init-bikeshed.js b/lib/tasks/init-bikeshed.js index d931ba9..76c7cc4 100644 --- a/lib/tasks/init-bikeshed.js +++ b/lib/tasks/init-bikeshed.js @@ -2,9 +2,9 @@ var jn = require("path").join; module.exports = [ - function (ctx, cb) { + function initBikeshed (ctx, cb) { this.updateContextForRepo(ctx, { repository: "webspecs/bikeshed", branch: "webspecs" }); - ctx.specDir = jn(ctx.dataDir, "bikeshed"); + ctx.specDir = "bikeshed"; cb(); } , require("./git-clone-or-fetch") diff --git a/lib/tasks/init-the-index.js b/lib/tasks/init-the-index.js index aaa2316..10c8bde 100644 --- a/lib/tasks/init-the-index.js +++ b/lib/tasks/init-the-index.js @@ -18,7 +18,5 @@ module.exports = [ ctx.subDirOnly = "index.html"; cb(); } - , require("./rsync") - , require("./purge-all") ] -; +; diff --git a/lib/tasks/process-spec.js b/lib/tasks/process-spec.js index 3516fc7..098fa89 100644 --- a/lib/tasks/process-spec.js +++ b/lib/tasks/process-spec.js @@ -28,7 +28,6 @@ module.exports = [ return this.runTask(require("./generate-bikeshed"), ctx, cb); cb(new Error("Failed to find a specification file in " + ctx.specDir)); } - , require("./rsync") - , require("./purge-all") + , require("./purge") ] ; diff --git a/lib/tasks/process-specs.js b/lib/tasks/process-specs.js index fadc469..342f6c3 100644 --- a/lib/tasks/process-specs.js +++ b/lib/tasks/process-specs.js @@ -3,16 +3,16 @@ var processSpec = require("./process-spec"); // expects // specsToProcess: an array of specs to process much like `theIndexRepositories` -module.exports = function (ctx, cb) { +module.exports = function processSpecs (ctx, cb) { var specs = ctx.specsToProcess.concat(); ctx.log.info("Processing " + specs.length + " specifications."); - var iterate = function () { + var iterate = function processSpecsIterate () { if (!specs.length) return cb(); var it = specs.shift() , ctx = this.getContext(it) ; ctx.subDirOnly = ctx.specSubDir; - this.runTask(processSpec, ctx, function (err) { + this.runTask(processSpec, ctx, function processSpecsRunTask (err) { if (err) ctx.log.error(err); iterate(); }); diff --git a/lib/tasks/process-static.js b/lib/tasks/process-static.js index 52deb41..8daeaea 100644 --- a/lib/tasks/process-static.js +++ b/lib/tasks/process-static.js @@ -2,7 +2,6 @@ module.exports = [ require("./git-clone-or-fetch") , require("./git-publish") - , require("./rsync") , require("./purge-all") ] -; +; diff --git a/lib/tasks/process-the-index.js b/lib/tasks/process-the-index.js index 3765f03..7f47a19 100644 --- a/lib/tasks/process-the-index.js +++ b/lib/tasks/process-the-index.js @@ -1,19 +1,19 @@ module.exports = [ require("./extract-the-index") // first time, get the old values - , function (ctx, cb) { + , function processTheIndexFirst (ctx, cb) { ctx.oldIndexRepositories = ctx.theIndexRepositories; ctx.theIndexRepositories = null; cb(); } , require("./init-the-index") , require("./extract-the-index") // get the updated list - , function (ctx, cb) { + , function processTheIndexSecond (ctx, cb) { ctx.log.info("Finding new specifications to process"); var newRepos = []; ctx.theIndexRepositories - .forEach(function (it) { - var isOld = ctx.oldIndexRepositories.some(function (old) { + .forEach(function processTheIndexRepositoryLooper (it) { + var isOld = ctx.oldIndexRepositories.some(function processTheIndexOldIsser (old) { return old.repository === it.repository && old.branch === it.branch && old.baseFileName === it.baseFileName; diff --git a/lib/tasks/purge-all.js b/lib/tasks/purge-all.js index 56cecd6..27a71c1 100644 --- a/lib/tasks/purge-all.js +++ b/lib/tasks/purge-all.js @@ -1,16 +1,35 @@ -var sua = require("superagent"); +var request = require("request"); // expects: // purgeAllURL: the URL to purge Fastly // purgeAllKey: the key for the Fastly service module.exports = function (ctx, cb) { - sua.post(ctx.purgeAllURL) - .set("Fastly-Key", ctx.purgeAllKey) - .set('Accept', 'application/json') - .end(function (err, res) { - if (!err && res.body && res.body.status === "ok") return cb(); - cb(new Error(res.body ? res.body.status : "No response body for purge")); - }) - ; + + if ( ctx.purgeAllKey === false ) { + return cb(); + } + + var options = { + url: ctx.purgeAllURL + , method: 'POST' + , headers: { + 'User-Agent': 'WebPlatformPublican/1' + , 'Fastly-Key': ctx.purgeAllKey + } + }; + + return request(options, function requestCallback(error, response, body) { + var info = JSON.parse(body); + if (!error && response.statusCode == 200) { + console.log('Purge worked', info); + return cb(); + } else { + var msg = info.msg||'Request was refused by Fastly and the we could not find error message in response'; + console.error('Error purging, its possible the key is invalid, message: ', msg ); + cb( new Error('Error purging ' + msg ) ); + } + }); + }; + diff --git a/lib/tasks/purge.js b/lib/tasks/purge.js new file mode 100644 index 0000000..e679d61 --- /dev/null +++ b/lib/tasks/purge.js @@ -0,0 +1,27 @@ + +var request = require("request"); + +module.exports = function (ctx, cb) { + + var parts = ctx.repository.split("/", 2); + var options = { + url: 'https://specs.webplatform.org/' + , method: 'PURGE' + , headers: { + 'User-Agent': 'WebPlatformPublican/1' + } + }; + + options.url += parts[1] + '/' + parts[0] + '/' + ctx.branch; + + return request(options, function purgeRequestHandler (error, response, body) { + var info = JSON.parse(body); + if (!error && response.statusCode == 200) { + console.log('Purge worked on ' + options.url, info); + return cb(); + } else { + return cb(new Error('Error purging ' + options.url, info)); + } + }); + +}; diff --git a/package.json b/package.json index 61316d2..fea7d27 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "name": "publican", - "version": "0.3.0", + "description": "The set of tools used to automate publishing in the WebSpecs project", + "version": "0.3.1", "license": "MIT", "dependencies": { "async": "0.9.0", @@ -14,10 +15,11 @@ "phantomjs": "1.9.15", "querystring": "^0.2.0", "respec": "3.2.40", - "superagent": "1.1.0", + "request": "2.42.0", "tmp": "0.0.25", "whacko": "0.17.4", "winston": "0.9.0", + "forever": "~0.14", "winston-mail": "0.3.2" }, "bin": {