diff --git a/.gitignore b/.gitignore index 1a3e5eafb2..8ecb1e1dd8 100644 --- a/.gitignore +++ b/.gitignore @@ -2,13 +2,9 @@ .history *~ .*.swp -templates/*html -www/policy.html -www/index.html -www/files -www/styles */archives/* */index.tar.gz */urls.txt -.DS_Store -.history +/coq-packages.json +/_build +/scripts/.merlin diff --git a/Makefile b/Makefile index 6ce8cb037f..3906b8822f 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,9 @@ -COQWEB=~/COQ/www/ -SUITES_COQPKGIDX = released -SUITES= $(SUITES_COQPKGIDX) core-dev extra-dev +SUITES= released core-dev extra-dev H=@ -COQV=8.8.2 -OCAMLV=4.02.3 -pp = (cd $(COQWEB); yamlpp-0.3/yamlpp -l en $(abspath $(1)) -o $(abspath $(2))) - -ifeq "$(shell test ! -z '$(COQWEB)' -a -d $(COQWEB) || echo false)" "false" -$(error "Please use 'make COQWEB=path/to/coq/www'") -endif - -# refresh opam indexes + generate website -all: check-deps - @./scripts/refresh-opam-indexes $(SUITES) - -run: all - $(H)echo "Starting a local web server for test" - $(H)echo "It is accessible at: http://localhost:8000" - $(H)cd www && python -m SimpleHTTPServer 8000 - -check-deps: \ - which-opam which-lua5.1 opam-config which-markdown yamlpp - -yamlpp: - $(H)ls $(COQWEB)/yamlpp-0.3/yamlpp > /dev/null || (echo "Cannot find yamlpp. Please build the website first"; false) +# refresh opam indexes +all: which-opam opam-config + $(H)./scripts/refresh-opam-indexes $(SUITES) which-%: $(H)which $* > /dev/null || (echo "Please install $*"; false) diff --git a/README.md b/README.md index 520277ea52..13e86dc15b 100644 --- a/README.md +++ b/README.md @@ -24,26 +24,12 @@ To activate the repositories: opam repo add coq-core-dev https://coq.inria.fr/opam/core-dev ``` -## Website preprocessor - -We follow the model of the Coq website. -One should invoke `make COQWEB=path/to/coq/www` to generate the web pages -using the same header, footer and `yamlpp` used by the Coq website (it is expected to be in `path/to/coq/www/yamlpp-0.3/yamlpp`. The -destination folder is `www/`. - -The templates are in `templates/`. The file `index.html.in` is first -processed by `scripts/archive2web` that fills in `` entries, then -`yamlpp` is used to insert the header and footer. - -The code in `www/filter.js` is used to interactively browse the contents -of the packages table in `index.html`. The css file -`www/index-style.css` is also part of the picture. - ## Website and OPAM metadata -The website is statically generated looking at the `opam` files. +The `scripts/archive2web.ml` program generates a JSON file +`coq-packages.json` by looking at the `opam` files. -In particular we use the `tags` field of the `opam` file as follows: +In particular, it uses the `tags` field of an `opam` file as follows: 1. strings beginning with `keyword:` are considered as `keywords` 2. strings beginning with `category:` are considered as `categories` @@ -63,9 +49,13 @@ tags: [ ] ``` -Finally the `homepage:`, `author:`, `maintainer:` and `doc:` fields are +The `homepage:`, `author:`, `maintainer:`, and `doc:` fields are also used to generate the package entry. +This JSON file is generated during continuous integration and copied to +the website. Some JavaScript code on the website then loads it to +dynamically generate the content of the webpage on client side. + See also [CEP3](https://github.com/coq/ceps/blob/master/text/003-opam-metadata.md) and the [deployed website](https://coq.inria.fr/opam/www/). diff --git a/scripts/archive2web b/scripts/archive2web deleted file mode 100755 index 87b225587a..0000000000 --- a/scripts/archive2web +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/lua5.1 --- requires dpkg and lua-filesystem - -function add_uniq(set, e) - if e and not(set[e]) then - set[e]=true - set[#set+1]=e - end -end - -require "lfs" - -function not_dot(x) return not(x:match("^%.")) end - -templ=arg[1] or error "1st argument: the template file" -_=arg[2] or error "following arguments: the repo root path(s)" -table.remove(arg,1) - -packages = {} - -function do_one_root(suite) - root = suite .. '/packages/' - for p in lfs.dir(root) do if not_dot(p) then - local pn, v = p:match('^([^%.]*)%.(.*)$') - if v then p = pn end - packages[p] = packages[p] or {} - local info = packages[p] - - info.suites = info.suites or {} - add_uniq(info.suites, suite) - - local versions = info.versions or {} - local curv - if v then --simply a package - curv = v - add_uniq(versions, v) - else -- a dir of packages - for pv in lfs.dir(root .. p .. '/') do if not_dot(pv) then - local v = assert(pv:match('^[^%.]*%.(.*)$'), - root .. p .. '/' .. pv .. " not pkg.version") - add_uniq(versions, v) - curv = v - end end - assert(curv,"Directory "..root .. p .." is odd") - end - table.sort(versions,function(a,b) - return 0 == os.execute('dpkg --compare-versions '..a..' gt '..b.. - ' 2>/dev/null') - end) - info.versions = versions - - local pkg - if v then pkg = root..p..'.'..curv - else pkg = root..p..'/'..p..'.'..curv end - - local opam = assert(io.open(pkg..'/opam')):read('*a') - local tags = opam:match('tags:%s*(%b[])') or "" - - local keywords = info.keywords or {} - local dates = info.dates or {} - local categories = info.categories or {} - for tag in tags:gmatch('(%b"")') do - add_uniq(categories, tag:match('"category: *([^"]*)"')) - add_uniq(keywords, tag:match('"keyword: *([^"]*)"')) - add_uniq(dates, tag:match('"date: *([^"]*)"')) - end - info.categories = categories - info.keywords = keywords - info.dates = dates - info.homepage = info.homepage or opam:match('homepage:%s*"([^"]*)"') - - local authors_txt = opam:match('authors:%s*(%b[])') or "" - local authors = info.authors or {} - for author in authors_txt:gmatch('(%b"")') do - add_uniq(authors, author:match('"([^<"]*)'):gsub(' *$','')) - end - info.authors = authors - - info.description = info.description or - opam:match('description:%s*"""%s*(.-)%s*"""') or - opam:match('description:%s*"([^"]*)"') or - opam:match('synopsis:%s*"([^"]*)"') or - assert(io.open(pkg..'/descr')):read('*a') - - end end -end - -for _, r in ipairs(arg) do - do_one_root(r) -end - -function plural(s,n,...) - if n >= 2 then return string.format(s,"s",...) - else return string.format(s,"",...) end -end -function pluraly(s,n,...) - if n >= 2 then return string.format(s,"ies",...) - else return string.format(s,"y",...) end -end - -function print_pkg(name,info,print) - print [[]] - if info.homepage ~= "" then print((''):format(info.homepage or "#")) end - print (([[

%s

]]):format(name)) - if info.homepage ~= "" then print "" end - print (([[

%s

]]):format(info.description)) - print [[
]] - if #info.authors > 0 then - print(plural('
author%s: ',#info.authors)) - for _,k in ipairs(info.authors) do - print (([[%s]]):format(k)) - end - print [[
]] - end - if #info.dates > 0 then - print [[
date: ]] - for _,k in ipairs(info.dates) do - print (([[%s]]):format(k)) - end - print [[
]] - end - if #info.categories > 0 then - print(pluraly('
categor%s: ',#info.categories)) - for _,k in ipairs(info.categories) do - print (([[%s]]):format(k)) - end - print [[
]] - end - if #info.keywords > 0 then - print(plural('
keyword%s: ',#info.keywords)) - for _,k in ipairs(info.keywords) do - print (([[%s]]):format(k)) - end - print [[
]] - end - print(plural('
version%s: ',#info.versions)) - for _,k in ipairs(info.versions) do - print (([[%s]]):format(k)) - end - print [[
]] - print(plural('
suite%s: ',#info.suites)) - for _,k in ipairs(info.suites) do - print (([[%s]]):format(k)) - end - print [[
]] - print [[
]] - print "\n" -end - -template = io.open(templ):read('*a') -output_fname = templ:gsub('%.in$','') -assert(template ~= output_fname, "template not ending in .in") ---print("Generating "..output_fname) -output = io.open(output_fname,'w') -output:write((template:gsub('@@PACKAGES@@', function() - local t = {} - local function print(s) t[#t+1] = s end - pkgs = {} - for p in pairs(packages) do table.insert(pkgs, p) end - table.sort(pkgs) - for _, p in ipairs(pkgs) do print_pkg(p,packages[p],print) end - return table.concat(t) -end))) - --- vim:set ts=2: diff --git a/scripts/description2json.py b/scripts/description2json.py deleted file mode 100755 index 16a955a1f4..0000000000 --- a/scripts/description2json.py +++ /dev/null @@ -1,143 +0,0 @@ -#! /usr/bin/python -# coding=utf8 - -import sys -import os -import glob -import re -import string -import subprocess -import shutil -import tarfile -import pickle -import simplejson as json - -# -# Parsing description files -# - -# The keys occurring in description files -NAMEKEY='Name' -URLKEY = 'Url' -AUTHORKEY = 'Author' -EMAILKEY = 'Email' -HOMEPAGEKEY = 'Homepage' -ADDRESSKEY = 'Address' -INSTKEY = 'Institution' -LICKEY = 'License' -DESCKEY = 'Description' -KWKEY = 'Keywords' -TITLEKEY = 'Title' -CATKEY = 'Category' -REFKEY = 'References' -REQUIREKEY = 'Require' -AVAILABILITYKEY = 'Availability' - -# Name of usual encodings: -UTF8 = 'utf-8' -LATIN = 'iso-8859-15' - -# 'Normalization' of strings -def recode(s): - try: - us = s.decode(UTF8) - except: - us = s.decode(LATIN) - return(us) - -def parse_description(dir): - """ Parses a description file and returns a dictionary associating a - field name to a string """ - - desc_file = os.path.join(dir, 'description') - fdesc = open(desc_file) - - res={} - res[AUTHORKEY] = {} - - exp = '^([a-zA-Z]+)[ \t]*:[ \t]*(.*)$' - - last_key="" - last_authors=[] - for line in fdesc.readlines(): - lg = recode(line) - - m = re.match(exp, lg) - if m != None: - key = m.group(1) - rem = m.group(2) - - if key in [KWKEY, CATKEY, REQUIREKEY]: - # rem may be a list of strings - l = [] - for e in rem.split(','): - f = string.strip(e) - if f != '': - if key == KWKEY: - # we perform some 'normalization' - f = f.lower() - f = f.replace('-', ' ') - f = f.replace('_', ' ') - l.append(f) - - # CATKEY, KWKEY and REQUIREKEY may appear several times - if res.has_key(key): - res[key].extend(l) - else: - res[key] = l - - elif key == AUTHORKEY: - # TODO: parse author lines of the form 'Firstname Lastname ' - last_authors.append(rem) - res[AUTHORKEY][rem] = {} - elif key in [INSTKEY, EMAILKEY, HOMEPAGEKEY, ADDRESSKEY]: - # One INSTKEY may reports to many authors - for a in last_authors: - if not res[AUTHORKEY][a].has_key(key): - res[AUTHORKEY][a][key] = rem - if key != INSTKEY: - break - elif key == URLKEY: - # Check if rem if a valid url - if re.match('http://', rem) != None: - res[key] = rem - else: - if key == DESCKEY: - res[key] = rem + '\n' - else: - res[key] = rem - - last_key = key - else: - # This is a continuation line, appends it to the last key - if last_key != "": - if last_key in [KWKEY, CATKEY, REQUIREKEY]: - # rem may be a list of strings - for e in lg.split(','): - f = string.strip(e) - if f != '': - if key == KWKEY: - # we perform some 'normalization' - f = f.lower() - f = f.replace('-', ' ') - f = f.replace('_', ' ') - res[last_key].append(f) - elif (last_key in [INSTKEY, ADDRESSKEY]): - auttab = res[AUTHORKEY][last_authors[len(last_authors)-1]] - if auttab.has_key(last_key): - auttab[last_key] = auttab[last_key] + ' ' + lg - elif not (last_key in [AUTHORKEY, EMAILKEY, HOMEPAGEKEY]): - res[last_key] = res[last_key] + ' ' + lg - - fdesc.close() - return res - -def descr_to_json(d): - x = parse_description(d) - print(json.dumps(x)) - -if __name__ == '__main__': - sys.exit(descr_to_json(sys.argv[1])) - -#if __name__ == '__main__': -# sys.exit(main()) diff --git a/scripts/json2opamauthor.lua b/scripts/json2opamauthor.lua deleted file mode 100755 index 0836695f4b..0000000000 --- a/scripts/json2opamauthor.lua +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/lua5.1 - -require "json" - -t = json.decode(io.open(arg[1],'r'):read('*a')) - -authors = {} - -for name, data in pairs(t.Author) do - authors[#authors+1] = - ('"%s <%s>"'):format(name,data.Email or data.Homepage or "") -end - -if #authors > 0 then - print(string.format("authors: [ %s ]",table.concat(authors,' '))) -end diff --git a/scripts/json2opamdescr.lua b/scripts/json2opamdescr.lua deleted file mode 100755 index 36c9bbd46c..0000000000 --- a/scripts/json2opamdescr.lua +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/lua5.1 - -require "json" - -t = json.decode(io.open(arg[1],'r'):read('*a')) - -print(((t.Title .. "\n\n" .. t.Description):gsub(' *\n[ \t][ \t]*','\n'):gsub('[ \t][ \t]*',' '):gsub('[\n ]*$',''))) diff --git a/scripts/json2opamtags.lua b/scripts/json2opamtags.lua deleted file mode 100755 index 4665e0456a..0000000000 --- a/scripts/json2opamtags.lua +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/lua5.1 - -require "json" - -t = json.decode(io.open(arg[1],'r'):read('*a')) - -tags = {} - -for k,v in pairs(t.Keywords or {}) do - tags[#tags+1] = string.format('"keyword:%s"',v:gsub('%.*$','')) -end -for k,v in pairs(t.Category or {}) do - tags[#tags+1] = string.format('"category:%s"',v:gsub('%.*$','')) -end -if t.Date then tags[#tags+1] = string.format('"date:%s"',t.Date:gsub('[\n ]*$','')) end - -if #tags > 0 then - print(string.format("tags: [ %s ]", table.concat(tags," "))) -end diff --git a/templates/index.html.in b/templates/index.html.in deleted file mode 100644 index 8dc77afcd1..0000000000 --- a/templates/index.html.in +++ /dev/null @@ -1,64 +0,0 @@ -<#def TITLE>Coq Package Index -<#def HEAD> - - - -<#include "incl/header.html"> -
- -
-
- Active filters -
-
- -
-
-
-
- -
-
- Available suites -
-
-
-
-
- -
-
- Available categories -
-
-
-
-
- -
-
- Available Keywords -
-
-
-
-
-
- -
- -
-
- - - @@PACKAGES@@ - -
-
- -
- - - - -<#include "incl/footer.html">