diff --git a/.dockerignore b/.dockerignore
index f7d90d4e531..28c6753f599 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,7 @@
+*~
+.dockerignore
.hg
+Dockerfile
# Remove the git objects, logs, etc. to make final image smaller.
# Some files still need to be in the .git directory, because Etherpad at
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 4906ca04fc2..dd84ea7824f 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,12 +1,11 @@
-* * *
-
+---
name: Bug report
about: Create a report to help us improve
title: ''
-labels: bug
-assignees:
+labels: ''
+assignees: ''
-* * *
+---
**Describe the bug**
A clear and concise description of what the bug is.
@@ -24,23 +23,16 @@ A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
-**Environment (please complete the following information):**
-
-- Etherpad Version: (e.g. 1.8.0)
-- Deployment (manual install, docker, ...)
-
**Desktop (please complete the following information):**
-
-- OS: (e.g. iOS)
-- Browser (e.g. chrome, safari)
-- Version (e.g. 22)
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
**Smartphone (please complete the following information):**
-
-- Device: (e.g. iPhone6)
-- OS: (e.g. iOS8.1)
-- Browser (e.g. stock browser, safari)
-- Version (e.g. 22)
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
**Additional context**
Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
index 41867e2701a..9f8fe9f89ec 100644
--- a/.github/ISSUE_TEMPLATE/feature_request.md
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -1,3 +1,12 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: Feature Request
+assignees: ''
+
+---
+
* * *
name: Feature request
@@ -19,3 +28,6 @@ A clear and concise description of any alternative solutions or features you've
**Additional context**
Add any other context or screenshots about the feature request here.
+
+**Plugin?**
+Might this feature be better suited to being a plugin? Usually features that can be plugins, should be.
diff --git a/.github/ISSUE_TEMPLATE/plugin-request-template.md b/.github/ISSUE_TEMPLATE/plugin-request-template.md
new file mode 100644
index 00000000000..59997398199
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/plugin-request-template.md
@@ -0,0 +1,30 @@
+---
+name: Plugin request template
+about: Suggest a plugin for Etherpad
+title: ''
+labels: Plugin Request
+assignees: JohnMcLear
+
+---
+
+* * *
+
+name: Plugin request
+about: Suggest a plugin for this project
+title: ''
+labels: plugin request
+assignees:
+
+* * *
+
+**Is your plugin request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when (...)
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the plugin request here.
diff --git a/.github/ISSUE_TEMPLATE/security-issue.md b/.github/ISSUE_TEMPLATE/security-issue.md
new file mode 100644
index 00000000000..c8b5067030a
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/security-issue.md
@@ -0,0 +1,10 @@
+---
+name: Security issue
+about: Notify the Etherpad foundation of a Security issue
+title: ''
+labels: security
+assignees: ''
+
+---
+
+Please email contact@etherpad.org with details of the security issue prior to posting here.
diff --git a/.github/ISSUE_TEMPLATE/security.md b/.github/ISSUE_TEMPLATE/security.md
new file mode 100644
index 00000000000..b2e6824ebcd
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/security.md
@@ -0,0 +1,13 @@
+* * *
+
+name: Security notification
+about: Disclose a security issue in Etherpad
+title: ''
+labels: security
+assignees:
+
+* * *
+
+**Our Security disclosure process**
+1. Please email contact@etherpad.org with detials of the exploit including steps to replicate.
+1. Once confirmed we will provide a confirmation, patch and CVE details.
diff --git a/.github/stale.yml b/.github/stale.yml
new file mode 100644
index 00000000000..2e530c6c097
--- /dev/null
+++ b/.github/stale.yml
@@ -0,0 +1,23 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 60
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+# Issues with these labels will never be considered stale
+exemptLabels:
+ - pinned
+ - security
+ - Bug
+ - Serious Bug
+ - Minor bug
+ - Black hole bug
+ - Special case Bug
+ - Upstream bug
+# Label to use when marking an issue as stale
+staleLabel: wontfix
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+ This issue has been automatically marked as stale because it has not had
+ recent activity. It will be closed if no further activity occurs. Thank you
+ for your contributions.
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 00000000000..f3b1cf2c27c
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,54 @@
+name: "CodeQL"
+
+on:
+ push:
+ branches: [develop, master]
+ pull_request:
+ # The branches below must be a subset of the branches above
+ branches: [develop]
+ schedule:
+ - cron: '0 13 * * 1'
+
+jobs:
+ analyze:
+ name: Analyze
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ # We must fetch at least the immediate parents so that if this is
+ # a pull request then we can checkout the head.
+ fetch-depth: 2
+
+ # If this run was triggered by a pull request event, then checkout
+ # the head of the pull request instead of the merge commit.
+ - run: git checkout HEAD^2
+ if: ${{ github.event_name == 'pull_request' }}
+
+ # Initializes the CodeQL tools for scanning.
+ - name: Initialize CodeQL
+ uses: github/codeql-action/init@v1
+ # Override language selection by uncommenting this and choosing your languages
+ # with:
+ # languages: go, javascript, csharp, python, cpp, java
+
+ # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
+ # If this step fails, then you should remove it and run the build manually (see below)
+ - name: Autobuild
+ uses: github/codeql-action/autobuild@v1
+
+ # âšī¸ Command-line programs to run using the OS shell.
+ # đ https://git.io/JvXDl
+
+ # âī¸ If the Autobuild fails above, remove it and uncomment the following three lines
+ # and modify them (or add more) to build your code if your project
+ # uses a compiled language
+
+ #- run: |
+ # make bootstrap
+ # make release
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v1
diff --git a/.travis.yml b/.travis.yml
index 6b3bb6c7c78..7b0ed03aef7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,13 +6,17 @@ node_js:
services:
- docker
+cache: false
+
+before_install:
+ - sudo add-apt-repository -y ppa:libreoffice/ppa
+ - sudo apt-get update
+ - sudo apt-get -y install libreoffice
+ - sudo apt-get -y install libreoffice-pdfimport
+
install:
- "bin/installDeps.sh"
- "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
- - "npm install ep_test_line_attrib"
-
-before_script:
- - "tests/frontend/travis/sauce_tunnel.sh"
script:
- "tests/frontend/travis/runner.sh"
@@ -24,25 +28,39 @@ env:
jobs:
include:
+ # we can only frontend tests from the ether/ organization and not from forks.
+ # To request tests to be run ask a maintainer to fork your repo to ether/
+ - if: fork = false
+ name: "Test the Frontend"
+ install:
+ #FIXME
+ - "sed 's/\"loglevel\": \"INFO\",/\"loglevel\": \"WARN\",/g' settings.json.template > settings.json"
+ - "tests/frontend/travis/sauce_tunnel.sh"
+ - "bin/installDeps.sh"
+ - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+ script:
+ - "tests/frontend/travis/runner.sh"
- name: "Run the Backend tests"
install:
+ - "bin/installDeps.sh"
- "cd src && npm install && cd -"
script:
- "tests/frontend/travis/runnerBackend.sh"
- - name: "Test the Frontend"
+## Temporarily commented out the Dockerfile tests
+# - name: "Test the Dockerfile"
+# install:
+# - "cd src && npm install && cd -"
+# script:
+# - "docker build -t etherpad:test ."
+# - "docker run -d -p 9001:9001 etherpad:test && sleep 3"
+# - "cd src && npm run test-container"
+ - name: "Load test Etherpad"
install:
- "bin/installDeps.sh"
- - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
- - "npm install ep_test_line_attrib"
- script:
- - "tests/frontend/travis/runner.sh"
- - name: "Test the Dockerfile"
- install:
- "cd src && npm install && cd -"
+ - "npm install -g etherpad-load-test"
script:
- - "docker build -t etherpad:test ."
- - "docker run -d -p 9001:9001 etherpad:test && sleep 3"
- - "cd src && npm run test-container"
+ - "tests/frontend/travis/runnerLoadTest.sh"
notifications:
irc:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 392611e4077..28bc0d2d814 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,47 @@
+# Develop -- TODO Change to 1.8.x.
+* ...
+
+# 1.8.5
+* IMPORTANT DROP OF SUPPORT: Drop support for IE. Browsers now need async/await.
+* IMPORTANT SECURITY: Rate limit Commits when env=production
+* SECURITY: Non completed uploads no longer crash Etherpad
+* SECURITY: Log authentication requests
+* FEATURE: Support ES6 (migrate from Uglify-JS to Terser)
+* FEATURE: Improve support for non-cookie enabled browsers
+* FEATURE: New hooks for ``index.html``
+* FEATURE: New script to delete sessions.
+* FEATURE: New setting to allow import withing an author session on a pad
+* FEATURE: Checks Etherpad version on startup and notifies if update is available. Also available in ``/admin`` interface.
+* FEATURE: Timeslider updates pad location to most recent edit
+* MINOR: Outdent UL/LI items on removal of list item
+* MINOR: Various UL/LI import/export bugs
+* MINOR: PDF export fix
+* MINOR: Front end tests no longer run (and subsequently error) on pull requests
+* MINOR: Fix issue with closing a list before it opens
+* MINOR: Fix bug where large pads would fire a console error in timeslider
+* MINOR: Fix ?showChat URL param issue
+* MINOR: Issue where timeslider URI fails to be correct if padID is numeric
+* MINOR: Include prompt for clear authorship when entire document is selected
+* MINOR: Include full document aText every 100 revisions to make pad restoration on database curruption achievable
+* MINOR: Several Colibris CSS fixes
+* MINOR: Use mime library for mime types instead of hard-coded.
+* MINOR: Don't show "new pad button" if instance is read only
+* MINOR: Use latest NodeJS when doing Windows build
+* MINOR: Change disconnect logic to reconnect instead of silently failing
+* MINOR: Update SocketIO, async, jQuery and Mocha which were stuck due to stale code.
+* MINOR: Rewrite the majority of the ``bin`` scripts to use more modern syntax
+* MINOR: Improved CSS anomation through prefers-reduced-motion
+* PERFORMANCE: Use workers (where possible) to minify CSS/JS on first page request. This improves initial startup times.
+* PERFORMANCE: Cache EJS files improving page load speed when maxAge > 0.
+* PERFORMANCE: Fix performance for large pads
+* TESTS: Additional test coverage for OL/LI/Import/Export
+* TESTS: Include Simulated Load Testing in CI.
+* TESTS: Include content collector tests to test contentcollector.js logic external to pad dependents.
+* TESTS: Include fuzzing import test.
+* TESTS: Ensure CI is no longer using any cache
+* TESTS: Fix various tests...
+* TESTS: Various additional Travis testing including libreoffice import/export
+
# 1.8.4
* FIX: fix a performance regression on MySQL introduced in 1.8.3
* FIX: when running behind a reverse proxy and exposed in an inner directory, fonts and toolbar icons should now be visible. This is a regression introduced in 1.8.3
diff --git a/Dockerfile b/Dockerfile
index 6e6a7c7d407..45601c8764f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,7 +42,7 @@ RUN bin/installDeps.sh && \
#
# Bash trick: in the for loop ${ETHERPAD_PLUGINS} is NOT quoted, in order to be
# able to split at spaces.
-RUN for PLUGIN_NAME in ${ETHERPAD_PLUGINS}; do npm install "${PLUGIN_NAME}"; done
+RUN for PLUGIN_NAME in ${ETHERPAD_PLUGINS}; do npm install "${PLUGIN_NAME}" || exit 1; done
# Copy the configuration file.
COPY --chown=etherpad:0 ./settings.json.docker /opt/etherpad-lite/settings.json
diff --git a/README.md b/README.md
index 5af71ef3c05..cf319d32225 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,8 @@
# A real-time collaborative editor for the web
-[![Travis (.org)](https://img.shields.io/travis/ether/etherpad-lite)](https://travis-ci.org/github/ether/etherpad-lite)
+
+[![Travis (.org)](https://api.travis-ci.org/ether/etherpad-lite.svg?branch=develop)](https://travis-ci.org/github/ether/etherpad-lite)
+
![Demo Etherpad Animated Jif](doc/images/etherpad_demo.gif "Etherpad in action")
# About
@@ -92,6 +94,13 @@ If you prefer, `ep_hash_auth` also gives you the option of storing the users in
Etherpad is very customizable through plugins. Instructions for installing themes and plugins can be found in [the plugin wiki article](https://github.com/ether/etherpad-lite/wiki/Available-Plugins).
+## Getting the full features
+Run the following command in your Etherpad folder to get all of the features visible in the demo gif:
+
+```
+npm install ep_headings2 ep_markdown ep_comments_page ep_align ep_page_view ep_font_color ep_webrtc ep_embedded_hyperlinks2
+```
+
## Customize the style with skin variants
Open in your browser and start playing !
@@ -130,10 +139,12 @@ OpenAPI (previously swagger) definitions for the API are exposed under `/api/ope
There is a [jQuery plugin](https://github.com/ether/etherpad-lite-jquery-plugin) that helps you to embed Pads into your website.
# Plugin Framework
-Etherpad offers a plugin framework, allowing you to easily add your own features. By default your Etherpad is extremely light-weight and it's up to you to customize your experience. Once you have Etherpad installed you should visit the plugin page and take control.
+Etherpad offers a plugin framework, allowing you to easily add your own features. By default your Etherpad is extremely light-weight and it's up to you to customize your experience. Once you have Etherpad installed you should [visit the plugin page](https://static.etherpad.org/) and take control.
# Translations / Localizations (i18n / l10n)
-Etherpad comes with translations into all languages thanks to the team at TranslateWiki.
+Etherpad comes with translations into all languages thanks to the team at [TranslateWiki](https://translatewiki.net/).
+
+If you require translations in [plugins](https://static.etherpad.org/) please send pull request to each plugin individually.
# FAQ
Visit the **[FAQ](https://github.com/ether/etherpad-lite/wiki/FAQ)**.
diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 00000000000..3e7857be679
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please email contact@etherpad.org to report security related issues.
diff --git a/bin/buildForWindows.sh b/bin/buildForWindows.sh
index 3d6b61882f6..818522ad07f 100755
--- a/bin/buildForWindows.sh
+++ b/bin/buildForWindows.sh
@@ -1,39 +1,29 @@
#!/bin/sh
-NODE_VERSION="10.20.1"
-
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
-
-#Was this script started in the bin folder? if yes move out
-if [ -d "../bin" ]; then
- cd "../"
-fi
-
-#Is wget installed?
-hash wget > /dev/null 2>&1 || {
- echo "Please install wget" >&2
- exit 1
-}
-
-#Is zip installed?
-hash zip > /dev/null 2>&1 || {
- echo "Please install zip" >&2
- exit 1
-}
-
-#Is zip installed?
-hash unzip > /dev/null 2>&1 || {
- echo "Please install unzip" >&2
- exit 1
-}
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
+
+# Is wget installed?
+is_cmd wget || fatal "Please install wget"
+
+# Is zip installed?
+is_cmd zip || fatal "Please install zip"
+
+# Is zip installed?
+is_cmd unzip || fatal "Please install unzip"
START_FOLDER=$(pwd);
TMP_FOLDER=$(mktemp -d)
-echo "create a clean environment in $TMP_FOLDER..."
-cp -ar . $TMP_FOLDER
-cd $TMP_FOLDER
+log "create a clean environment in $TMP_FOLDER..."
+cp -ar . "$TMP_FOLDER"
+cd "$TMP_FOLDER"
rm -rf node_modules
rm -f etherpad-lite-win.zip
@@ -41,33 +31,33 @@ rm -f etherpad-lite-win.zip
# making the windows package smaller
export NODE_ENV=production
-echo "do a normal unix install first..."
+log "do a normal unix install first..."
bin/installDeps.sh || exit 1
-echo "copy the windows settings template..."
+log "copy the windows settings template..."
cp settings.json.template settings.json
-echo "resolve symbolic links..."
+log "resolve symbolic links..."
cp -rL node_modules node_modules_resolved
rm -rf node_modules
mv node_modules_resolved node_modules
-echo "download windows node..."
+log "download windows node..."
cd bin
-wget "https://nodejs.org/dist/v$NODE_VERSION/win-x86/node.exe" -O ../node.exe
+wget "https://nodejs.org/dist/latest-erbium/win-x86/node.exe" -O ../node.exe
-echo "remove git history to reduce folder size"
+log "remove git history to reduce folder size"
rm -rf .git/objects
-echo "remove windows jsdom-nocontextify/test folder"
-rm -rf $TMP_FOLDER/src/node_modules/wd/node_modules/request/node_modules/form-data/node_modules/combined-stream/test
-rm -rf $TMP_FOLDER/src/node_modules/nodemailer/node_modules/mailcomposer/node_modules/mimelib/node_modules/encoding/node_modules/iconv-lite/encodings/tables
+log "remove windows jsdom-nocontextify/test folder"
+rm -rf "$TMP_FOLDER"/src/node_modules/wd/node_modules/request/node_modules/form-data/node_modules/combined-stream/test
+rm -rf "$TMP_FOLDER"/src/node_modules/nodemailer/node_modules/mailcomposer/node_modules/mimelib/node_modules/encoding/node_modules/iconv-lite/encodings/tables
-echo "create the zip..."
-cd $TMP_FOLDER
-zip -9 -r $START_FOLDER/etherpad-lite-win.zip ./*
+log "create the zip..."
+cd "$TMP_FOLDER"
+zip -9 -r "$START_FOLDER"/etherpad-lite-win.zip ./*
-echo "clean up..."
-rm -rf $TMP_FOLDER
+log "clean up..."
+rm -rf "$TMP_FOLDER"
-echo "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"
+log "Finished. You can find the zip in the Etherpad root folder, it's called etherpad-lite-win.zip"
diff --git a/bin/convert.js b/bin/convert.js
index 69bb2cbc7e7..82e0f757919 100644
--- a/bin/convert.js
+++ b/bin/convert.js
@@ -349,7 +349,7 @@ function convertPad(padId, callback)
//generate new author values
var authorID = "a." + randomString(16);
- var authorColorID = authors[i].colorId || Math.floor(Math.random()*32);
+ var authorColorID = authors[i].colorId || Math.floor(Math.random()*(exports.getColorPalette().length));
var authorName = authors[i].name || null;
//overwrite the authorID of the attribute pool
diff --git a/bin/createUserSession.js b/bin/createUserSession.js
new file mode 100644
index 00000000000..0cba0d8e3e9
--- /dev/null
+++ b/bin/createUserSession.js
@@ -0,0 +1,47 @@
+/*
+ * A tool for generating a test user session which can be used for debugging configs
+ * that require sessions.
+ */
+const m = (f) => __dirname + '/../' + f;
+
+const fs = require('fs');
+const path = require('path');
+const querystring = require('querystring');
+const request = require(m('src/node_modules/request'));
+const settings = require(m('src/node/utils/Settings'));
+const supertest = require(m('src/node_modules/supertest'));
+
+(async () => {
+ const api = supertest('http://'+settings.ip+':'+settings.port);
+
+ const filePath = path.join(__dirname, '../APIKEY.txt');
+ const apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+
+ let res;
+
+ res = await api.get('/api/');
+ const apiVersion = res.body.currentVersion;
+ if (!apiVersion) throw new Error('No version set in API');
+ const uri = (cmd, args) => `/api/${apiVersion}/${cmd}?${querystring.stringify(args)}`;
+
+ res = await api.post(uri('createGroup', {apikey}));
+ if (res.body.code === 1) throw new Error(`Error creating group: ${res.body}`);
+ const groupID = res.body.data.groupID;
+ console.log('groupID', groupID);
+
+ res = await api.post(uri('createGroupPad', {apikey, groupID}));
+ if (res.body.code === 1) throw new Error(`Error creating group pad: ${res.body}`);
+ console.log('Test Pad ID ====> ', res.body.data.padID);
+
+ res = await api.post(uri('createAuthor', {apikey}));
+ if (res.body.code === 1) throw new Error(`Error creating author: ${res.body}`);
+ const authorID = res.body.data.authorID;
+ console.log('authorID', authorID);
+
+ const validUntil = Math.floor(new Date() / 1000) + 60000;
+ console.log('validUntil', validUntil);
+ res = await api.post(uri('createSession', {apikey, groupID, authorID, validUntil}));
+ if (res.body.code === 1) throw new Error(`Error creating session: ${res.body}`);
+ console.log('Session made: ====> create a cookie named sessionID and set the value to',
+ res.body.data.sessionID);
+})();
diff --git a/bin/debugRun.sh b/bin/debugRun.sh
index 246d5390035..d9b18aaa24d 100755
--- a/bin/debugRun.sh
+++ b/bin/debugRun.sh
@@ -1,20 +1,15 @@
#!/bin/sh
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
-#Was this script started in the bin folder? if yes move out
-if [ -d "../bin" ]; then
- cd "../"
-fi
-
-#Prepare the environment
+# Prepare the environment
bin/installDeps.sh || exit 1
echo "If you are new to debugging Node.js with Chrome DevTools, take a look at this page:"
echo "https://medium.com/@paul_irish/debugging-node-js-nightlies-with-chrome-devtools-7c4a1b95ae27"
echo "Open 'chrome://inspect' on Chrome to start debugging."
-#Use 0.0.0.0 to allow external connections to the debugger
-#(ex: running Etherpad on a docker container). Use default port # (9229)
+# Use 0.0.0.0 to allow external connections to the debugger
+# (ex: running Etherpad on a docker container). Use default port # (9229)
node --inspect=0.0.0.0:9229 node_modules/ep_etherpad-lite/node/server.js "$@"
diff --git a/bin/deleteAllGroupSessions.js b/bin/deleteAllGroupSessions.js
new file mode 100644
index 00000000000..cda4a3a59a9
--- /dev/null
+++ b/bin/deleteAllGroupSessions.js
@@ -0,0 +1,51 @@
+/*
+* A tool for deleting ALL GROUP sessions Etherpad user sessions from the CLI,
+* because sometimes a brick is required to fix a face.
+*/
+
+const request = require('../src/node_modules/request');
+const settings = require(__dirname+'/../tests/container/loadSettings').loadSettings();
+const supertest = require(__dirname+'/../src/node_modules/supertest');
+const api = supertest('http://'+settings.ip+":"+settings.port);
+const path = require('path');
+const fs = require('fs');
+
+// get the API Key
+var filePath = path.join(__dirname, '../APIKEY.txt');
+var apikey = fs.readFileSync(filePath, {encoding: 'utf-8'});
+
+// Set apiVersion to base value, we change this later.
+var apiVersion = 1;
+var guids;
+
+// Update the apiVersion
+api.get('/api/')
+.expect(function(res){
+ apiVersion = res.body.currentVersion;
+ if (!res.body.currentVersion) throw new Error("No version set in API");
+ return;
+})
+.then(function(){
+ let guri = '/api/'+apiVersion+'/listAllGroups?apikey='+apikey;
+ api.get(guri)
+ .then(function(res){
+ guids = res.body.data.groupIDs;
+ guids.forEach(function(groupID){
+ let luri = '/api/'+apiVersion+'/listSessionsOfGroup?apikey='+apikey + "&groupID="+groupID;
+ api.get(luri)
+ .then(function(res){
+ if(res.body.data){
+ Object.keys(res.body.data).forEach(function(sessionID){
+ if(sessionID){
+ console.log("Deleting", sessionID);
+ let duri = '/api/'+apiVersion+'/deleteSession?apikey='+apikey + "&sessionID="+sessionID;
+ api.post(duri); // deletes
+ }
+ })
+ }else{
+ // no session in this group.
+ }
+ })
+ })
+ })
+})
diff --git a/bin/deletePad.js b/bin/deletePad.js
index 7745e1767b2..2ce82f8a428 100644
--- a/bin/deletePad.js
+++ b/bin/deletePad.js
@@ -4,7 +4,7 @@
*/
const request = require('../src/node_modules/request');
-const settings = require(__dirname+'/../tests/backend/loadSettings').loadSettings();
+const settings = require(__dirname+'/../tests/container/loadSettings').loadSettings();
const supertest = require(__dirname+'/../src/node_modules/supertest');
const api = supertest('http://'+settings.ip+":"+settings.port);
const path = require('path');
diff --git a/bin/installDeps.sh b/bin/installDeps.sh
index 50310d9a1e1..5e0bbb931eb 100755
--- a/bin/installDeps.sh
+++ b/bin/installDeps.sh
@@ -8,6 +8,12 @@ REQUIRED_NODE_MINOR=13
REQUIRED_NPM_MAJOR=5
REQUIRED_NPM_MINOR=5
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+is_cmd() { command -v "$@" >/dev/null 2>&1; }
+
require_minimal_version() {
PROGRAM_LABEL="$1"
VERSION_STRING="$2"
@@ -16,71 +22,50 @@ require_minimal_version() {
# Flag -s (--only-delimited on GNU cut) ensures no string is returned
# when there is no match
- DETECTED_MAJOR=$(echo $VERSION_STRING | cut -s -d "." -f 1)
- DETECTED_MINOR=$(echo $VERSION_STRING | cut -s -d "." -f 2)
+ DETECTED_MAJOR=$(pecho "$VERSION_STRING" | cut -s -d "." -f 1)
+ DETECTED_MINOR=$(pecho "$VERSION_STRING" | cut -s -d "." -f 2)
- if [ -z "$DETECTED_MAJOR" ]; then
- printf 'Cannot extract %s major version from version string "%s"\n' "$PROGRAM_LABEL" "$VERSION_STRING" >&2
- exit 1
- fi
+ [ -n "$DETECTED_MAJOR" ] || fatal "Cannot extract $PROGRAM_LABEL major version from version string \"$VERSION_STRING\""
- if [ -z "$DETECTED_MINOR" ]; then
- printf 'Cannot extract %s minor version from version string "%s"\n' "$PROGRAM_LABEL" "$VERSION_STRING" >&2
- exit 1
- fi
+ [ -n "$DETECTED_MINOR" ] || fatal "Cannot extract $PROGRAM_LABEL minor version from version string \"$VERSION_STRING\""
case "$DETECTED_MAJOR" in
''|*[!0-9]*)
- printf '%s major version from "%s" is not a number. Detected: "%s"\n' "$PROGRAM_LABEL" "$VERSION_STRING" "$DETECTED_MAJOR" >&2
- exit 1
+ fatal "$PROGRAM_LABEL major version from \"$VERSION_STRING\" is not a number. Detected: \"$DETECTED_MAJOR\""
;;
esac
case "$DETECTED_MINOR" in
''|*[!0-9]*)
- printf '%s minor version from "%s" is not a number. Detected: "%s"\n' "$PROGRAM_LABEL" "$VERSION_STRING" "$DETECTED_MINOR" >&2
- exit 1
+ fatal "$PROGRAM_LABEL minor version from \"$VERSION_STRING\" is not a number. Detected: \"$DETECTED_MINOR\""
esac
- if [ "$DETECTED_MAJOR" -lt "$REQUIRED_MAJOR" ] || ([ "$DETECTED_MAJOR" -eq "$REQUIRED_MAJOR" ] && [ "$DETECTED_MINOR" -lt "$REQUIRED_MINOR" ]); then
- printf 'Your %s version "%s" is too old. %s %d.%d.x or higher is required.\n' "$PROGRAM_LABEL" "$VERSION_STRING" "$PROGRAM_LABEL" "$REQUIRED_MAJOR" "$REQUIRED_MINOR" >&2
- exit 1
- fi
+ [ "$DETECTED_MAJOR" -gt "$REQUIRED_MAJOR" ] || ([ "$DETECTED_MAJOR" -eq "$REQUIRED_MAJOR" ] && [ "$DETECTED_MINOR" -ge "$REQUIRED_MINOR" ]) \
+ || fatal "Your $PROGRAM_LABEL version \"$VERSION_STRING\" is too old. $PROGRAM_LABEL $REQUIRED_MAJOR.$REQUIRED_MINOR.x or higher is required."
}
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
-#Was this script started in the bin folder? if yes move out
-if [ -d "../bin" ]; then
- cd "../"
-fi
+# Is node installed?
+# Not checking io.js, default installation creates a symbolic link to node
+is_cmd node || fatal "Please install node.js ( https://nodejs.org )"
-#Is node installed?
-#Not checking io.js, default installation creates a symbolic link to node
-hash node > /dev/null 2>&1 || {
- echo "Please install node.js ( https://nodejs.org )" >&2
- exit 1
-}
-
-#Is npm installed?
-hash npm > /dev/null 2>&1 || {
- echo "Please install npm ( https://npmjs.org )" >&2
- exit 1
-}
+# Is npm installed?
+is_cmd npm || fatal "Please install npm ( https://npmjs.org )"
-#Check npm version
+# Check npm version
NPM_VERSION_STRING=$(npm --version)
require_minimal_version "npm" "$NPM_VERSION_STRING" "$REQUIRED_NPM_MAJOR" "$REQUIRED_NPM_MINOR"
-#Check node version
+# Check node version
NODE_VERSION_STRING=$(node --version)
NODE_VERSION_STRING=${NODE_VERSION_STRING#"v"}
require_minimal_version "nodejs" "$NODE_VERSION_STRING" "$REQUIRED_NODE_MAJOR" "$REQUIRED_NODE_MINOR"
-#Get the name of the settings file
+# Get the name of the settings file
settings="settings.json"
a='';
for arg in "$@"; do
@@ -88,26 +73,26 @@ for arg in "$@"; do
a=$arg
done
-#Does a $settings exist? if not copy the template
-if [ ! -f $settings ]; then
- echo "Copy the settings template to $settings..."
- cp settings.json.template $settings || exit 1
+# Does a $settings exist? if not copy the template
+if [ ! -f "$settings" ]; then
+ log "Copy the settings template to $settings..."
+ cp settings.json.template "$settings" || exit 1
fi
-echo "Ensure that all dependencies are up to date... If this is the first time you have run Etherpad please be patient."
+log "Ensure that all dependencies are up to date... If this is the first time you have run Etherpad please be patient."
(
mkdir -p node_modules
cd node_modules
[ -e ep_etherpad-lite ] || ln -s ../src ep_etherpad-lite
cd ep_etherpad-lite
- npm install --save --loglevel warn
+ npm ci
) || {
rm -rf src/node_modules
exit 1
}
-#Remove all minified data to force node creating it new
-echo "Clearing minified cache..."
+# Remove all minified data to force node creating it new
+log "Clearing minified cache..."
rm -f var/minified*
exit 0
diff --git a/bin/installOnWindows.bat b/bin/installOnWindows.bat
index 75982aaff47..3c6bf58c01c 100644
--- a/bin/installOnWindows.bat
+++ b/bin/installOnWindows.bat
@@ -14,7 +14,7 @@ cd /D node_modules
mklink /D "ep_etherpad-lite" "..\src"
cd /D "ep_etherpad-lite"
-cmd /C npm install --loglevel warn || exit /B 1
+cmd /C npm ci || exit /B 1
cd /D "%~dp0\.."
diff --git a/bin/plugins/README.md b/bin/plugins/README.md
new file mode 100755
index 00000000000..dc929798c29
--- /dev/null
+++ b/bin/plugins/README.md
@@ -0,0 +1,46 @@
+The files in this folder are for Plugin developers.
+
+# Get suggestions to improve your Plugin
+
+This code will check your plugin for known usual issues and some suggestions for improvements. No changes will be made to your project.
+
+```
+node bin/plugins/checkPlugin.js $PLUGIN_NAME$
+```
+
+# Basic Example:
+```
+node bin/plugins/checkPlugin.js ep_webrtc
+```
+
+## Autofixing - will autofix any issues it can
+```
+node bin/plugins/checkPlugins.js ep_whatever autofix
+```
+
+## Autocommitting, push, npm minor patch and npm publish (highly dangerous)
+```
+node bin/plugins/checkPlugins.js ep_whatever autofix autocommit
+```
+
+# All the plugins
+Replace johnmclear with your github username
+
+```
+# Clones
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+# autofixes and autocommits /pushes & npm publishes
+for dir in `ls node_modules`;
+do
+# echo $0
+if [[ $dir == *"ep_"* ]]; then
+if [[ $dir != "ep_etherpad-lite" ]]; then
+node bin/plugins/checkPlugin.js $dir autofix autocommit
+fi
+fi
+# echo $dir
+done
+```
diff --git a/bin/plugins/checkPlugin.js b/bin/plugins/checkPlugin.js
new file mode 100755
index 00000000000..0fccb4f1203
--- /dev/null
+++ b/bin/plugins/checkPlugin.js
@@ -0,0 +1,246 @@
+// pro usage for all your plugins, replace johnmclear with your github username
+/*
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+for dir in `ls node_modules`;
+do
+# echo $0
+if [[ $dir == *"ep_"* ]]; then
+if [[ $dir != "ep_etherpad-lite" ]]; then
+node bin/plugins/checkPlugin.js $dir autofix autocommit
+fi
+fi
+# echo $dir
+done
+*/
+
+/*
+*
+* Usage
+*
+* Normal usage: node bin/plugins/checkPlugins.js ep_whatever
+* Auto fix the things it can: node bin/plugins/checkPlugins.js ep_whatever autofix
+* Auto commit, push and publish(to npm) * highly dangerous:
+node bin/plugins/checkPlugins.js ep_whatever autofix autocommit
+
+*/
+
+const fs = require("fs");
+const { exec } = require("child_process");
+
+// get plugin name & path from user input
+const pluginName = process.argv[2];
+const pluginPath = "node_modules/"+pluginName;
+
+console.log("Checking the plugin: "+ pluginName)
+
+// Should we autofix?
+if (process.argv[3] && process.argv[3] === "autofix") var autoFix = true;
+
+// Should we update files where possible?
+if (process.argv[5] && process.argv[5] === "autoupdate") var autoUpdate = true;
+
+// Should we automcommit and npm publish?!
+if (process.argv[4] && process.argv[4] === "autocommit") var autoCommit = true;
+
+
+if(autoCommit){
+ console.warn("Auto commit is enabled, I hope you know what you are doing...")
+}
+
+fs.readdir(pluginPath, function (err, rootFiles) {
+ //handling error
+ if (err) {
+ return console.log('Unable to scan directory: ' + err);
+ }
+
+ // rewriting files to lower case
+ var files = [];
+
+ // some files we need to know the actual file name. Not compulsory but might help in the future.
+ var readMeFileName;
+ var repository;
+ var hasAutofixed = false;
+
+ for (var i = 0; i < rootFiles.length; i++) {
+ if(rootFiles[i].toLowerCase().indexOf("readme") !== -1) readMeFileName = rootFiles[i];
+ files.push(rootFiles[i].toLowerCase());
+ }
+
+ if(files.indexOf("package.json") === -1){
+ console.warn("no package.json, please create");
+ }
+
+ if(files.indexOf("package.json") !== -1){
+ let packageJSON = fs.readFileSync(pluginPath+"/package.json", {encoding:'utf8', flag:'r'});
+
+ if(packageJSON.toLowerCase().indexOf("repository") === -1){
+ console.warn("No repository in package.json");
+ if(autoFix){
+ console.warn("Repository not detected in package.json. Please add repository section manually.")
+ }
+ }else{
+ // useful for creating README later.
+ repository = JSON.parse(packageJSON).repository.url;
+ }
+
+ }
+ if(files.indexOf("readme") === -1 && files.indexOf("readme.md") === -1){
+ console.warn("README.md file not found, please create");
+ if(autoFix){
+ console.log("Autofixing missing README.md file, please edit the README.md file further to include plugin specific details.");
+ let readme = fs.readFileSync("bin/plugins/lib/README.md", {encoding:'utf8', flag:'r'})
+ readme = readme.replace(/\[plugin_name\]/g, pluginName);
+ if(repository){
+ let org = repository.split("/")[3];
+ let name = repository.split("/")[4];
+ readme = readme.replace(/\[org_name\]/g, org);
+ readme = readme.replace(/\[repo_url\]/g, name);
+ fs.writeFileSync(pluginPath+"/README.md", readme);
+ }else{
+ console.warn("Unable to find repository in package.json, aborting.")
+ }
+ }
+ }
+
+ if(files.indexOf("readme") !== -1 && files.indexOf("readme.md") !== -1){
+ let readme = fs.readFileSync(pluginPath+"/"+readMeFileName, {encoding:'utf8', flag:'r'});
+ if(readme.toLowerCase().indexOf("license") === -1){
+ console.warn("No license section in README");
+ if(autoFix){
+ console.warn("Please add License section to README manually.")
+ }
+ }
+ }
+
+ if(files.indexOf("license") === -1 && files.indexOf("license.md") === -1){
+ console.warn("LICENSE.md file not found, please create");
+ if(autoFix){
+ hasAutofixed = true;
+ console.log("Autofixing missing LICENSE.md file, including Apache 2 license.");
+ exec("git config user.name", (error, name, stderr) => {
+ if (error) {
+ console.log(`error: ${error.message}`);
+ return;
+ }
+ if (stderr) {
+ console.log(`stderr: ${stderr}`);
+ return;
+ }
+ let license = fs.readFileSync("bin/plugins/lib/LICENSE.md", {encoding:'utf8', flag:'r'});
+ license = license.replace("[yyyy]", new Date().getFullYear());
+ license = license.replace("[name of copyright owner]", name)
+ fs.writeFileSync(pluginPath+"/LICENSE.md", license);
+ });
+ }
+ }
+
+ var travisConfig = fs.readFileSync("bin/plugins/lib/travis.yml", {encoding:'utf8', flag:'r'});
+ travisConfig = travisConfig.replace(/\[plugin_name\]/g, pluginName);
+
+ if(files.indexOf(".travis.yml") === -1){
+ console.warn(".travis.yml file not found, please create. .travis.yml is used for automatically CI testing Etherpad. It is useful to know if your plugin breaks another feature for example.")
+ // TODO: Make it check version of the .travis file to see if it needs an update.
+ if(autoFix){
+ hasAutofixed = true;
+ console.log("Autofixing missing .travis.yml file");
+ fs.writeFileSync(pluginPath+"/.travis.yml", travisConfig);
+ console.log("Travis file created, please sign into travis and enable this repository")
+ }
+ }
+ if(autoFix && autoUpdate){
+ // checks the file versioning of .travis and updates it to the latest.
+ let existingConfig = fs.readFileSync(pluginPath + "/.travis.yml", {encoding:'utf8', flag:'r'});
+ let existingConfigLocation = existingConfig.indexOf("##ETHERPAD_TRAVIS_V=");
+ let existingValue = existingConfig.substr(existingConfigLocation+20, existingConfig.length);
+
+ let newConfigLocation = travisConfig.indexOf("##ETHERPAD_TRAVIS_V=");
+ let newValue = travisConfig.substr(newConfigLocation+20, travisConfig.length);
+
+ if(existingConfigLocation === -1){
+ console.warn("no previous .travis.yml version found so writing new.")
+ // we will write the newTravisConfig to the location.
+ fs.writeFileSync(pluginPath + "/.travis.yml", travisConfig);
+ }else{
+ if(newValue > existingValue){
+ console.log("updating .travis.yml");
+ fs.writeFileSync(pluginPath + "/.travis.yml", travisConfig);
+ hasAutofixed = true;
+ }
+ }
+ }
+
+ if(files.indexOf(".gitignore") === -1){
+ console.warn(".gitignore file not found, please create. .gitignore files are useful to ensure files aren't incorrectly commited to a repository.")
+ if(autoFix){
+ hasAutofixed = true;
+ console.log("Autofixing missing .gitignore file");
+ let gitignore = fs.readFileSync("bin/plugins/lib/gitignore", {encoding:'utf8', flag:'r'});
+ fs.writeFileSync(pluginPath+"/.gitignore", gitignore);
+ }
+ }
+
+ if(files.indexOf("locales") === -1){
+ console.warn("Translations not found, please create. Translation files help with Etherpad accessibility.");
+ }
+
+
+ if(files.indexOf(".ep_initialized") !== -1){
+ console.warn(".ep_initialized found, please remove. .ep_initialized should never be commited to git and should only exist once the plugin has been executed one time.")
+ if(autoFix){
+ hasAutofixed = true;
+ console.log("Autofixing incorrectly existing .ep_initialized file");
+ fs.unlinkSync(pluginPath+"/.ep_initialized");
+ }
+ }
+
+ if(files.indexOf("npm-debug.log") !== -1){
+ console.warn("npm-debug.log found, please remove. npm-debug.log should never be commited to your repository.")
+ if(autoFix){
+ hasAutofixed = true;
+ console.log("Autofixing incorrectly existing npm-debug.log file");
+ fs.unlinkSync(pluginPath+"/npm-debug.log");
+ }
+ }
+
+ if(files.indexOf("static") !== -1){
+ fs.readdir(pluginPath+"/static", function (errRead, staticFiles) {
+ if(staticFiles.indexOf("tests") === -1){
+ console.warn("Test files not found, please create tests. https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin")
+ }
+ })
+ }else{
+ console.warn("Test files not found, please create tests. https://github.com/ether/etherpad-lite/wiki/Creating-a-plugin#writing-and-running-front-end-tests-for-your-plugin")
+ }
+
+ if(hasAutofixed){
+ console.log("Fixes applied, please check git diff then run the following command:\n\n")
+ // bump npm Version
+ if(autoCommit){
+ // holy shit you brave.
+ console.log("Attempting autocommit and auto publish to npm")
+ exec("cd node_modules/"+ pluginName + " && git add -A && git commit --allow-empty -m 'autofixes from Etherpad checkPlugins.js' && npm version patch && git add package.json && git commit --allow-empty -m 'bump version' && git push && npm publish && cd ../..", (error, name, stderr) => {
+ if (error) {
+ console.log(`error: ${error.message}`);
+ return;
+ }
+ if (stderr) {
+ console.log(`stderr: ${stderr}`);
+ return;
+ }
+ console.log("I think she's got it! By George she's got it!")
+ process.exit(0)
+ });
+ }else{
+ console.log("cd node_modules/"+ pluginName + " && git add -A && git commit --allow-empty -m 'autofixes from Etherpad checkPlugins.js' && npm version patch && git add package.json && git commit --allow-empty -m 'bump version' && git push && npm publish && cd ../..")
+ }
+ }
+
+ //listing all files using forEach
+ files.forEach(function (file) {
+ // Do whatever you want to do with the file
+ // console.log(file.toLowerCase());
+ });
+});
diff --git a/bin/plugins/lib/LICENSE.md b/bin/plugins/lib/LICENSE.md
new file mode 100755
index 00000000000..8cb6bc0c609
--- /dev/null
+++ b/bin/plugins/lib/LICENSE.md
@@ -0,0 +1,13 @@
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/bin/plugins/lib/README.md b/bin/plugins/lib/README.md
new file mode 100755
index 00000000000..c3a3b1fbf37
--- /dev/null
+++ b/bin/plugins/lib/README.md
@@ -0,0 +1,28 @@
+[![Travis (.org)](https://api.travis-ci.org/[org_name]/[repo_url].svg?branch=develop)](https://travis-ci.org/github/[org_name]/[repo_url])
+
+# My awesome plugin README example
+Explain what your plugin does and who it's useful for.
+
+## Example animated gif of usage if appropriate
+
+## Installing
+npm install [plugin_name]
+
+or Use the Etherpad ``/admin`` interface.
+
+## Settings
+Document settings if any
+
+## Testing
+Document how to run backend / frontend tests.
+
+### Frontend
+
+Visit http://whatever/tests/frontend/ to run the frontend tests.
+
+### backend
+
+Type ``cd src && npm run test`` to run the backend tests.
+
+## LICENSE
+Apache 2.0
diff --git a/bin/plugins/lib/gitignore b/bin/plugins/lib/gitignore
new file mode 100755
index 00000000000..f6d13a09674
--- /dev/null
+++ b/bin/plugins/lib/gitignore
@@ -0,0 +1,5 @@
+.ep_initialized
+.DS_Store
+node_modules/
+node_modules
+npm-debug.log
diff --git a/bin/plugins/lib/travis.yml b/bin/plugins/lib/travis.yml
new file mode 100755
index 00000000000..81e7d336e1d
--- /dev/null
+++ b/bin/plugins/lib/travis.yml
@@ -0,0 +1,68 @@
+language: node_js
+
+node_js:
+ - "lts/*"
+
+cache: false
+
+before_install:
+ - sudo add-apt-repository -y ppa:libreoffice/ppa
+ - sudo apt-get update
+ - sudo apt-get -y install libreoffice
+ - sudo apt-get -y install libreoffice-pdfimport
+
+services:
+ - docker
+
+install:
+ - "bin/installDeps.sh"
+ - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+
+before_script:
+ - "tests/frontend/travis/sauce_tunnel.sh"
+
+script:
+ - "tests/frontend/travis/runner.sh"
+
+env:
+ global:
+ - secure: "WMGxFkOeTTlhWB+ChMucRtIqVmMbwzYdNHuHQjKCcj8HBEPdZLfCuK/kf4rG\nVLcLQiIsyllqzNhBGVHG1nyqWr0/LTm8JRqSCDDVIhpyzp9KpCJQQJG2Uwjk\n6/HIJJh/wbxsEdLNV2crYU/EiVO3A4Bq0YTHUlbhUqG3mSCr5Ec="
+ - secure: "gejXUAHYscbR6Bodw35XexpToqWkv2ifeECsbeEmjaLkYzXmUUNWJGknKSu7\nEUsSfQV8w+hxApr1Z+jNqk9aX3K1I4btL3cwk2trnNI8XRAvu1c1Iv60eerI\nkE82Rsd5lwUaMEh+/HoL8ztFCZamVndoNgX7HWp5J/NRZZMmh4g="
+
+jobs:
+ include:
+ - name: "Run the Backend tests"
+ install:
+ - "npm install"
+ - "mkdir [plugin_name]"
+ - "mv !([plugin_name]) [plugin_name]"
+ - "git clone https://github.com/ether/etherpad-lite.git etherpad"
+ - "cd etherpad"
+ - "mkdir node_modules"
+ - "mv ../[plugin_name] node_modules"
+ - "bin/installDeps.sh"
+ - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+ - "cd src && npm install && cd -"
+ script:
+ - "tests/frontend/travis/runnerBackend.sh"
+ - name: "Test the Frontend"
+ install:
+ - "npm install"
+ - "mkdir [plugin_name]"
+ - "mv !([plugin_name]) [plugin_name]"
+ - "git clone https://github.com/ether/etherpad-lite.git etherpad"
+ - "cd etherpad"
+ - "mkdir node_modules"
+ - "mv ../[plugin_name] node_modules"
+ - "bin/installDeps.sh"
+ - "export GIT_HASH=$(git rev-parse --verify --short HEAD)"
+ script:
+ - "tests/frontend/travis/runner.sh"
+
+notifications:
+ irc:
+ channels:
+ - "irc.freenode.org#etherpad-lite-dev"
+
+##ETHERPAD_TRAVIS_V=3
+## Travis configuration automatically created using bin/plugins/updateAllPluginsScript.sh
diff --git a/bin/plugins/reTestAllPlugins.sh b/bin/plugins/reTestAllPlugins.sh
new file mode 100755
index 00000000000..319d378d4e4
--- /dev/null
+++ b/bin/plugins/reTestAllPlugins.sh
@@ -0,0 +1,14 @@
+echo "herp";
+for dir in `ls node_modules`;
+do
+ echo $dir
+ if [[ $dir == *"ep_"* ]]; then
+ if [[ $dir != "ep_etherpad-lite" ]]; then
+ # node bin/plugins/checkPlugin.js $dir autofix autocommit autoupdate
+ cd node_modules/$dir
+ git commit -m "Automatic update: bump update to re-run latest Etherpad tests" --allow-empty
+ git push origin master
+ cd ../..
+ fi
+ fi
+done
diff --git a/bin/plugins/updateAllPluginsScript.sh b/bin/plugins/updateAllPluginsScript.sh
new file mode 100755
index 00000000000..763724fcaf9
--- /dev/null
+++ b/bin/plugins/updateAllPluginsScript.sh
@@ -0,0 +1,17 @@
+cd node_modules
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=2" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=3" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+GHUSER=johnmclear; curl "https://api.github.com/users/$GHUSER/repos?per_page=1000&page=4" | grep -o 'git@[^"]*' | grep /ep_ | xargs -L1 git clone
+cd ..
+
+for dir in `ls node_modules`;
+do
+ # echo $0
+ if [[ $dir == *"ep_"* ]]; then
+ if [[ $dir != "ep_etherpad-lite" ]]; then
+ node bin/plugins/checkPlugin.js $dir autofix autocommit autoupdate
+ fi
+ fi
+ # echo $dir
+done
diff --git a/bin/run.sh b/bin/run.sh
index 74fa56d683e..ff6b3de093c 100755
--- a/bin/run.sh
+++ b/bin/run.sh
@@ -1,39 +1,37 @@
#!/bin/sh
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
-#Was this script started in the bin folder? if yes move out
-if [ -d "../bin" ]; then
- cd "../"
-fi
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
ignoreRoot=0
-for ARG in "$@"
-do
+for ARG in "$@"; do
if [ "$ARG" = "--root" ]; then
ignoreRoot=1
fi
done
-#Stop the script if it's started as root
-if [ "$(id -u)" -eq 0 ] && [ $ignoreRoot -eq 0 ]; then
- echo "You shouldn't start Etherpad as root!"
- echo "Please type 'Etherpad rocks my socks' or supply the '--root' argument if you still want to start it as root"
+# Stop the script if it's started as root
+if [ "$(id -u)" -eq 0 ] && [ "$ignoreRoot" -eq 0 ]; then
+ cat <&2
+You shouldn't start Etherpad as root!
+Please type 'Etherpad rocks my socks' (or restart with the '--root'
+argument) if you still want to start it as root:
+EOF
+ printf "> " >&2
read rocks
- if [ ! "$rocks" == "Etherpad rocks my socks" ]
- then
- echo "Your input was incorrect"
- exit 1
- fi
+ [ "$rocks" = "Etherpad rocks my socks" ] || fatal "Your input was incorrect"
fi
-#Prepare the environment
+# Prepare the environment
bin/installDeps.sh "$@" || exit 1
-#Move to the node folder and start
-echo "Started Etherpad..."
+# Move to the node folder and start
+log "Starting Etherpad..."
SCRIPTPATH=$(pwd -P)
exec node "$SCRIPTPATH/node_modules/ep_etherpad-lite/node/server.js" "$@"
-
diff --git a/bin/safeRun.sh b/bin/safeRun.sh
index 99a72bcc03c..6d43e3035b4 100755
--- a/bin/safeRun.sh
+++ b/bin/safeRun.sh
@@ -1,10 +1,11 @@
#!/bin/sh
-#This script ensures that ep-lite is automatically restarting after an error happens
+# This script ensures that ep-lite is automatically restarting after
+# an error happens
-#Handling Errors
-# 0 silent
-# 1 email
+# Handling Errors
+# 0 silent
+# 1 email
ERROR_HANDLING=0
# Your email address which should receive the error messages
EMAIL_ADDRESS="no-reply@example.com"
@@ -15,54 +16,54 @@ TIME_BETWEEN_EMAILS=600 # 10 minutes
# DON'T EDIT AFTER THIS LINE
+pecho() { printf %s\\n "$*"; }
+log() { pecho "$@"; }
+error() { log "ERROR: $@" >&2; }
+fatal() { error "$@"; exit 1; }
+
LAST_EMAIL_SEND=0
-LOG="$1"
-#Move to the folder where ep-lite is installed
-cd $(dirname $0)
+# Move to the folder where ep-lite is installed
+cd "$(dirname "$0")"/..
-#Was this script started in the bin folder? if yes move out
-if [ -d "../bin" ]; then
- cd "../"
-fi
+# Check if a logfile parameter is set
+LOG="$1"
+[ -n "${LOG}" ] || fatal "Set a logfile as the first parameter"
+shift
-#Check if a logfile parameter is set
-if [ -z "${LOG}" ]; then
- echo "Set a logfile as the first parameter"
- exit 1
-fi
+while true; do
+ # Try to touch the file if it doesn't exist
+ [ -f "${LOG}" ] || touch "${LOG}" || fatal "Logfile '${LOG}' is not writeable"
-shift
-while [ 1 ]
-do
- #Try to touch the file if it doesn't exist
- if [ ! -f ${LOG} ]; then
- touch ${LOG} || ( echo "Logfile '${LOG}' is not writeable" && exit 1 )
- fi
+ # Check if the file is writeable
+ [ -w "${LOG}" ] || fatal "Logfile '${LOG}' is not writeable"
- #Check if the file is writeable
- if [ ! -w ${LOG} ]; then
- echo "Logfile '${LOG}' is not writeable"
- exit 1
- fi
+ # Start the application
+ bin/run.sh "$@" >>${LOG} 2>>${LOG}
- #Start the application
- bin/run.sh $@ >>${LOG} 2>>${LOG}
+ TIME_FMT=$(date +%Y-%m-%dT%H:%M:%S%z)
- #Send email
- if [ $ERROR_HANDLING = 1 ]; then
+ # Send email
+ if [ "$ERROR_HANDLING" = 1 ]; then
TIME_NOW=$(date +%s)
TIME_SINCE_LAST_SEND=$(($TIME_NOW - $LAST_EMAIL_SEND))
- if [ $TIME_SINCE_LAST_SEND -gt $TIME_BETWEEN_EMAILS ]; then
- printf "Server was restarted at: $(date)\nThe last 50 lines of the log before the error happens:\n $(tail -n 50 ${LOG})" | mail -s "Pad Server was restarted" $EMAIL_ADDRESS
+ if [ "$TIME_SINCE_LAST_SEND" -gt "$TIME_BETWEEN_EMAILS" ]; then
+ {
+ cat <>${LOG}
+ pecho "RESTART! ${TIME_FMT}" >>${LOG}
- #Sleep 10 seconds before restart
+ # Sleep 10 seconds before restart
sleep 10
done
diff --git a/doc/api/hooks_overview.md b/doc/api/hooks_overview.md
index c252aa8409f..1de547c9009 100644
--- a/doc/api/hooks_overview.md
+++ b/doc/api/hooks_overview.md
@@ -1,11 +1,51 @@
# Hooks
-All hooks are called with two arguments:
-1. name - the name of the hook being called
-2. context - an object with some relevant information about the context of the call
+A hook function is registered with a hook via the plugin's `ep.json` file. See
+the Plugins section for details. A hook may have many registered functions from
+different plugins.
+
+When a hook is invoked, its registered functions are called with three
+arguments:
+
+1. hookName - The name of the hook being invoked.
+2. context - An object with some relevant information about the context of the
+ call. See the hook-specific documentation for details.
+3. callback - Function to call when done. This callback takes a single argument,
+ the meaning of which depends on the hook. See the "Return values" section for
+ general information that applies to most hooks. The value returned by this
+ callback must be returned by the hook function unless otherwise specified.
## Return values
-A hook should always return a list or undefined. Returning undefined is equivalent to returning an empty list.
-All the returned lists are appended to each other, so if the return values where `[1, 2]`, `undefined`, `[3, 4,]`, `undefined` and `[5]`, the value returned by callHook would be `[1, 2, 3, 4, 5]`.
-This is, because it should never matter if you have one plugin or several plugins doing some work - a single plugin should be able to make callHook return the same value a set of plugins are able to return collectively. So, any plugin can return a list of values, of any length, not just one value.
\ No newline at end of file
+Note: This section applies to every hook unless the hook-specific documentation
+says otherwise.
+
+Hook functions return zero or more values to Etherpad by passing an array to the
+provided callback. Hook functions typically provide a single value (array of
+length one). If the function does not want to or need to provide a value, it may
+pass an empty array or `undefined` (which is treated the same as an empty
+array). Hook functions may also provide more than one value (array of length two
+or more).
+
+Some hooks concatenate the arrays provided by its registered functions. For
+example, if a hook's registered functions pass `[1, 2]`, `undefined`, `[3, 4]`,
+`[]`, and `[5]` to the provided callback, then the hook's return value is `[1,
+2, 3, 4, 5]`.
+
+Other hooks only use the first non-empty array provided by a registered
+function. In this case, each of the hook's registered functions is called one at
+a time until one provides a non-empty array. The remaining functions are
+skipped. If none of the functions provide a non-empty array, or there are no
+registered functions, the hook's return value is `[]`.
+
+Example:
+
+```
+exports.abstractHook = (hookName, context, callback) => {
+ if (notApplicableToThisPlugin(context)) {
+ return callback();
+ }
+ const value = doSomeProcessing(context);
+ return callback([value]);
+};
+```
diff --git a/doc/api/hooks_server-side.md b/doc/api/hooks_server-side.md
index a89ee560bd8..55b4574331b 100644
--- a/doc/api/hooks_server-side.md
+++ b/doc/api/hooks_server-side.md
@@ -93,7 +93,10 @@ Available blocks in `pad.html` are:
`index.html` blocks:
+ * `indexCustomStyles` - contains the `index.css` `` tag, allows you to add your own or to customize the one provided by the active skin
* `indexWrapper` - contains the form for creating new pads
+ * `indexCustomScripts` - contains the `index.js` ` \
- \
- "
- );
+ res.send("");
});
}
diff --git a/src/node/handler/PadMessageHandler.js b/src/node/handler/PadMessageHandler.js
index bf69a3d2580..fc3b0a19a43 100644
--- a/src/node/handler/PadMessageHandler.js
+++ b/src/node/handler/PadMessageHandler.js
@@ -27,7 +27,7 @@ var authorManager = require("../db/AuthorManager");
var readOnlyManager = require("../db/ReadOnlyManager");
var settings = require('../utils/Settings');
var securityManager = require("../db/SecurityManager");
-var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins.js");
+var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugin_defs.js");
var log4js = require('log4js');
var messageLogger = log4js.getLogger("message");
var accessLogger = log4js.getLogger("access");
@@ -36,7 +36,14 @@ var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js");
var channels = require("channels");
var stats = require('../stats');
var remoteAddress = require("../utils/RemoteAddress").remoteAddress;
+const assert = require('assert').strict;
const nodeify = require("nodeify");
+const { RateLimiterMemory } = require('rate-limiter-flexible');
+
+const rateLimiter = new RateLimiterMemory({
+ points: settings.commitRateLimiting.points,
+ duration: settings.commitRateLimiting.duration
+});
/**
* A associative array that saves informations about a session
@@ -164,6 +171,19 @@ exports.handleDisconnect = async function(client)
*/
exports.handleMessage = async function(client, message)
{
+ var env = process.env.NODE_ENV || 'development';
+
+ if (env === 'production') {
+ try {
+ await rateLimiter.consume(client.handshake.address); // consume 1 point per event from IP
+ }catch(e){
+ console.warn("Rate limited: ", client.handshake.address, " to reduce the amount of rate limiting that happens edit the rateLimit values in settings.json");
+ stats.meter('rateLimited').mark();
+ client.json.send({disconnect:"rateLimited"});
+ return;
+ }
+ }
+
if (message == null) {
return;
}
@@ -239,35 +259,13 @@ exports.handleMessage = async function(client, message)
}
}
- /*
- * In a previous version of this code, an "if (message)" wrapped the
- * following series of async calls [now replaced with await calls]
- * This ugly "!Boolean(message)" is a lame way to exactly negate the truthy
- * condition and replace it with an early return, while being sure to leave
- * the original behaviour unchanged.
- *
- * A shallower code could maybe make more evident latent logic errors.
- */
- if (!Boolean(message)) {
- return;
- }
-
let dropMessage = await handleMessageHook();
if (!dropMessage) {
-
- // check permissions
-
if (message.type == "CLIENT_READY") {
// client tried to auth for the first time (first msg from the client)
createSessionInfo(client, message);
}
- // Note: message.sessionID is an entirely different kind of
- // session from the sessions we use here! Beware!
- // FIXME: Call our "sessions" "connections".
- // FIXME: Use a hook instead
- // FIXME: Allow to override readwrite access with readonly
-
// the session may have been dropped during earlier processing
if (!sessioninfos[client.id]) {
messageLogger.warn("Dropping message from a connection that has gone away.")
@@ -895,12 +893,6 @@ async function handleClientReady(client, message)
// Get ro/rw id:s
let padIds = await readOnlyManager.getIds(message.padId);
- // check permissions
-
- // Note: message.sessionID is an entierly different kind of
- // session from the sessions we use here! Beware!
- // FIXME: Call our "sessions" "connections".
- // FIXME: Use a hook instead
// FIXME: Allow to override readwrite access with readonly
let statusObject = await securityManager.checkAccess(padIds.padId, message.sessionID, message.token, message.password);
let accessStatus = statusObject.accessStatus;
@@ -914,6 +906,7 @@ async function handleClientReady(client, message)
let author = statusObject.authorID;
// get all authordata of this new user
+ assert(author);
let value = await authorManager.getAuthor(author);
let authorColorId = value.colorId;
let authorName = value.name;
@@ -1131,6 +1124,7 @@ async function handleClientReady(client, message)
},
"initialChangesets": [], // FIXME: REMOVE THIS SHIT
"thisUserHasEditedThisPad": thisUserHasEditedThisPad,
+ "allowAnyoneToImport": settings.allowAnyoneToImport
}
// Add a username to the clientVars if one avaiable
@@ -1139,7 +1133,7 @@ async function handleClientReady(client, message)
}
// call the clientVars-hook so plugins can modify them before they get sent to the client
- let messages = await hooks.aCallAll("clientVars", { clientVars: clientVars, pad: pad });
+ let messages = await hooks.aCallAll('clientVars', {clientVars, pad, socket: client});
// combine our old object with the new attributes from the hook
for (let msg of messages) {
@@ -1294,7 +1288,7 @@ async function handleChangesetRequest(client, message)
data.requestID = message.data.requestID;
client.json.send({ type: "CHANGESET_REQ", data });
} catch (err) {
- console.error('Error while handling a changeset request for ' + padIds.padId, err, message.data);
+ console.error('Error while handling a changeset request for ' + padIds.padId, err.toString(), message.data);
}
}
@@ -1495,8 +1489,12 @@ exports.padUsers = async function(padID) {
let s = sessioninfos[roomClient.id];
if (s) {
return authorManager.getAuthor(s.author).then(author => {
- author.id = s.author;
- padUsers.push(author);
+ // Fixes: https://github.com/ether/etherpad-lite/issues/4120
+ // On restart author might not be populated?
+ if(author){
+ author.id = s.author;
+ padUsers.push(author);
+ }
});
}
}));
diff --git a/src/node/hooks/express/adminplugins.js b/src/node/hooks/express/adminplugins.js
index 7cfb160b92f..f6f184ed396 100644
--- a/src/node/hooks/express/adminplugins.js
+++ b/src/node/hooks/express/adminplugins.js
@@ -1,13 +1,13 @@
var eejs = require('ep_etherpad-lite/node/eejs');
var settings = require('ep_etherpad-lite/node/utils/Settings');
var installer = require('ep_etherpad-lite/static/js/pluginfw/installer');
-var plugins = require('ep_etherpad-lite/static/js/pluginfw/plugins');
+var plugins = require('ep_etherpad-lite/static/js/pluginfw/plugin_defs');
var _ = require('underscore');
var semver = require('semver');
+const UpdateCheck = require('ep_etherpad-lite/node/utils/UpdateCheck');
exports.expressCreateServer = function(hook_name, args, cb) {
args.app.get('/admin/plugins', function(req, res) {
- var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
var render_args = {
plugins: plugins.plugins,
search_results: {},
@@ -23,7 +23,8 @@ exports.expressCreateServer = function(hook_name, args, cb) {
res.send(eejs.require("ep_etherpad-lite/templates/admin/plugins-info.html", {
gitCommit: gitCommit,
- epVersion: epVersion
+ epVersion: epVersion,
+ latestVersion: UpdateCheck.getLatestVersion()
}));
});
}
diff --git a/src/node/hooks/express/adminsettings.js b/src/node/hooks/express/adminsettings.js
index baa0bb70a0e..1e0d6004f78 100644
--- a/src/node/hooks/express/adminsettings.js
+++ b/src/node/hooks/express/adminsettings.js
@@ -29,9 +29,9 @@ exports.socketio = function (hook_name, args, cb) {
return console.log(err);
}
- //if showSettingsInAdminPage is set to false, then return NOT_ALLOWED in the result
+ // if showSettingsInAdminPage is set to false, then return NOT_ALLOWED in the result
if(settings.showSettingsInAdminPage === false) {
- socket.emit("settings", {results:'NOT_ALLOWED'});
+ socket.emit("settings", {results: 'NOT_ALLOWED'});
}
else {
socket.emit("settings", {results: data});
diff --git a/src/node/hooks/express/importexport.js b/src/node/hooks/express/importexport.js
index 95d02775d48..85ab6e12971 100644
--- a/src/node/hooks/express/importexport.js
+++ b/src/node/hooks/express/importexport.js
@@ -1,3 +1,4 @@
+const assert = require('assert').strict;
var hasPadAccess = require("../../padaccess");
var settings = require('../../utils/Settings');
var exportHandler = require('../../handler/ExportHandler');
@@ -5,6 +6,7 @@ var importHandler = require('../../handler/ImportHandler');
var padManager = require("../../db/PadManager");
var authorManager = require("../../db/AuthorManager");
const rateLimit = require("express-rate-limit");
+const securityManager = require("../../db/SecurityManager");
settings.importExportRateLimiting.onLimitReached = function(req, res, options) {
// when the rate limiter triggers, write a warning in the logs
@@ -51,57 +53,41 @@ exports.expressCreateServer = function (hook_name, args, cb) {
// handle import requests
args.app.use('/p/:pad/import', limiter);
args.app.post('/p/:pad/import', async function(req, res, next) {
- if (await hasPadAccess(req, res)) {
- let exists = await padManager.doesPadExists(req.params.pad);
- if (!exists) {
- console.warn(`Someone tried to import into a pad that doesn't exist (${req.params.pad})`);
- return next();
- }
-
- /*
- * Starting from Etherpad 1.8.3 onwards, importing into a pad is allowed
- * only if a user has his browser opened and connected to the pad (i.e. a
- * Socket.IO session is estabilished for him) and he has already
- * contributed to that specific pad.
- *
- * Note that this does not have anything to do with the "session", used
- * for logging into "group pads". That kind of session is not needed here.
- *
- * This behaviour does not apply to API requests, only to /p/$PAD$/import
- *
- * See: https://github.com/ether/etherpad-lite/pull/3833#discussion_r407490205
- */
- if (!req.cookies) {
- console.warn(`Unable to import file into "${req.params.pad}". No cookies included in request`);
- return next();
- }
-
- if (!req.cookies.token) {
- console.warn(`Unable to import file into "${req.params.pad}". No token in the cookies`);
- return next();
- }
-
- let author = await authorManager.getAuthor4Token(req.cookies.token);
- // author is of the form: "a.g2droBYw1prY7HW9"
- if (!author) {
- console.warn(`Unable to import file into "${req.params.pad}". No Author found for token ${req.cookies.token}`);
-
- return next();
- }
+ if (!(await padManager.doesPadExists(req.params.pad))) {
+ console.warn(`Someone tried to import into a pad that doesn't exist (${req.params.pad})`);
+ return next();
+ }
- let authorsPads = await authorManager.listPadsOfAuthor(author);
+ const {accessStatus, authorID} = await securityManager.checkAccess(
+ req.params.pad, req.cookies.sessionID, req.cookies.token, req.cookies.password);
+ if (accessStatus !== 'grant') return res.status(403).send('Forbidden');
+ assert(authorID);
+
+ /*
+ * Starting from Etherpad 1.8.3 onwards, importing into a pad is allowed
+ * only if a user has his browser opened and connected to the pad (i.e. a
+ * Socket.IO session is estabilished for him) and he has already
+ * contributed to that specific pad.
+ *
+ * Note that this does not have anything to do with the "session", used
+ * for logging into "group pads". That kind of session is not needed here.
+ *
+ * This behaviour does not apply to API requests, only to /p/$PAD$/import
+ *
+ * See: https://github.com/ether/etherpad-lite/pull/3833#discussion_r407490205
+ */
+ if (!settings.allowAnyoneToImport) {
+ const authorsPads = await authorManager.listPadsOfAuthor(authorID);
if (!authorsPads) {
- console.warn(`Unable to import file into "${req.params.pad}". Author "${author}" exists but he never contributed to any pad`);
+ console.warn(`Unable to import file into "${req.params.pad}". Author "${authorID}" exists but he never contributed to any pad`);
return next();
}
-
- let authorsPadIDs = authorsPads.padIDs;
- if (authorsPadIDs.indexOf(req.params.pad) === -1) {
- console.warn(`Unable to import file into "${req.params.pad}". Author "${author}" exists but he never contributed to this pad`);
+ if (authorsPads.padIDs.indexOf(req.params.pad) === -1) {
+ console.warn(`Unable to import file into "${req.params.pad}". Author "${authorID}" exists but he never contributed to this pad`);
return next();
}
-
- importHandler.doImport(req, res, req.params.pad);
}
+
+ importHandler.doImport(req, res, req.params.pad);
});
}
diff --git a/src/node/hooks/express/openapi.js b/src/node/hooks/express/openapi.js
index 26f898b963c..76ed6693247 100644
--- a/src/node/hooks/express/openapi.js
+++ b/src/node/hooks/express/openapi.js
@@ -688,7 +688,7 @@ exports.expressCreateServer = async (_, args) => {
// support jsonp response format
if (req.query.jsonp && isValidJSONPName.check(req.query.jsonp)) {
res.header('Content-Type', 'application/javascript');
- response = `${req.query.jsonp}(${JSON.stringify(response)}`;
+ response = `${req.query.jsonp}(${JSON.stringify(response)})`;
}
// send response
diff --git a/src/node/hooks/express/socketio.js b/src/node/hooks/express/socketio.js
index fde514e66e3..03fa7bbe60c 100644
--- a/src/node/hooks/express/socketio.js
+++ b/src/node/hooks/express/socketio.js
@@ -47,25 +47,35 @@ exports.expressCreateServer = function (hook_name, args, cb) {
io.use(function(socket, accept) {
var data = socket.request;
// Use a setting if we want to allow load Testing
- if(!data.headers.cookie && settings.loadTest){
+
+ // Sometimes browsers might not have cookies at all, for example Safari in iFrames Cross domain
+ // https://github.com/ether/etherpad-lite/issues/4031
+ // if requireSession is false we can allow them to still get on the pad.
+ // Note that this does make security less tight because any socketIO connection can be established without
+ // any logic on the client to do any handshaking.. I am not concerned about this though, the real solution
+ // here is to implement rateLimiting on SocketIO ACCEPT_COMMIT messages.
+
+ if(!data.headers.cookie && (settings.loadTest || !settings.requireSession)){
accept(null, true);
}else{
if (!data.headers.cookie) return accept('No session cookie transmitted.', false);
}
- cookieParserFn(data, {}, function(err){
- if(err) {
- console.error(err);
- accept("Couldn't parse request cookies. ", false);
- return;
- }
+ if(data.headers.cookie){
+ cookieParserFn(data, {}, function(err){
+ if(err) {
+ console.error(err);
+ accept("Couldn't parse request cookies. ", false);
+ return;
+ }
- data.sessionID = data.signedCookies.express_sid;
- args.app.sessionStore.get(data.sessionID, function (err, session) {
- if (err || !session) return accept('Bad session / session has expired', false);
- data.session = new sessionModule.Session(data, session);
- accept(null, true);
+ data.sessionID = data.signedCookies.express_sid;
+ args.app.sessionStore.get(data.sessionID, function (err, session) {
+ if (err || !session) return accept('Bad session / session has expired', false);
+ data.session = new sessionModule.Session(data, session);
+ accept(null, true);
+ });
});
- });
+ }
});
// var socketIOLogger = log4js.getLogger("socket.io");
diff --git a/src/node/hooks/express/specialpages.js b/src/node/hooks/express/specialpages.js
index 2ca1384bb6e..b11f77a0075 100644
--- a/src/node/hooks/express/specialpages.js
+++ b/src/node/hooks/express/specialpages.js
@@ -29,7 +29,7 @@ exports.expressCreateServer = function (hook_name, args, cb) {
var filePath = path.join(settings.root, "src", "static", "skins", settings.skinName, "robots.txt");
res.sendFile(filePath, function(err)
{
- //there is no custom favicon, send the default robots.txt which dissallows all
+ //there is no custom robots.txt, send the default robots.txt which dissallows all
if(err)
{
filePath = path.join(settings.root, "src", "static", "robots.txt");
diff --git a/src/node/hooks/express/static.js b/src/node/hooks/express/static.js
index 4c17fbe3b0d..b8c6c9d52b7 100644
--- a/src/node/hooks/express/static.js
+++ b/src/node/hooks/express/static.js
@@ -1,5 +1,5 @@
var minify = require('../../utils/Minify');
-var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
+var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugin_defs");
var CachingMiddleware = require('../../utils/caching_middleware');
var settings = require("../../utils/Settings");
var Yajsml = require('etherpad-yajsml');
diff --git a/src/node/hooks/express/tests.js b/src/node/hooks/express/tests.js
index efc03fa489d..216715d43a0 100644
--- a/src/node/hooks/express/tests.js
+++ b/src/node/hooks/express/tests.js
@@ -17,7 +17,8 @@ exports.expressCreateServer = function (hook_name, args, cb) {
files = files.filter(el => !/\.swp$/.test(el))
console.debug("Sent browser the following test specs:", files);
- res.send("var specs_list = " + JSON.stringify(files) + ";\n");
+ res.setHeader('content-type', 'text/javascript');
+ res.end("var specs_list = " + JSON.stringify(files) + ";\n");
});
// path.join seems to normalize by default, but we'll just be explicit
diff --git a/src/node/hooks/express/webaccess.js b/src/node/hooks/express/webaccess.js
index ffb132ae1ad..761a46ab802 100644
--- a/src/node/hooks/express/webaccess.js
+++ b/src/node/hooks/express/webaccess.js
@@ -1,116 +1,131 @@
-var express = require('express');
-var log4js = require('log4js');
-var httpLogger = log4js.getLogger("http");
-var settings = require('../../utils/Settings');
-var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
-var ueberStore = require('../../db/SessionStore');
-var stats = require('ep_etherpad-lite/node/stats');
-var sessionModule = require('express-session');
-var cookieParser = require('cookie-parser');
-
-//checks for basic http auth
-exports.basicAuth = function (req, res, next) {
- var hookResultMangle = function (cb) {
- return function (err, data) {
+const express = require('express');
+const log4js = require('log4js');
+const httpLogger = log4js.getLogger('http');
+const settings = require('../../utils/Settings');
+const hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks');
+const ueberStore = require('../../db/SessionStore');
+const stats = require('ep_etherpad-lite/node/stats');
+const sessionModule = require('express-session');
+const cookieParser = require('cookie-parser');
+
+exports.checkAccess = (req, res, next) => {
+ const hookResultMangle = (cb) => {
+ return (err, data) => {
return cb(!err && data.length && data[0]);
- }
- }
+ };
+ };
- var authorize = function (cb) {
+ // This may be called twice per access: once before authentication is checked and once after (if
+ // settings.requireAuthorization is true).
+ const authorize = (cb) => {
// Do not require auth for static paths and the API...this could be a bit brittle
if (req.path.match(/^\/(static|javascripts|pluginfw|api)/)) return cb(true);
- if (req.path.toLowerCase().indexOf('/admin') != 0) {
+ if (req.path.toLowerCase().indexOf('/admin') !== 0) {
if (!settings.requireAuthentication) return cb(true);
if (!settings.requireAuthorization && req.session && req.session.user) return cb(true);
}
if (req.session && req.session.user && req.session.user.is_admin) return cb(true);
- hooks.aCallFirst("authorize", {req: req, res:res, next:next, resource: req.path}, hookResultMangle(cb));
- }
-
- var authenticate = function (cb) {
- // If auth headers are present use them to authenticate...
- if (req.headers.authorization && req.headers.authorization.search('Basic ') === 0) {
- var userpass = Buffer.from(req.headers.authorization.split(' ')[1], 'base64').toString().split(":")
- var username = userpass.shift();
- var password = userpass.join(':');
- var fallback = function(success) {
- if (success) return cb(true);
- if (settings.users[username] != undefined && settings.users[username].password === password) {
- settings.users[username].username = username;
- req.session.user = settings.users[username];
- return cb(true);
- }
- return cb(false);
- };
- return hooks.aCallFirst("authenticate", {req: req, res:res, next:next, username: username, password: password}, hookResultMangle(fallback));
- }
- hooks.aCallFirst("authenticate", {req: req, res:res, next:next}, hookResultMangle(cb));
- }
-
+ hooks.aCallFirst('authorize', {req, res, next, resource: req.path}, hookResultMangle(cb));
+ };
/* Authentication OR authorization failed. */
- var failure = function () {
- return hooks.aCallFirst("authFailure", {req: req, res:res, next:next}, hookResultMangle(function (ok) {
- if (ok) return;
- /* No plugin handler for invalid auth. Return Auth required
- * Headers, delayed for 1 second, if authentication failed
- * before. */
+ const failure = () => {
+ return hooks.aCallFirst('authFailure', {req, res, next}, hookResultMangle((ok) => {
+ if (ok) return;
+ // No plugin handled the authn/authz failure. Fall back to basic authentication.
res.header('WWW-Authenticate', 'Basic realm="Protected Area"');
- if (req.headers.authorization) {
- setTimeout(function () {
- res.status(401).send('Authentication required');
- }, 1000);
- } else {
- res.status(401).send('Authentication required');
+ // Delay the error response for 1s to slow down brute force attacks.
+ setTimeout(() => {
+ res.status(401).send('Authentication Required');
+ }, 1000);
+ }));
+ };
+
+ // Access checking is done in three steps:
+ //
+ // 1) Try to just access the thing. If access fails (perhaps authentication has not yet completed,
+ // or maybe different credentials are required), go to the next step.
+ // 2) Try to authenticate. (Or, if already logged in, reauthenticate with different credentials if
+ // supported by the authn scheme.) If authentication fails, give the user a 401 error to
+ // request new credentials. Otherwise, go to the next step.
+ // 3) Try to access the thing again. If this fails, give the user a 401 error.
+ //
+ // Plugins can use the 'next' callback (from the hook's context) to break out at any point (e.g.,
+ // to process an OAuth callback). Plugins can use the authFailure hook to override the default
+ // error handling behavior (e.g., to redirect to a login page).
+
+ let step1PreAuthenticate, step2Authenticate, step3Authorize;
+
+ step1PreAuthenticate = () => {
+ authorize((ok) => {
+ if (ok) return next();
+ step2Authenticate();
+ });
+ };
+
+ step2Authenticate = () => {
+ const ctx = {req, res, next};
+ // If the HTTP basic auth header is present, extract the username and password so it can be
+ // given to authn plugins.
+ const httpBasicAuth =
+ req.headers.authorization && req.headers.authorization.search('Basic ') === 0;
+ if (httpBasicAuth) {
+ const userpass =
+ Buffer.from(req.headers.authorization.split(' ')[1], 'base64').toString().split(':');
+ ctx.username = userpass.shift();
+ ctx.password = userpass.join(':');
+ }
+ hooks.aCallFirst('authenticate', ctx, hookResultMangle((ok) => {
+ if (!ok) {
+ // Fall back to HTTP basic auth.
+ if (!httpBasicAuth) return failure();
+ if (!(ctx.username in settings.users)) {
+ httpLogger.info(`Failed authentication from IP ${req.ip} - no such user`);
+ return failure();
+ }
+ if (settings.users[ctx.username].password !== ctx.password) {
+ httpLogger.info(`Failed authentication from IP ${req.ip} for user ${ctx.username} - incorrect password`);
+ return failure();
+ }
+ httpLogger.info(`Successful authentication from IP ${req.ip} for user ${ctx.username}`);
+ settings.users[ctx.username].username = ctx.username;
+ req.session.user = settings.users[ctx.username];
}
+ step3Authorize();
}));
- }
-
-
- /* This is the actual authentication/authorization hoop. It is done in four steps:
-
- 1) Try to just access the thing
- 2) If not allowed using whatever creds are in the current session already, try to authenticate
- 3) If authentication using already supplied credentials succeeds, try to access the thing again
- 4) If all els fails, give the user a 401 to request new credentials
+ };
- Note that the process could stop already in step 3 with a redirect to login page.
-
- */
-
- authorize(function (ok) {
- if (ok) return next();
- authenticate(function (ok) {
- if (!ok) return failure();
- authorize(function (ok) {
- if (ok) return next();
- failure();
- });
+ step3Authorize = () => {
+ authorize((ok) => {
+ if (ok) return next();
+ failure();
});
- });
-}
+ };
+
+ step1PreAuthenticate();
+};
exports.secret = null;
-exports.expressConfigure = function (hook_name, args, cb) {
+exports.expressConfigure = (hook_name, args, cb) => {
// Measure response time
- args.app.use(function(req, res, next) {
- var stopWatch = stats.timer('httpRequests').start();
- var sendFn = res.send
- res.send = function() {
- stopWatch.end()
- sendFn.apply(res, arguments)
- }
- next()
- })
+ args.app.use((req, res, next) => {
+ const stopWatch = stats.timer('httpRequests').start();
+ const sendFn = res.send;
+ res.send = function() { // function, not arrow, due to use of 'arguments'
+ stopWatch.end();
+ sendFn.apply(res, arguments);
+ };
+ next();
+ });
// If the log level specified in the config file is WARN or ERROR the application server never starts listening to requests as reported in issue #158.
// Not installing the log4js connect logger when the log level has a higher severity than INFO since it would not log at that level anyway.
- if (!(settings.loglevel === "WARN" || settings.loglevel == "ERROR"))
- args.app.use(log4js.connectLogger(httpLogger, { level: log4js.levels.DEBUG, format: ':status, :method :url'}));
+ if (!(settings.loglevel === 'WARN' || settings.loglevel === 'ERROR'))
+ args.app.use(log4js.connectLogger(httpLogger, {level: log4js.levels.DEBUG, format: ':status, :method :url'}));
/* Do not let express create the session, so that we can retain a
* reference to it for socket.io to use. Also, set the key (cookie
@@ -122,6 +137,8 @@ exports.expressConfigure = function (hook_name, args, cb) {
exports.secret = settings.sessionKey;
}
+ const sameSite = settings.ssl ? 'Strict' : 'Lax';
+
args.app.sessionStore = exports.sessionStore;
args.app.use(sessionModule({
secret: exports.secret,
@@ -131,6 +148,12 @@ exports.expressConfigure = function (hook_name, args, cb) {
name: 'express_sid',
proxy: true,
cookie: {
+ /*
+ * Firefox started enforcing sameSite, see https://github.com/ether/etherpad-lite/issues/3989
+ * for details. In response we set it based on if SSL certs are set in Etherpad. Note that if
+ * You use Nginx or so for reverse proxy this may cause problems. Use Certificate pinning to remedy.
+ */
+ sameSite: sameSite,
/*
* The automatic express-session mechanism for determining if the
* application is being served over ssl is similar to the one used for
@@ -157,5 +180,5 @@ exports.expressConfigure = function (hook_name, args, cb) {
args.app.use(cookieParser(settings.sessionKey, {}));
- args.app.use(exports.basicAuth);
-}
+ args.app.use(exports.checkAccess);
+};
diff --git a/src/node/hooks/i18n.js b/src/node/hooks/i18n.js
index 122efdd38cd..2265978bf0e 100644
--- a/src/node/hooks/i18n.js
+++ b/src/node/hooks/i18n.js
@@ -3,9 +3,10 @@ var languages = require('languages4translatewiki')
, path = require('path')
, _ = require('underscore')
, npm = require('npm')
- , plugins = require('ep_etherpad-lite/static/js/pluginfw/plugins.js').plugins
+ , plugins = require('ep_etherpad-lite/static/js/pluginfw/plugin_defs.js').plugins
, semver = require('semver')
, existsSync = require('../utils/path_exists')
+ , settings = require('../utils/Settings')
;
@@ -43,7 +44,7 @@ function getAllLocales() {
//add plugins languages (if any)
for(var pluginName in plugins) extractLangs(path.join(npm.root, pluginName, 'locales'));
- // Build a locale index (merge all locale data)
+ // Build a locale index (merge all locale data other than user-supplied overrides)
var locales = {}
_.each (locales2paths, function(files, langcode) {
locales[langcode]={};
@@ -54,6 +55,22 @@ function getAllLocales() {
});
});
+ // Add custom strings from settings.json
+ // Since this is user-supplied, we'll do some extra sanity checks
+ const wrongFormatErr = Error(
+ "customLocaleStrings in wrong format. See documentation " +
+ "for Customization for Administrators, under Localization.")
+ if (settings.customLocaleStrings) {
+ if (typeof settings.customLocaleStrings !== "object") throw wrongFormatErr
+ _.each(settings.customLocaleStrings, function(overrides, langcode) {
+ if (typeof overrides !== "object") throw wrongFormatErr
+ _.each(overrides, function(localeString, key) {
+ if (typeof localeString !== "string") throw wrongFormatErr
+ locales[langcode][key] = localeString
+ })
+ })
+ }
+
return locales;
}
diff --git a/src/node/server.js b/src/node/server.js
index d98767225fb..a1f62df4ff6 100755
--- a/src/node/server.js
+++ b/src/node/server.js
@@ -21,8 +21,9 @@
* limitations under the License.
*/
-var log4js = require('log4js')
+const log4js = require('log4js')
, NodeVersion = require('./utils/NodeVersion')
+ , UpdateCheck = require('./utils/UpdateCheck')
;
log4js.replaceConsole();
@@ -38,6 +39,9 @@ NodeVersion.enforceMinNodeVersion('10.13.0');
*/
NodeVersion.checkDeprecationStatus('10.13.0', '1.8.3');
+// Check if Etherpad version is up-to-date
+UpdateCheck.check();
+
/*
* start up stats counting system
*/
@@ -57,7 +61,6 @@ npm.load({}, function() {
var db = require('./db/DB');
var plugins = require("ep_etherpad-lite/static/js/pluginfw/plugins");
var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks");
- hooks.plugins = plugins;
db.init()
.then(plugins.update)
diff --git a/src/node/utils/ExportHelper.js b/src/node/utils/ExportHelper.js
index 99921aa3b42..f6ec4486ed8 100644
--- a/src/node/utils/ExportHelper.js
+++ b/src/node/utils/ExportHelper.js
@@ -63,6 +63,13 @@ exports._analyzeLine = function(text, aline, apool){
}
}
}
+ var opIter2 = Changeset.opIterator(aline);
+ if (opIter2.hasNext()){
+ var start = Changeset.opAttributeValue(opIter2.next(), 'start', apool);
+ if (start){
+ line.start = start;
+ }
+ }
}
if (lineMarker){
line.text = text.substring(1);
diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js
index aa7b0abcc4c..d0ebf20de8a 100644
--- a/src/node/utils/ExportHtml.js
+++ b/src/node/utils/ExportHtml.js
@@ -298,7 +298,7 @@ function getHTMLFromAtext(pad, atext, authorColors)
});
}
processNextChars(text.length - idx);
-
+
return _processSpaces(assem.toString());
} // end getLineHTML
var pieces = [css];
@@ -316,7 +316,6 @@ function getHTMLFromAtext(pad, atext, authorColors)
var context;
var line = _analyzeLine(textLines[i], attribLines[i], apool);
var lineContent = getLineHTML(line.text, line.aline);
-
if (line.listLevel)//If we are inside a list
{
context = {
@@ -361,12 +360,50 @@ function getHTMLFromAtext(pad, atext, authorColors)
if (prevPiece.indexOf("
") === 0)
{
- pieces.push("
");
+ /*
+ uncommenting this breaks nested ols..
+ if the previous item is NOT a ul, NOT an ol OR closing li then close the list
+ so we consider this HTML, I inserted ** where it throws a problem in Example Wrong..
+
one
1.1
1.1.1
two
+
+ Note that closing the li then re-opening for another li item here is wrong. The correct markup is
+
one
1.1
1.1.1
two
+
+ Exmaple Right:
one
1.1
1.1.1
two
+ Example Wrong:
one
****
1.1
****
1.1.1
two
+ So it's firing wrong where the current piece is an li and the previous piece is an ol and next piece is an ol
+ So to remedy this we can say if next piece is NOT an OL or UL.
+ // pieces.push("");
+
+ */
+ if( (nextLine.listTypeName === 'number') && (nextLine.text === '') ){
+ // is the listTypeName check needed here? null text might be completely fine!
+ // TODO Check against Uls
+ // don't do anything because the next item is a nested ol openener so we need to keep the li open
+ }else{
+ pieces.push("
");
+ }
+
+
}
if (line.listTypeName === "number")
{
- pieces.push("");
+ // We introduce line.start here, this is useful for continuing Ordered list line numbers
+ // in case you have a bullet in a list IE you Want
+ // 1. hello
+ // * foo
+ // 2. world
+ // Without this line.start logic it would be
+ // 1. hello * foo 1. world because the bullet would kill the OL
+
+ // TODO: This logic could also be used to continue OL with indented content
+ // but that's a job for another day....
+ if(line.start){
+ pieces.push("");
+ }else{
+ pieces.push("");
+ }
}
else
{
@@ -375,13 +412,24 @@ function getHTMLFromAtext(pad, atext, authorColors)
}
}
}
-
- pieces.push("
", context.lineContent);
+ // if we're going up a level we shouldn't be adding..
+ if(context.lineContent){
+ pieces.push("
", context.lineContent);
+ }
// To close list elements
if (nextLine && nextLine.listLevel === line.listLevel && line.listTypeName === nextLine.listTypeName)
{
- pieces.push("
");
+ if(context.lineContent){
+ if( (nextLine.listTypeName === 'number') && (nextLine.text === '') ){
+ // is the listTypeName check needed here? null text might be completely fine!
+ // TODO Check against Uls
+ // don't do anything because the next item is a nested ol openener so we need to keep the li open
+ }else{
+ pieces.push("
");
+ }
+
+ }
}
if ((!nextLine || !nextLine.listLevel || nextLine.listLevel < line.listLevel) || (nextLine && line.listTypeName !== nextLine.listTypeName))
{
diff --git a/src/node/utils/ExportTxt.js b/src/node/utils/ExportTxt.js
index 304f77b8a83..4a9c0ba407a 100644
--- a/src/node/utils/ExportTxt.js
+++ b/src/node/utils/ExportTxt.js
@@ -190,7 +190,12 @@ function getTXTFromAtext(pad, atext, authorColors)
// so we want to do something reasonable there. We also
// want to deal gracefully with blank lines.
// => keeps track of the parents level of indentation
+
+ var listNumbers = {};
+ var prevListLevel;
+
for (var i = 0; i < textLines.length; i++) {
+
var line = _analyzeLine(textLines[i], attribLines[i], apool);
var lineContent = getLineTXT(line.text, line.aline);
@@ -198,15 +203,49 @@ function getTXTFromAtext(pad, atext, authorColors)
lineContent = "* " + lineContent; // add a bullet
}
+ if (line.listTypeName !== "number") {
+ // We're no longer in an OL so we can reset counting
+ for (var key in listNumbers) {
+ delete listNumbers[key];
+ }
+ }
+
if (line.listLevel > 0) {
for (var j = line.listLevel - 1; j >= 0; j--) {
- pieces.push('\t');
+ pieces.push('\t'); // tab indent list numbers..
+ if(!listNumbers[line.listLevel]){
+ listNumbers[line.listLevel] = 0;
+ }
}
if (line.listTypeName == "number") {
- pieces.push(line.listLevel + ". ");
- // This is bad because it doesn't truly reflect what the user
- // sees because browsers do magic on nested