From 5469ce85cdb341316c11b01d4bf7252780290466 Mon Sep 17 00:00:00 2001 From: ilmar Date: Tue, 31 Oct 2017 22:46:24 +0200 Subject: [PATCH 01/11] exportHTML update --- src/node/utils/ExportHtml.js | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index bd177ac4..13140bb5 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -356,7 +356,15 @@ function getHTMLFromAtext(pad, atext, authorColors) } } } - + var context = { + line: line, + lineContent: lineContent, + apool: apool, + attribLine: attribLines[i], + text: textLines[i], + padId: pad.id + } + var lineContentFromHook = hooks.callAll("getLineHTMLForExport", context); if (whichList >= lists.length)//means we are on a deeper level of indentation than the previous line { if(lists.length > 0){ @@ -373,14 +381,14 @@ function getHTMLFromAtext(pad, atext, authorColors) if(toOpen > 0){ pieces.push(new Array(toOpen + 1).join('
    ')) } - pieces.push('
    1. ', lineContent || '
      '); + pieces.push('
      1. ', context.lineContent || '
        '); } else { if(toOpen > 0){ pieces.push(new Array(toOpen + 1).join('
          ')) } - pieces.push('
          • ', lineContent || '
            '); + pieces.push('
            • ', context.lineContent || '
              '); } } //the following code *seems* dead after my patch. @@ -416,16 +424,16 @@ function getHTMLFromAtext(pad, atext, authorColors) if(lists[lists.length - 1][1] == "number") { pieces.push(new Array(toClose+1).join('
      ')) - pieces.push('
    2. ', lineContent || '
      '); + pieces.push('
    3. ', context.lineContent || '
      '); } else { pieces.push(new Array(toClose+1).join('')) - pieces.push('
    4. ', lineContent || '
      '); + pieces.push('
    5. ', context.lineContent || '
      '); } lists = lists.slice(0,whichList+1) } else { - pieces.push('
    6. ', lineContent || '
      '); + pieces.push('
    7. ', context.lineContent || '
      '); } } } @@ -451,16 +459,9 @@ function getHTMLFromAtext(pad, atext, authorColors) padId: pad.id } - var lineContentFromHook = hooks.callAllStr("getLineHTMLForExport", context, " ", " ", ""); + hooks.callAll("getLineHTMLForExport", context); - if (lineContentFromHook) - { - pieces.push(lineContentFromHook, ''); - } - else - { - pieces.push(lineContent, '
      '); - } + pieces.push(context.lineContent, '
      '); } } From 76f211b0bfc273e96a70bd20ac3072d7ac46b324 Mon Sep 17 00:00:00 2001 From: ilmar Date: Wed, 1 Nov 2017 09:48:23 +0200 Subject: [PATCH 02/11] ExportHtml.js update --- src/node/utils/ExportHtml.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index 13140bb5..e6bfef37 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -364,7 +364,7 @@ function getHTMLFromAtext(pad, atext, authorColors) text: textLines[i], padId: pad.id } - var lineContentFromHook = hooks.callAll("getLineHTMLForExport", context); + hooks.callAll("getLineHTMLForExport", context); if (whichList >= lists.length)//means we are on a deeper level of indentation than the previous line { if(lists.length > 0){ From 30400509ba25f57eaa96cd141f12a84917decd00 Mon Sep 17 00:00:00 2001 From: ilmar Date: Fri, 3 Nov 2017 10:47:54 +0200 Subject: [PATCH 03/11] added tiblus ep_prefs_different_cookie_for_different_protocol --- src/static/js/pad.js | 2 +- src/static/js/pad_cookie.js | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/static/js/pad.js b/src/static/js/pad.js index c967e461..9c3de091 100644 --- a/src/static/js/pad.js +++ b/src/static/js/pad.js @@ -453,7 +453,7 @@ var pad = { // This will check if the prefs-cookie is set. // Otherwise it shows up a message to the user. padcookie.init(); - if (!readCookie("prefs")) + if (!padcookie.isCookiesEnabled()) { $('#loading').hide(); $('#noCookie').show(); diff --git a/src/static/js/pad_cookie.js b/src/static/js/pad_cookie.js index b563a7e6..e43ac874 100644 --- a/src/static/js/pad_cookie.js +++ b/src/static/js/pad_cookie.js @@ -23,6 +23,8 @@ var padcookie = (function() { + var cookieName = isHttpsScheme() ? "prefs" : "prefsHttp"; + function getRawCookie() { // returns null if can't get cookie text @@ -31,7 +33,7 @@ var padcookie = (function() return null; } // look for (start of string OR semicolon) followed by whitespace followed by prefs=(something); - var regexResult = document.cookie.match(/(?:^|;)\s*prefs=([^;]*)(?:;|$)/); + var regexResult = document.cookie.match(new RegExp("(?:^|;)\s*" + cookieName + "=([^;]*)(?:;|$)")); if ((!regexResult) || (!regexResult[1])) { return null; @@ -44,7 +46,7 @@ var padcookie = (function() var expiresDate = new Date(); expiresDate.setFullYear(3000); var secure = isHttpsScheme() ? ";secure" : ""; - document.cookie = ('prefs=' + safeText + ';expires=' + expiresDate.toGMTString() + secure); + document.cookie = (cookieName + "=" + safeText + ";expires=" + expiresDate.toGMTString() + secure); } function parseCookie(text) @@ -122,6 +124,9 @@ var padcookie = (function() { return wasNoCookie; }, + isCookiesEnabled: function() { + return !!getRawCookie(); + }, getPref: function(prefName) { return cookieData[prefName]; From cf82177b36f1b5f38a481614620425d49102ec01 Mon Sep 17 00:00:00 2001 From: ilmar Date: Mon, 26 Mar 2018 17:57:00 +0300 Subject: [PATCH 04/11] ueberDB2 update --- src/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.json b/src/package.json index ca86258f..78b46645 100644 --- a/src/package.json +++ b/src/package.json @@ -17,7 +17,7 @@ "etherpad-require-kernel" : "1.0.9", "resolve" : "1.1.7", "socket.io" : "1.6.0", - "ueberdb2" : "0.3.0", + "ueberdb2" : "git+https://github.com/tiblu/ueberDB#master", "express" : "4.13.4", "express-session" : "1.13.0", "cookie-parser" : "1.3.4", From a4819b21f20f25c480094849a0de354c5ec23b07 Mon Sep 17 00:00:00 2001 From: Mikk Andresen Date: Wed, 4 Apr 2018 13:40:02 +0300 Subject: [PATCH 05/11] Upgrade Ueberdb2 to 0.3.7 to fix https://github.com/ether/etherpad-lite/issues/3348 --- src/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.json b/src/package.json index 4a0c96a7..6f16e3cf 100644 --- a/src/package.json +++ b/src/package.json @@ -17,7 +17,7 @@ "etherpad-require-kernel" : "1.0.9", "resolve" : "1.1.7", "socket.io" : "1.7.3", - "ueberdb2" : "git+https://github.com/tiblu/ueberDB#master", + "ueberdb2" : "0.3.7", "express" : "4.13.4", "express-session" : "1.13.0", "cookie-parser" : "1.3.4", From b4ad7cf45290113df8d539db7f67f94938fb7762 Mon Sep 17 00:00:00 2001 From: ilmar Date: Thu, 5 Apr 2018 23:27:02 +0300 Subject: [PATCH 06/11] Export lists fix + code linting and readability update --- src/node/utils/ExportHtml.js | 984 +++++++++++++++++------------------ 1 file changed, 464 insertions(+), 520 deletions(-) diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index e6bfef37..d885c682 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -1,3 +1,5 @@ +'use strict'; + /** * Copyright 2009 Google Inc. * @@ -15,572 +17,514 @@ */ -var async = require("async"); -var Changeset = require("ep_etherpad-lite/static/js/Changeset"); -var padManager = require("../db/PadManager"); -var ERR = require("async-stacktrace"); +var async = require('async'); +var Changeset = require('ep_etherpad-lite/static/js/Changeset'); +var padManager = require('../db/PadManager'); +var eRR = require('async-stacktrace'); var _ = require('underscore'); var Security = require('ep_etherpad-lite/static/js/security'); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); var eejs = require('ep_etherpad-lite/node/eejs'); var _analyzeLine = require('./ExportHelper')._analyzeLine; var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace; +// copied from ACE +var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; +var _REGEX_URLCHAR = new RegExp('(' + (/[-:@a-zA-Z0-9_.,~%+/\\?=&#;()$]/).source + '|' + _REGEX_WORDCHAR.source + ')'); +var _REGEX_URL = new RegExp(/(?:(?:https?|s?ftp|ftps|file|smb|afp|nfs|(x-)?man|gopher|txmt):\/\/|mailto:)/.source + _REGEX_URLCHAR.source + '*(?![:.,;])' + _REGEX_URLCHAR.source, 'g'); -function getPadHTML(pad, revNum, callback) -{ - var atext = pad.atext; - var html; - async.waterfall([ - // fetch revision atext - function (callback) - { - if (revNum != undefined) - { - pad.getInternalRevisionAText(revNum, function (err, revisionAtext) - { - if(ERR(err, callback)) return; - atext = revisionAtext; - callback(); - }); +// returns null if no URLs, or [[startIndex1, url1], [startIndex2, url2], ...] +// copied from ACE +function _processSpaces (s) { + var doesWrap = true; + if (s.indexOf('<') < 0 && !doesWrap) { + // short-cut + return s.replace(/ /g, ' '); } - else - { - callback(null); + var parts = []; + s.replace(/<[^>]*>?| |[^ <]+/g, function (m) { + parts.push(m); + }); + if (doesWrap) { + var endOfLine = true; + var beforeSpace = false; + // last space in a run is normal, others are nbsp, + // end of line is nbsp + for (var i = parts.length - 1; i >= 0; i--) { + var p = parts[i]; + if (p === ' ') { + if (endOfLine || beforeSpace) parts[i] = ' '; + endOfLine = false; + beforeSpace = true; + } else if (p.charAt(0) !== '<') { + endOfLine = false; + beforeSpace = false; + } + } + // beginning of line is nbsp + for (i = 0; i < parts.length; i++) { + p = parts[i]; + if (p === ' ') { + parts[i] = ' '; + break; + } else if (p.charAt(0) !== '<') { + break; + } + } + } else { + for (i = 0; i < parts.length; i++) { + p = parts[i]; + if (p === ' ') { + parts[i] = ' '; + } + } } - }, - // convert atext to html - - - function (callback) - { - html = getHTMLFromAtext(pad, atext); - callback(null); - }], - // run final callback - - - function (err) - { - if(ERR(err, callback)) return; - callback(null, html); - }); + return parts.join(''); } -exports.getPadHTML = getPadHTML; -exports.getHTMLFromAtext = getHTMLFromAtext; +function _findURLs (text) { + _REGEX_URL.lastIndex = 0; + var urls = null; + var execResult; + while (execResult = _REGEX_URL.exec(text)) { + urls = urls || []; + var startIndex = execResult.index; + var url = execResult[0]; + urls.push([startIndex, url]); + } -function getHTMLFromAtext(pad, atext, authorColors) -{ - var apool = pad.apool(); - var textLines = atext.text.slice(0, -1).split('\n'); - var attribLines = Changeset.splitAttributionLines(atext.attribs, atext.text); + return urls; +} - var tags = ['h1', 'h2', 'strong', 'em', 'u', 's']; - var props = ['heading1', 'heading2', 'bold', 'italic', 'underline', 'strikethrough']; +/** + * Return docx from input html + * + * @param {text} [pad] text + * @param {text} [atext] text + * @param {text} [authorColors] text + * + * @returns {html} html + * + * @public + */ +function getHTMLFromAtext (pad, atext, authorColors) { + var apool = pad.apool(); + var textLines = atext.text.slice(0, -1).split('\n'); + var attribLines = Changeset.splitAttributionLines(atext.attribs, atext.text); - // prepare tags stored as ['tag', true] to be exported - hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){ - newProps.forEach(function (propName, i){ - tags.push(propName); - props.push(propName); + var tags = ['h1', 'h2', 'strong', 'em', 'u', 's']; + var props = ['heading1', 'heading2', 'bold', 'italic', 'underline', 'strikethrough']; + + // prepare tags stored as ['tag', true] to be exported + hooks.aCallAll('exportHtmlAdditionalTags', pad, function (err, newProps) { + if (err) { + return err; + } + newProps.forEach(function (propName) { + tags.push(propName); + props.push(propName); + }); }); - }); - // prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML - // with tags like - hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){ - newProps.forEach(function (propName, i){ - tags.push('span data-' + propName[0] + '="' + propName[1] + '"'); - props.push(propName); + // prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML + // with tags like + hooks.aCallAll('exportHtmlAdditionalTagsWithData', pad, function (err, newProps) { + if (err) { + return err; + } + newProps.forEach(function (propName) { + tags.push('span data-' + propName[0] + '="' + propName[1] + '"'); + props.push(propName); + }); }); - }); - // holds a map of used styling attributes (*1, *2, etc) in the apool - // and maps them to an index in props - // *3:2 -> the attribute *3 means strong - // *2:5 -> the attribute *2 means s(trikethrough) - var anumMap = {}; - var css = ""; + // holds a map of used styling attributes (*1, *2, etc) in the apool + // and maps them to an index in props + // *3:2 -> the attribute *3 means strong + // *2:5 -> the attribute *2 means s(trikethrough) + var anumMap = {}; + var css = ''; - var stripDotFromAuthorID = function(id){ - return id.replace(/\./g,'_'); - }; + var stripDotFromAuthorID = function (id) { + return id.replace(/\./g, '_'); + }; - if(authorColors){ - css+="'; } - css+=""; - } + // iterates over all props(h1,h2,strong,...), checks if it is used in + // this pad, and if yes puts its attrib id->props value into anumMap + props.forEach(function (propName, i) { + var attrib = [propName, true]; + if (_.isArray(propName)) { + // propName can be in the form of ['color', 'red'], + // see hook exportHtmlAdditionalTagsWithData + attrib = propName; + } + var propTrueNum = apool.putAttrib(attrib, true); + if (propTrueNum >= 0) { + anumMap[propTrueNum] = i; + } + }); - // iterates over all props(h1,h2,strong,...), checks if it is used in - // this pad, and if yes puts its attrib id->props value into anumMap - props.forEach(function (propName, i) - { - var attrib = [propName, true]; - if (_.isArray(propName)) { - // propName can be in the form of ['color', 'red'], - // see hook exportHtmlAdditionalTagsWithData - attrib = propName; - } - var propTrueNum = apool.putAttrib(attrib, true); - if (propTrueNum >= 0) - { - anumMap[propTrueNum] = i; - } - }); - - function getLineHTML(text, attribs) - { + function getLineHTML (text, attribs) { // Use order of tags (b/i/u) as order of nesting, for simplicity // and decent nesting. For example, // Just bold Bold and italics Just italics // becomes // Just bold Bold and italics Just italics - var taker = Changeset.stringIterator(text); - var assem = Changeset.stringAssembler(); - var openTags = []; + var taker = Changeset.stringIterator(text); + var assem = Changeset.stringAssembler(); + var openTags = []; - function getSpanClassFor(i){ - //return if author colors are disabled - if (!authorColors) return false; + function getSpanClassFor (i) { + //return if author colors are disabled + if (!authorColors) return false; - var property = props[i]; + var property = props[i]; - // we are not insterested on properties in the form of ['color', 'red'], - // see hook exportHtmlAdditionalTagsWithData - if (_.isArray(property)) { - return false; - } + // we are not insterested on properties in the form of ['color', 'red'], + // see hook exportHtmlAdditionalTagsWithData + if (_.isArray(property)) { + return false; + } - if(property.substr(0,6) === "author"){ - return stripDotFromAuthorID(property); - } + if (property.substr(0, 6) === 'author') { + return stripDotFromAuthorID(property); + } - if(property === "removed"){ - return "removed"; - } + if (property === 'removed') { + return 'removed'; + } - return false; + return false; + } + + // tags added by exportHtmlAdditionalTagsWithData will be exported as with + // data attributes + function isSpanWithData (i) { + var property = props[i]; + + return _.isArray(property); + } + + function emitOpenTag (i) { + openTags.unshift(i); + var spanClass = getSpanClassFor(i); + + if (spanClass) { + assem.append(''); + } else { + assem.append('<'); + assem.append(tags[i]); + assem.append('>'); + } + } + + // this closes an open tag and removes its reference from openTags + function emitCloseTag (i) { + openTags.shift(); + var spanClass = getSpanClassFor(i); + var spanWithData = isSpanWithData(i); + + if (spanClass || spanWithData) { + assem.append(''); + } else { + assem.append(''); + } + } + + var urls = _findURLs(text); + + var idx = 0; + + function processNextChars (numChars) { + if (numChars <= 0) { + return; + } + + var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars)); + idx += numChars; + + // this iterates over every op string and decides which tags to open or to close + // based on the attribs used + while (iter.hasNext()) { + var o = iter.next(); + var usedAttribs = []; + + // mark all attribs as used + Changeset.eachAttribNumber(o.attribs, function (a) { + if (a in anumMap) { + usedAttribs.push(anumMap[a]); // i = 0 => bold, etc. + } + }); + var outermostTag = -1; + // find the outer most open tag that is no longer used + for (var i = openTags.length - 1; i >= 0; i--) { + if (usedAttribs.indexOf(openTags[i]) === -1) { + outermostTag = i; + break; + } + } + + // close all tags upto the outer most + if (outermostTag !== -1) { + while (outermostTag >= 0) { + emitCloseTag(openTags[0]); + outermostTag--; + } + } + + // open all tags that are used but not open + for (i = 0; i < usedAttribs.length; i++) { + if (openTags.indexOf(usedAttribs[i]) === -1) { + emitOpenTag(usedAttribs[i]); + } + } + + var chars = o.chars; + if (o.lines) { + chars--; // exclude newline at end of line, if present + } + + var s = taker.take(chars); + + //removes the characters with the code 12. Don't know where they come + //from but they break the abiword parser and are completly useless + s = s.replace(String.fromCharCode(12), ''); + + assem.append(_encodeWhitespace(Security.escapeHTML(s))); + } // end iteration over spans in line + + // close all the tags that are open after the last op + while (openTags.length > 0) { + emitCloseTag(openTags[0]); + } + } // end processNextChars + + if (urls) { + urls.forEach(function (urlData) { + var startIndex = urlData[0]; + var url = urlData[1]; + var urlLength = url.length; + processNextChars(startIndex - idx); + assem.append(''); + processNextChars(urlLength); + assem.append(''); + }); + } + + processNextChars(text.length - idx); + + return _processSpaces(assem.toString()); + } // end getLineHTML + + var pieces = [css]; + + // Need to deal with constraints imposed on HTML lists; can + // only gain one level of nesting at once, can't change type + // mid-list, etc. + // People might use weird indenting, e.g. skip a level, + // so we want to do something reasonable there. We also + // want to deal gracefully with blank lines. + // => keeps track of the parents level of indentation + var lists = []; // e.g. [[1,'bullet'], [3,'bullet'], ...] + var listLevels = []; + for (var i = 0; i < textLines.length; i++) { + var context; + var line = _analyzeLine(textLines[i], attribLines[i], apool); + var lineContent = getLineHTML(line.text, line.aline); + listLevels.push(line.listLevel); + + if (line.listLevel) { //If we are inside a list + // do list stuff + var whichList = -1; // index into lists or -1 + if (line.listLevel) { + whichList = lists.length; + for (var j = lists.length - 1; j >= 0; j--) { + if (line.listLevel <= lists[j][0]) { + whichList = j; + } + } + } + context = { + line: line, + lineContent: lineContent, + apool: apool, + attribLine: attribLines[i], + text: textLines[i], + padId: pad.id + }; + hooks.callAll('getLineHTMLForExport', context); + if (whichList >= lists.length) { + if (lists.length > 0) { + pieces.push('
      8. '); + } + lists.push([line.listLevel, line.listTypeName]); + + // if there is a previous list we need to open x tags, where x is the difference of the levels + // if there is no previous list we need to open x tags, where x is the wanted level + var toOpen = lists.length > 1 ? line.listLevel - lists[lists.length - 2][0] - 1 : line.listLevel - 1 + + if (line.listTypeName === 'number') { + if (toOpen > 0) { + pieces.push(new Array(toOpen + 1).join('
      1. ')); + } + pieces.push('
        1. ', context.lineContent || '
          '); + } else { + if (toOpen > 0) { + pieces.push(new Array(toOpen + 1).join('
          • ')); + } + pieces.push('
            • ', context.lineContent || '
              '); + } + } else { //means we are getting closer to the lowest level of indentation or are at the same level + var toClose = lists.length > 0 ? listLevels[listLevels.length - 2] - line.listLevel : 0 + if (toClose > 0) { + pieces.push('
              • '); + if (lists[lists.length - 1][1] === 'number') { + pieces.push(new Array(toClose + 1).join('
        2. ')); + pieces.push('
      2. ', context.lineContent || '
        '); + } else { + pieces.push(new Array(toClose + 1).join('
        3. ')); + pieces.push('
      3. ', context.lineContent || '
        '); + } + lists = lists.slice(0, whichList + 1); + } else { + pieces.push('
      4. ', context.lineContent || '
        '); + } + } + } else { //outside any list, need to close line.listLevel of lists + + if (lists.length > 0) { + if (lists[lists.length - 1][1] === 'number') { + pieces.push('
      '); + pieces.push(new Array(listLevels[listLevels.length - 2]).join('
    ')); + } else { + pieces.push(''); + pieces.push(new Array(listLevels[listLevels.length - 2]).join('')); + } + } + lists = []; + + context = { + line: line, + lineContent: lineContent, + apool: apool, + attribLine: attribLines[i], + text: textLines[i], + padId: pad.id + }; + + hooks.callAll('getLineHTMLForExport', context); + pieces.push(context.lineContent, '
    '); + } } - // tags added by exportHtmlAdditionalTagsWithData will be exported as with - // data attributes - function isSpanWithData(i){ - var property = props[i]; - return _.isArray(property); - } - - function emitOpenTag(i) - { - openTags.unshift(i); - var spanClass = getSpanClassFor(i); - - if(spanClass){ - assem.append(''); - } else { - assem.append('<'); - assem.append(tags[i]); - assem.append('>'); - } - } - - // this closes an open tag and removes its reference from openTags - function emitCloseTag(i) - { - openTags.shift(); - var spanClass = getSpanClassFor(i); - var spanWithData = isSpanWithData(i); - - if(spanClass || spanWithData){ - assem.append(''); - } else { - assem.append(''); - } - } - - var urls = _findURLs(text); - - var idx = 0; - - function processNextChars(numChars) - { - if (numChars <= 0) - { - return; - } - - var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars)); - idx += numChars; - - // this iterates over every op string and decides which tags to open or to close - // based on the attribs used - while (iter.hasNext()) - { - var o = iter.next(); - var usedAttribs = []; - - // mark all attribs as used - Changeset.eachAttribNumber(o.attribs, function (a) - { - if (a in anumMap) - { - usedAttribs.push(anumMap[a]); // i = 0 => bold, etc. - } - }); - var outermostTag = -1; - // find the outer most open tag that is no longer used - for (var i = openTags.length - 1; i >= 0; i--) - { - if (usedAttribs.indexOf(openTags[i]) === -1) - { - outermostTag = i; - break; - } - } - - // close all tags upto the outer most - if (outermostTag != -1) - { - while ( outermostTag >= 0 ) - { - emitCloseTag(openTags[0]); - outermostTag--; - } - } - - // open all tags that are used but not open - for (i=0; i < usedAttribs.length; i++) - { - if (openTags.indexOf(usedAttribs[i]) === -1) - { - emitOpenTag(usedAttribs[i]) - } - } - - var chars = o.chars; - if (o.lines) - { - chars--; // exclude newline at end of line, if present - } - - var s = taker.take(chars); - - //removes the characters with the code 12. Don't know where they come - //from but they break the abiword parser and are completly useless - s = s.replace(String.fromCharCode(12), ""); - - assem.append(_encodeWhitespace(Security.escapeHTML(s))); - } // end iteration over spans in line - - // close all the tags that are open after the last op - while (openTags.length > 0) - { - emitCloseTag(openTags[0]) - } - } // end processNextChars - if (urls) - { - urls.forEach(function (urlData) - { - var startIndex = urlData[0]; - var url = urlData[1]; - var urlLength = url.length; - processNextChars(startIndex - idx); - assem.append(''); - processNextChars(urlLength); - assem.append(''); - }); - } - processNextChars(text.length - idx); - - return _processSpaces(assem.toString()); - } // end getLineHTML - var pieces = [css]; - - // Need to deal with constraints imposed on HTML lists; can - // only gain one level of nesting at once, can't change type - // mid-list, etc. - // People might use weird indenting, e.g. skip a level, - // so we want to do something reasonable there. We also - // want to deal gracefully with blank lines. - // => keeps track of the parents level of indentation - var lists = []; // e.g. [[1,'bullet'], [3,'bullet'], ...] - var listLevels = [] - for (var i = 0; i < textLines.length; i++) - { - var line = _analyzeLine(textLines[i], attribLines[i], apool); - var lineContent = getLineHTML(line.text, line.aline); - listLevels.push(line.listLevel) - - if (line.listLevel)//If we are inside a list - { - // do list stuff - var whichList = -1; // index into lists or -1 - if (line.listLevel) - { - whichList = lists.length; - for (var j = lists.length - 1; j >= 0; j--) - { - if (line.listLevel <= lists[j][0]) - { - whichList = j; - } - } - } - var context = { - line: line, - lineContent: lineContent, - apool: apool, - attribLine: attribLines[i], - text: textLines[i], - padId: pad.id - } - hooks.callAll("getLineHTMLForExport", context); - if (whichList >= lists.length)//means we are on a deeper level of indentation than the previous line - { - if(lists.length > 0){ - pieces.push('') - } - lists.push([line.listLevel, line.listTypeName]); - - // if there is a previous list we need to open x tags, where x is the difference of the levels - // if there is no previous list we need to open x tags, where x is the wanted level - var toOpen = lists.length > 1 ? line.listLevel - lists[lists.length - 2][0] - 1 : line.listLevel - 1 - - if(line.listTypeName == "number") - { - if(toOpen > 0){ - pieces.push(new Array(toOpen + 1).join('
      ')) - } - pieces.push('
      1. ', context.lineContent || '
        '); - } - else - { - if(toOpen > 0){ - pieces.push(new Array(toOpen + 1).join('
          ')) - } - pieces.push('
          • ', context.lineContent || '
            '); - } - } - //the following code *seems* dead after my patch. - //I keep it just in case I'm wrong... - /*else if (whichList == -1)//means we are not inside a list - { - if (line.text) - { - console.log('trace 1'); - // non-blank line, end all lists - if(line.listTypeName == "number") - { - pieces.push(new Array(lists.length + 1).join('
      ')); - } - else - { - pieces.push(new Array(lists.length + 1).join('')); - } - lists.length = 0; - pieces.push(lineContent, '
      '); - } - else - { - console.log('trace 2'); - pieces.push('

      '); - } - }*/ - else//means we are getting closer to the lowest level of indentation or are at the same level - { - var toClose = lists.length > 0 ? listLevels[listLevels.length - 2] - line.listLevel : 0 - if( toClose > 0){ - pieces.push('') - if(lists[lists.length - 1][1] == "number") - { - pieces.push(new Array(toClose+1).join('
    ')) - pieces.push('
  1. ', context.lineContent || '
    '); - } - else - { - pieces.push(new Array(toClose+1).join('')) - pieces.push('
  2. ', context.lineContent || '
    '); - } - lists = lists.slice(0,whichList+1) + for (var k = lists.length - 1; k >= 0; k--) { + if (lists[k][1] === 'number') { + pieces.push('
'); } else { - pieces.push('
  • ', context.lineContent || '
    '); + pieces.push('
    • '); } - } } - else//outside any list, need to close line.listLevel of lists - { - if(lists.length > 0){ - if(lists[lists.length - 1][1] == "number"){ - pieces.push(''); - pieces.push(new Array(listLevels[listLevels.length - 2]).join('')) - } else { - pieces.push(''); - pieces.push(new Array(listLevels[listLevels.length - 2]).join('')) - } - } - lists = [] - var context = { - line: line, - lineContent: lineContent, - apool: apool, - attribLine: attribLines[i], - text: textLines[i], - padId: pad.id - } - - hooks.callAll("getLineHTMLForExport", context); - - pieces.push(context.lineContent, '
    '); - } - } - - for (var k = lists.length - 1; k >= 0; k--) - { - if(lists[k][1] == "number") - { - pieces.push(''); - } - else - { - pieces.push(''); - } - } - - return pieces.join(''); + return pieces.join(''); } -exports.getPadHTMLDocument = function (padId, revNum, callback) -{ - padManager.getPad(padId, function (err, pad) - { - if(ERR(err, callback)) return; +function getPadHTML (pad, revNum, callback) { + var atext = pad.atext; + var html; + async.waterfall( + [ + // fetch revision atext + function (callback) { + if (revNum !== undefined) { + pad.getInternalRevisionAText(revNum, function (err, revisionAtext) { + if (eRR(err, callback)) return; + atext = revisionAtext; - var stylesForExportCSS = ""; - // Include some Styles into the Head for Export - hooks.aCallAll("stylesForExport", padId, function(err, stylesForExport){ - stylesForExport.forEach(function(css){ - stylesForExportCSS += css; - }); + return callback(); + }); + } else { + return callback(null); + } + }, - getPadHTML(pad, revNum, function (err, html) - { - if(ERR(err, callback)) return; - var exportedDoc = eejs.require("ep_etherpad-lite/templates/export_html.html", { - body: html, - padId: Security.escapeHTML(padId), - extraCSS: stylesForExportCSS + // convert atext to html + + + function (callback) { + html = getHTMLFromAtext(pad, atext); + callback(null); + } + ], + // run final callback + + + function (err) { + if (eRR(err, callback)) return; + callback(null, html); + } + ); +} + +exports.getPadHTML = getPadHTML; +exports.getHTMLFromAtext = getHTMLFromAtext; +exports.getPadHTMLDocument = function (padId, revNum, callback) { + padManager.getPad(padId, function (err, pad) { + if (eRR(err, callback)) return; + + var stylesForExportCSS = ''; + // Include some Styles into the Head for Export + hooks.aCallAll('stylesForExport', padId, function (err, stylesForExport) { + if (eRR(err, callback)) return; + stylesForExport.forEach(function (css) { + stylesForExportCSS += css; + }); + + getPadHTML(pad, revNum, function (err, html) { + if (eRR(err, callback)) return; + var exportedDoc = eejs.require('ep_etherpad-lite/templates/export_html.html', { + body: html, + padId: Security.escapeHTML(padId), + extraCSS: stylesForExportCSS + }); + callback(null, exportedDoc); + }); }); - callback(null, exportedDoc); - }); }); - }); }; - -// copied from ACE -var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; -var _REGEX_SPACE = /\s/; -var _REGEX_URLCHAR = new RegExp('(' + /[-:@a-zA-Z0-9_.,~%+\/\\?=&#;()$]/.source + '|' + _REGEX_WORDCHAR.source + ')'); -var _REGEX_URL = new RegExp(/(?:(?:https?|s?ftp|ftps|file|smb|afp|nfs|(x-)?man|gopher|txmt):\/\/|mailto:)/.source + _REGEX_URLCHAR.source + '*(?![:.,;])' + _REGEX_URLCHAR.source, 'g'); - -// returns null if no URLs, or [[startIndex1, url1], [startIndex2, url2], ...] - - -function _findURLs(text) -{ - _REGEX_URL.lastIndex = 0; - var urls = null; - var execResult; - while ((execResult = _REGEX_URL.exec(text))) - { - urls = (urls || []); - var startIndex = execResult.index; - var url = execResult[0]; - urls.push([startIndex, url]); - } - - return urls; -} - - -// copied from ACE -function _processSpaces(s){ - var doesWrap = true; - if (s.indexOf("<") < 0 && !doesWrap){ - // short-cut - return s.replace(/ /g, ' '); - } - var parts = []; - s.replace(/<[^>]*>?| |[^ <]+/g, function (m){ - parts.push(m); - }); - if (doesWrap){ - var endOfLine = true; - var beforeSpace = false; - // last space in a run is normal, others are nbsp, - // end of line is nbsp - for (var i = parts.length - 1; i >= 0; i--){ - var p = parts[i]; - if (p == " "){ - if (endOfLine || beforeSpace) parts[i] = ' '; - endOfLine = false; - beforeSpace = true; - } - else if (p.charAt(0) != "<"){ - endOfLine = false; - beforeSpace = false; - } - } - // beginning of line is nbsp - for (i = 0; i < parts.length; i++){ - p = parts[i]; - if (p == " "){ - parts[i] = ' '; - break; - } - else if (p.charAt(0) != "<"){ - break; - } - } - } - else - { - for (i = 0; i < parts.length; i++){ - p = parts[i]; - if (p == " "){ - parts[i] = ' '; - } - } - } - return parts.join(''); -} From 8502c04beeedf86e7b6aa425f0cf77fe3a4293bd Mon Sep 17 00:00:00 2001 From: ilmar Date: Mon, 9 Apr 2018 15:37:28 +0300 Subject: [PATCH 07/11] html lists export fix --- src/node/utils/ExportHtml.js | 128 +++++++++++++++++------------------ 1 file changed, 63 insertions(+), 65 deletions(-) diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index d885c682..84909f8c 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -362,25 +362,14 @@ function getHTMLFromAtext (pad, atext, authorColors) { // so we want to do something reasonable there. We also // want to deal gracefully with blank lines. // => keeps track of the parents level of indentation - var lists = []; // e.g. [[1,'bullet'], [3,'bullet'], ...] - var listLevels = []; + var openLists = []; for (var i = 0; i < textLines.length; i++) { var context; var line = _analyzeLine(textLines[i], attribLines[i], apool); var lineContent = getLineHTML(line.text, line.aline); - listLevels.push(line.listLevel); if (line.listLevel) { //If we are inside a list - // do list stuff - var whichList = -1; // index into lists or -1 - if (line.listLevel) { - whichList = lists.length; - for (var j = lists.length - 1; j >= 0; j--) { - if (line.listLevel <= lists[j][0]) { - whichList = j; - } - } - } + context = { line: line, lineContent: lineContent, @@ -389,57 +378,74 @@ function getHTMLFromAtext (pad, atext, authorColors) { text: textLines[i], padId: pad.id }; + var prevLine = null; + var nextLine = null; + if (i > 0) { + prevLine = _analyzeLine(textLines[i - 1], attribLines[i - 1], apool); + } + if (i < textLines.length) { + nextLine = _analyzeLine(textLines[i + 1], attribLines[i + 1], apool); + } hooks.callAll('getLineHTMLForExport', context); - if (whichList >= lists.length) { - if (lists.length > 0) { - pieces.push(''); - } - lists.push([line.listLevel, line.listTypeName]); - - // if there is a previous list we need to open x tags, where x is the difference of the levels - // if there is no previous list we need to open x tags, where x is the wanted level - var toOpen = lists.length > 1 ? line.listLevel - lists[lists.length - 2][0] - 1 : line.listLevel - 1 - - if (line.listTypeName === 'number') { - if (toOpen > 0) { - pieces.push(new Array(toOpen + 1).join('
    1. ')); + //To create list parent elements + if ((!prevLine || prevLine.listLevel !== line.listLevel) || (prevLine && line.listTypeName !== prevLine.listTypeName)) { + //pieces.push('
        '); + var exists = _.find(openLists, function (item) { + return (item.level === line.listLevel && item.type === line.listTypeName); + }); + if (!exists) { + var prevLevel = prevLine.listLevel || 0; + if (prevLine && line.listTypeName !== prevLine.listTypeName) { + prevLevel = 0; } - pieces.push('
        1. ', context.lineContent || '
          '); - } else { - if (toOpen > 0) { - pieces.push(new Array(toOpen + 1).join('
          • ')); - } - pieces.push('
            • ', context.lineContent || '
              '); + + for (var diff = prevLevel; diff < line.listLevel; diff++) { + openLists.push({level: diff, type: line.listTypeName}); + var prevPiece = pieces[pieces.length - 1]; + + if (prevPiece.indexOf('') === 0) { + pieces.push('
            • '); + } + + if (line.listTypeName === 'number') { + pieces.push('
                '); + } else { + pieces.push('
                  '); + } + } } - } else { //means we are getting closer to the lowest level of indentation or are at the same level - var toClose = lists.length > 0 ? listLevels[listLevels.length - 2] - line.listLevel : 0 - if (toClose > 0) { - pieces.push(''); - if (lists[lists.length - 1][1] === 'number') { - pieces.push(new Array(toClose + 1).join('
              • ')); - pieces.push('
            • ', context.lineContent || '
              '); + + } + + pieces.push('
            • ', context.lineContent); + + // To close list elements + if (nextLine && nextLine.listLevel === line.listLevel && line.listTypeName === nextLine.listTypeName) { + pieces.push('
              • '); + } + if ((!nextLine || !nextLine.listLevel || nextLine.listLevel < line.listLevel) || (nextLine && line.listTypeName !== nextLine.listTypeName)) { + var nextLevel = nextLine.listLevel || 0; + if (nextLine && line.listTypeName !== nextLine.listTypeName) { + nextLevel = 0; + } + + for (var diff = nextLevel; diff < line.listLevel; diff++) { + openLists = openLists.filter(function(el) { + return el.level !== diff && el.type !== line.listTypeName; + }); + + if (pieces[pieces.length - 1].indexOf(''); + } + + if (line.listTypeName === 'number') { + pieces.push('
        '); } else { - pieces.push(new Array(toClose + 1).join('
      2. ')); - pieces.push('
    2. ', context.lineContent || '
      '); + pieces.push(''); } - lists = lists.slice(0, whichList + 1); - } else { - pieces.push('
    3. ', context.lineContent || '
      '); - } + } } } else { //outside any list, need to close line.listLevel of lists - - if (lists.length > 0) { - if (lists[lists.length - 1][1] === 'number') { - pieces.push('
    '); - pieces.push(new Array(listLevels[listLevels.length - 2]).join('')); - } else { - pieces.push(''); - pieces.push(new Array(listLevels[listLevels.length - 2]).join('')); - } - } - lists = []; - context = { line: line, lineContent: lineContent, @@ -454,14 +460,6 @@ function getHTMLFromAtext (pad, atext, authorColors) { } } - for (var k = lists.length - 1; k >= 0; k--) { - if (lists[k][1] === 'number') { - pieces.push(''); - } else { - pieces.push(''); - } - } - return pieces.join(''); } From 517b249394f5c284a1d0056b941947cdcd6bc4b8 Mon Sep 17 00:00:00 2001 From: ilmar Date: Tue, 10 Apr 2018 00:08:42 +0300 Subject: [PATCH 08/11] D --- src/node/utils/ExportHtml.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index 84909f8c..cc8d8420 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -483,8 +483,6 @@ function getPadHTML (pad, revNum, callback) { }, // convert atext to html - - function (callback) { html = getHTMLFromAtext(pad, atext); callback(null); From d6fa065ef2ffbb56f7f553f7e21794b6a6651e41 Mon Sep 17 00:00:00 2001 From: ilmar Date: Tue, 24 Apr 2018 12:13:31 +0300 Subject: [PATCH 09/11] export html to original structure --- src/node/utils/ExportHtml.js | 957 ++++++++++++++++++----------------- 1 file changed, 497 insertions(+), 460 deletions(-) diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index cc8d8420..5276553a 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -1,5 +1,3 @@ -'use strict'; - /** * Copyright 2009 Google Inc. * @@ -17,510 +15,549 @@ */ -var async = require('async'); -var Changeset = require('ep_etherpad-lite/static/js/Changeset'); -var padManager = require('../db/PadManager'); -var eRR = require('async-stacktrace'); +var async = require("async"); +var Changeset = require("ep_etherpad-lite/static/js/Changeset"); +var padManager = require("../db/PadManager"); +var ERR = require("async-stacktrace"); var _ = require('underscore'); var Security = require('ep_etherpad-lite/static/js/security'); var hooks = require('ep_etherpad-lite/static/js/pluginfw/hooks'); var eejs = require('ep_etherpad-lite/node/eejs'); var _analyzeLine = require('./ExportHelper')._analyzeLine; var _encodeWhitespace = require('./ExportHelper')._encodeWhitespace; -// copied from ACE -var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; -var _REGEX_URLCHAR = new RegExp('(' + (/[-:@a-zA-Z0-9_.,~%+/\\?=&#;()$]/).source + '|' + _REGEX_WORDCHAR.source + ')'); -var _REGEX_URL = new RegExp(/(?:(?:https?|s?ftp|ftps|file|smb|afp|nfs|(x-)?man|gopher|txmt):\/\/|mailto:)/.source + _REGEX_URLCHAR.source + '*(?![:.,;])' + _REGEX_URLCHAR.source, 'g'); -// returns null if no URLs, or [[startIndex1, url1], [startIndex2, url2], ...] -// copied from ACE -function _processSpaces (s) { - var doesWrap = true; - if (s.indexOf('<') < 0 && !doesWrap) { - // short-cut - return s.replace(/ /g, ' '); +function getPadHTML(pad, revNum, callback) +{ + var atext = pad.atext; + var html; + async.waterfall([ + // fetch revision atext + function (callback) + { + if (revNum != undefined) + { + pad.getInternalRevisionAText(revNum, function (err, revisionAtext) + { + if(ERR(err, callback)) return; + atext = revisionAtext; + callback(); + }); } - var parts = []; - s.replace(/<[^>]*>?| |[^ <]+/g, function (m) { - parts.push(m); - }); - if (doesWrap) { - var endOfLine = true; - var beforeSpace = false; - // last space in a run is normal, others are nbsp, - // end of line is nbsp - for (var i = parts.length - 1; i >= 0; i--) { - var p = parts[i]; - if (p === ' ') { - if (endOfLine || beforeSpace) parts[i] = ' '; - endOfLine = false; - beforeSpace = true; - } else if (p.charAt(0) !== '<') { - endOfLine = false; - beforeSpace = false; - } - } - // beginning of line is nbsp - for (i = 0; i < parts.length; i++) { - p = parts[i]; - if (p === ' ') { - parts[i] = ' '; - break; - } else if (p.charAt(0) !== '<') { - break; - } - } - } else { - for (i = 0; i < parts.length; i++) { - p = parts[i]; - if (p === ' ') { - parts[i] = ' '; - } - } + else + { + callback(null); } + }, - return parts.join(''); + // convert atext to html + + + function (callback) + { + html = getHTMLFromAtext(pad, atext); + callback(null); + }], + // run final callback + + + function (err) + { + if(ERR(err, callback)) return; + callback(null, html); + }); } -function _findURLs (text) { - _REGEX_URL.lastIndex = 0; - var urls = null; - var execResult; - while (execResult = _REGEX_URL.exec(text)) { - urls = urls || []; - var startIndex = execResult.index; - var url = execResult[0]; - urls.push([startIndex, url]); - } +exports.getPadHTML = getPadHTML; +exports.getHTMLFromAtext = getHTMLFromAtext; - return urls; -} +function getHTMLFromAtext(pad, atext, authorColors) +{ + var apool = pad.apool(); + var textLines = atext.text.slice(0, -1).split('\n'); + var attribLines = Changeset.splitAttributionLines(atext.attribs, atext.text); -/** - * Return docx from input html - * - * @param {text} [pad] text - * @param {text} [atext] text - * @param {text} [authorColors] text - * - * @returns {html} html - * - * @public - */ -function getHTMLFromAtext (pad, atext, authorColors) { - var apool = pad.apool(); - var textLines = atext.text.slice(0, -1).split('\n'); - var attribLines = Changeset.splitAttributionLines(atext.attribs, atext.text); + var tags = ['h1', 'h2', 'strong', 'em', 'u', 's']; + var props = ['heading1', 'heading2', 'bold', 'italic', 'underline', 'strikethrough']; - var tags = ['h1', 'h2', 'strong', 'em', 'u', 's']; - var props = ['heading1', 'heading2', 'bold', 'italic', 'underline', 'strikethrough']; - - // prepare tags stored as ['tag', true] to be exported - hooks.aCallAll('exportHtmlAdditionalTags', pad, function (err, newProps) { - if (err) { - return err; - } - newProps.forEach(function (propName) { - tags.push(propName); - props.push(propName); - }); + // prepare tags stored as ['tag', true] to be exported + hooks.aCallAll("exportHtmlAdditionalTags", pad, function(err, newProps){ + newProps.forEach(function (propName, i){ + tags.push(propName); + props.push(propName); }); - // prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML - // with tags like - hooks.aCallAll('exportHtmlAdditionalTagsWithData', pad, function (err, newProps) { - if (err) { - return err; - } - newProps.forEach(function (propName) { - tags.push('span data-' + propName[0] + '="' + propName[1] + '"'); - props.push(propName); - }); + }); + // prepare tags stored as ['tag', 'value'] to be exported. This will generate HTML + // with tags like + hooks.aCallAll("exportHtmlAdditionalTagsWithData", pad, function(err, newProps){ + newProps.forEach(function (propName, i){ + tags.push('span data-' + propName[0] + '="' + propName[1] + '"'); + props.push(propName); }); + }); - // holds a map of used styling attributes (*1, *2, etc) in the apool - // and maps them to an index in props - // *3:2 -> the attribute *3 means strong - // *2:5 -> the attribute *2 means s(trikethrough) - var anumMap = {}; - var css = ''; + // holds a map of used styling attributes (*1, *2, etc) in the apool + // and maps them to an index in props + // *3:2 -> the attribute *3 means strong + // *2:5 -> the attribute *2 means s(trikethrough) + var anumMap = {}; + var css = ""; - var stripDotFromAuthorID = function (id) { - return id.replace(/\./g, '_'); - }; + var stripDotFromAuthorID = function(id){ + return id.replace(/\./g,'_'); + }; - if (authorColors) { - css += ''; + } } - // iterates over all props(h1,h2,strong,...), checks if it is used in - // this pad, and if yes puts its attrib id->props value into anumMap - props.forEach(function (propName, i) { - var attrib = [propName, true]; - if (_.isArray(propName)) { - // propName can be in the form of ['color', 'red'], - // see hook exportHtmlAdditionalTagsWithData - attrib = propName; - } - var propTrueNum = apool.putAttrib(attrib, true); - if (propTrueNum >= 0) { - anumMap[propTrueNum] = i; - } - }); + css+=""; + } - function getLineHTML (text, attribs) { + // iterates over all props(h1,h2,strong,...), checks if it is used in + // this pad, and if yes puts its attrib id->props value into anumMap + props.forEach(function (propName, i) + { + var attrib = [propName, true]; + if (_.isArray(propName)) { + // propName can be in the form of ['color', 'red'], + // see hook exportHtmlAdditionalTagsWithData + attrib = propName; + } + var propTrueNum = apool.putAttrib(attrib, true); + if (propTrueNum >= 0) + { + anumMap[propTrueNum] = i; + } + }); + + function getLineHTML(text, attribs) + { // Use order of tags (b/i/u) as order of nesting, for simplicity // and decent nesting. For example, // Just bold Bold and italics Just italics // becomes // Just bold Bold and italics Just italics - var taker = Changeset.stringIterator(text); - var assem = Changeset.stringAssembler(); - var openTags = []; + var taker = Changeset.stringIterator(text); + var assem = Changeset.stringAssembler(); + var openTags = []; - function getSpanClassFor (i) { - //return if author colors are disabled - if (!authorColors) return false; + function getSpanClassFor(i){ + //return if author colors are disabled + if (!authorColors) return false; - var property = props[i]; + var property = props[i]; - // we are not insterested on properties in the form of ['color', 'red'], - // see hook exportHtmlAdditionalTagsWithData - if (_.isArray(property)) { - return false; - } + // we are not insterested on properties in the form of ['color', 'red'], + // see hook exportHtmlAdditionalTagsWithData + if (_.isArray(property)) { + return false; + } - if (property.substr(0, 6) === 'author') { - return stripDotFromAuthorID(property); - } + if(property.substr(0,6) === "author"){ + return stripDotFromAuthorID(property); + } - if (property === 'removed') { - return 'removed'; - } + if(property === "removed"){ + return "removed"; + } - return false; - } - - // tags added by exportHtmlAdditionalTagsWithData will be exported as with - // data attributes - function isSpanWithData (i) { - var property = props[i]; - - return _.isArray(property); - } - - function emitOpenTag (i) { - openTags.unshift(i); - var spanClass = getSpanClassFor(i); - - if (spanClass) { - assem.append(''); - } else { - assem.append('<'); - assem.append(tags[i]); - assem.append('>'); - } - } - - // this closes an open tag and removes its reference from openTags - function emitCloseTag (i) { - openTags.shift(); - var spanClass = getSpanClassFor(i); - var spanWithData = isSpanWithData(i); - - if (spanClass || spanWithData) { - assem.append(''); - } else { - assem.append(''); - } - } - - var urls = _findURLs(text); - - var idx = 0; - - function processNextChars (numChars) { - if (numChars <= 0) { - return; - } - - var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars)); - idx += numChars; - - // this iterates over every op string and decides which tags to open or to close - // based on the attribs used - while (iter.hasNext()) { - var o = iter.next(); - var usedAttribs = []; - - // mark all attribs as used - Changeset.eachAttribNumber(o.attribs, function (a) { - if (a in anumMap) { - usedAttribs.push(anumMap[a]); // i = 0 => bold, etc. - } - }); - var outermostTag = -1; - // find the outer most open tag that is no longer used - for (var i = openTags.length - 1; i >= 0; i--) { - if (usedAttribs.indexOf(openTags[i]) === -1) { - outermostTag = i; - break; - } - } - - // close all tags upto the outer most - if (outermostTag !== -1) { - while (outermostTag >= 0) { - emitCloseTag(openTags[0]); - outermostTag--; - } - } - - // open all tags that are used but not open - for (i = 0; i < usedAttribs.length; i++) { - if (openTags.indexOf(usedAttribs[i]) === -1) { - emitOpenTag(usedAttribs[i]); - } - } - - var chars = o.chars; - if (o.lines) { - chars--; // exclude newline at end of line, if present - } - - var s = taker.take(chars); - - //removes the characters with the code 12. Don't know where they come - //from but they break the abiword parser and are completly useless - s = s.replace(String.fromCharCode(12), ''); - - assem.append(_encodeWhitespace(Security.escapeHTML(s))); - } // end iteration over spans in line - - // close all the tags that are open after the last op - while (openTags.length > 0) { - emitCloseTag(openTags[0]); - } - } // end processNextChars - - if (urls) { - urls.forEach(function (urlData) { - var startIndex = urlData[0]; - var url = urlData[1]; - var urlLength = url.length; - processNextChars(startIndex - idx); - assem.append(''); - processNextChars(urlLength); - assem.append(''); - }); - } - - processNextChars(text.length - idx); - - return _processSpaces(assem.toString()); - } // end getLineHTML - - var pieces = [css]; - - // Need to deal with constraints imposed on HTML lists; can - // only gain one level of nesting at once, can't change type - // mid-list, etc. - // People might use weird indenting, e.g. skip a level, - // so we want to do something reasonable there. We also - // want to deal gracefully with blank lines. - // => keeps track of the parents level of indentation - var openLists = []; - for (var i = 0; i < textLines.length; i++) { - var context; - var line = _analyzeLine(textLines[i], attribLines[i], apool); - var lineContent = getLineHTML(line.text, line.aline); - - if (line.listLevel) { //If we are inside a list - - context = { - line: line, - lineContent: lineContent, - apool: apool, - attribLine: attribLines[i], - text: textLines[i], - padId: pad.id - }; - var prevLine = null; - var nextLine = null; - if (i > 0) { - prevLine = _analyzeLine(textLines[i - 1], attribLines[i - 1], apool); - } - if (i < textLines.length) { - nextLine = _analyzeLine(textLines[i + 1], attribLines[i + 1], apool); - } - hooks.callAll('getLineHTMLForExport', context); - //To create list parent elements - if ((!prevLine || prevLine.listLevel !== line.listLevel) || (prevLine && line.listTypeName !== prevLine.listTypeName)) { - //pieces.push('
      '); - var exists = _.find(openLists, function (item) { - return (item.level === line.listLevel && item.type === line.listTypeName); - }); - if (!exists) { - var prevLevel = prevLine.listLevel || 0; - if (prevLine && line.listTypeName !== prevLine.listTypeName) { - prevLevel = 0; - } - - for (var diff = prevLevel; diff < line.listLevel; diff++) { - openLists.push({level: diff, type: line.listTypeName}); - var prevPiece = pieces[pieces.length - 1]; - - if (prevPiece.indexOf('') === 0) { - pieces.push('
    • '); - } - - if (line.listTypeName === 'number') { - pieces.push('
        '); - } else { - pieces.push('
          '); - } - } - } - - } - - pieces.push('
        • ', context.lineContent); - - // To close list elements - if (nextLine && nextLine.listLevel === line.listLevel && line.listTypeName === nextLine.listTypeName) { - pieces.push('
          • '); - } - if ((!nextLine || !nextLine.listLevel || nextLine.listLevel < line.listLevel) || (nextLine && line.listTypeName !== nextLine.listTypeName)) { - var nextLevel = nextLine.listLevel || 0; - if (nextLine && line.listTypeName !== nextLine.listTypeName) { - nextLevel = 0; - } - - for (var diff = nextLevel; diff < line.listLevel; diff++) { - openLists = openLists.filter(function(el) { - return el.level !== diff && el.type !== line.listTypeName; - }); - - if (pieces[pieces.length - 1].indexOf(''); - } - - if (line.listTypeName === 'number') { - pieces.push('
      '); - } else { - pieces.push('
    '); - } - } - } - } else { //outside any list, need to close line.listLevel of lists - context = { - line: line, - lineContent: lineContent, - apool: apool, - attribLine: attribLines[i], - text: textLines[i], - padId: pad.id - }; - - hooks.callAll('getLineHTMLForExport', context); - pieces.push(context.lineContent, '
    '); - } + return false; } - return pieces.join(''); -} + // tags added by exportHtmlAdditionalTagsWithData will be exported as with + // data attributes + function isSpanWithData(i){ + var property = props[i]; + return _.isArray(property); + } -function getPadHTML (pad, revNum, callback) { - var atext = pad.atext; - var html; - async.waterfall( - [ - // fetch revision atext - function (callback) { - if (revNum !== undefined) { - pad.getInternalRevisionAText(revNum, function (err, revisionAtext) { - if (eRR(err, callback)) return; - atext = revisionAtext; + function emitOpenTag(i) + { + openTags.unshift(i); + var spanClass = getSpanClassFor(i); - return callback(); - }); - } else { - return callback(null); - } - }, + if(spanClass){ + assem.append(''); + } else { + assem.append('<'); + assem.append(tags[i]); + assem.append('>'); + } + } - // convert atext to html - function (callback) { - html = getHTMLFromAtext(pad, atext); - callback(null); - } - ], - // run final callback + // this closes an open tag and removes its reference from openTags + function emitCloseTag(i) + { + openTags.shift(); + var spanClass = getSpanClassFor(i); + var spanWithData = isSpanWithData(i); + if(spanClass || spanWithData){ + assem.append(''); + } else { + assem.append(''); + } + } - function (err) { - if (eRR(err, callback)) return; - callback(null, html); - } - ); -} + var urls = _findURLs(text); -exports.getPadHTML = getPadHTML; -exports.getHTMLFromAtext = getHTMLFromAtext; -exports.getPadHTMLDocument = function (padId, revNum, callback) { - padManager.getPad(padId, function (err, pad) { - if (eRR(err, callback)) return; + var idx = 0; - var stylesForExportCSS = ''; - // Include some Styles into the Head for Export - hooks.aCallAll('stylesForExport', padId, function (err, stylesForExport) { - if (eRR(err, callback)) return; - stylesForExport.forEach(function (css) { - stylesForExportCSS += css; - }); + function processNextChars(numChars) + { + if (numChars <= 0) + { + return; + } - getPadHTML(pad, revNum, function (err, html) { - if (eRR(err, callback)) return; - var exportedDoc = eejs.require('ep_etherpad-lite/templates/export_html.html', { - body: html, - padId: Security.escapeHTML(padId), - extraCSS: stylesForExportCSS - }); - callback(null, exportedDoc); - }); + var iter = Changeset.opIterator(Changeset.subattribution(attribs, idx, idx + numChars)); + idx += numChars; + + // this iterates over every op string and decides which tags to open or to close + // based on the attribs used + while (iter.hasNext()) + { + var o = iter.next(); + var usedAttribs = []; + + // mark all attribs as used + Changeset.eachAttribNumber(o.attribs, function (a) + { + if (a in anumMap) + { + usedAttribs.push(anumMap[a]); // i = 0 => bold, etc. + } }); + var outermostTag = -1; + // find the outer most open tag that is no longer used + for (var i = openTags.length - 1; i >= 0; i--) + { + if (usedAttribs.indexOf(openTags[i]) === -1) + { + outermostTag = i; + break; + } + } + + // close all tags upto the outer most + if (outermostTag !== -1) + { + while ( outermostTag >= 0 ) + { + emitCloseTag(openTags[0]); + outermostTag--; + } + } + + // open all tags that are used but not open + for (i=0; i < usedAttribs.length; i++) + { + if (openTags.indexOf(usedAttribs[i]) === -1) + { + emitOpenTag(usedAttribs[i]); + } + } + + var chars = o.chars; + if (o.lines) + { + chars--; // exclude newline at end of line, if present + } + + var s = taker.take(chars); + + //removes the characters with the code 12. Don't know where they come + //from but they break the abiword parser and are completly useless + s = s.replace(String.fromCharCode(12), ""); + + assem.append(_encodeWhitespace(Security.escapeHTML(s))); + } // end iteration over spans in line + + // close all the tags that are open after the last op + while (openTags.length > 0) + { + emitCloseTag(openTags[0]); + } + } // end processNextChars + if (urls) + { + urls.forEach(function (urlData) + { + var startIndex = urlData[0]; + var url = urlData[1]; + var urlLength = url.length; + processNextChars(startIndex - idx); + assem.append(''); + processNextChars(urlLength); + assem.append(''); + }); + } + processNextChars(text.length - idx); + + return _processSpaces(assem.toString()); + } // end getLineHTML + var pieces = [css]; + + // Need to deal with constraints imposed on HTML lists; can + // only gain one level of nesting at once, can't change type + // mid-list, etc. + // People might use weird indenting, e.g. skip a level, + // so we want to do something reasonable there. We also + // want to deal gracefully with blank lines. + // => keeps track of the parents level of indentation + var openLists = []; + for (var i = 0; i < textLines.length; i++) + { + var context; + var line = _analyzeLine(textLines[i], attribLines[i], apool); + var lineContent = getLineHTML(line.text, line.aline); + + if (line.listLevel)//If we are inside a list + { + context = { + line: line, + lineContent: lineContent, + apool: apool, + attribLine: attribLines[i], + text: textLines[i], + padId: pad.id + }; + var prevLine = null; + var nextLine = null; + if (i > 0) + { + prevLine = _analyzeLine(textLines[i -1], attribLines[i -1], apool); + } + if (i < textLines.length) + { + nextLine = _analyzeLine(textLines[i + 1], attribLines[i + 1], apool); + } + hooks.callAll('getLineHTMLForExport', context); + //To create list parent elements + if ((!prevLine || prevLine.listLevel !== line.listLevel) || (prevLine && line.listTypeName !== prevLine.listTypeName)) + { + var exists = _.find(openLists, function (item) + { + return (item.level === line.listLevel && item.type === line.listTypeName); + }); + if (!exists) { + var prevLevel = prevLine.listLevel || 0; + if (prevLine && line.listTypeName !== prevLine.listTypeName) + { + prevLevel = 0; + } + + for (var diff = prevLevel; diff < line.listLevel; diff++) { + openLists.push({level: diff, type: line.listTypeName}); + var prevPiece = pieces[pieces.length - 1]; + + if (prevPiece.indexOf("") === 0) + { + pieces.push("
  • "); + } + + if (line.listTypeName === "number") + { + pieces.push("
      "); + } + else + { + pieces.push("
        "); + } + } + } + } + + pieces.push("
      • ", context.lineContent); + + // To close list elements + if (nextLine && nextLine.listLevel === line.listLevel && line.listTypeName === nextLine.listTypeName) + { + pieces.push("
        • "); + } + if ((!nextLine || !nextLine.listLevel || nextLine.listLevel < line.listLevel) || (nextLine && line.listTypeName !== nextLine.listTypeName)) + { + var nextLevel = nextLine.listLevel || 0; + if (nextLine && line.listTypeName !== nextLine.listTypeName) + { + nextLevel = 0; + } + + for (var diff = nextLevel; diff < line.listLevel; diff++) + { + openLists = openLists.filter(function(el) + { + return el.level !== diff && el.type !== line.listTypeName; + }); + + if (pieces[pieces.length - 1].indexOf(""); + } + + if (line.listTypeName === "number") + { + pieces.push("
    "); + } + else + { + pieces.push(""); + } + } + } + } + else//outside any list, need to close line.listLevel of lists + { + context = { + line: line, + lineContent: lineContent, + apool: apool, + attribLine: attribLines[i], + text: textLines[i], + padId: pad.id + }; + + hooks.callAll("getLineHTMLForExport", context); + pieces.push(context.lineContent, "
    "); + } + } + + return pieces.join(''); +} + +exports.getPadHTMLDocument = function (padId, revNum, callback) +{ + padManager.getPad(padId, function (err, pad) + { + if(ERR(err, callback)) return; + + var stylesForExportCSS = ""; + // Include some Styles into the Head for Export + hooks.aCallAll("stylesForExport", padId, function(err, stylesForExport){ + stylesForExport.forEach(function(css){ + stylesForExportCSS += css; + }); + + getPadHTML(pad, revNum, function (err, html) + { + if(ERR(err, callback)) return; + var exportedDoc = eejs.require("ep_etherpad-lite/templates/export_html.html", { + body: html, + padId: Security.escapeHTML(padId), + extraCSS: stylesForExportCSS + }); + callback(null, exportedDoc); + }); }); + }); }; + +// copied from ACE +var _REGEX_WORDCHAR = /[\u0030-\u0039\u0041-\u005A\u0061-\u007A\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF\u0100-\u1FFF\u3040-\u9FFF\uF900-\uFDFF\uFE70-\uFEFE\uFF10-\uFF19\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFDC]/; +var _REGEX_SPACE = /\s/; +var _REGEX_URLCHAR = new RegExp('(' + /[-:@a-zA-Z0-9_.,~%+\/\\?=&#;()$]/.source + '|' + _REGEX_WORDCHAR.source + ')'); +var _REGEX_URL = new RegExp(/(?:(?:https?|s?ftp|ftps|file|smb|afp|nfs|(x-)?man|gopher|txmt):\/\/|mailto:)/.source + _REGEX_URLCHAR.source + '*(?![:.,;])' + _REGEX_URLCHAR.source, 'g'); + +// returns null if no URLs, or [[startIndex1, url1], [startIndex2, url2], ...] + + +function _findURLs(text) +{ + _REGEX_URL.lastIndex = 0; + var urls = null; + var execResult; + while ((execResult = _REGEX_URL.exec(text))) + { + urls = (urls || []); + var startIndex = execResult.index; + var url = execResult[0]; + urls.push([startIndex, url]); + } + + return urls; +} + + +// copied from ACE +function _processSpaces(s){ + var doesWrap = true; + if (s.indexOf("<") < 0 && !doesWrap){ + // short-cut + return s.replace(/ /g, ' '); + } + var parts = []; + s.replace(/<[^>]*>?| |[^ <]+/g, function (m){ + parts.push(m); + }); + if (doesWrap){ + var endOfLine = true; + var beforeSpace = false; + // last space in a run is normal, others are nbsp, + // end of line is nbsp + for (var i = parts.length - 1; i >= 0; i--){ + var p = parts[i]; + if (p == " "){ + if (endOfLine || beforeSpace) parts[i] = ' '; + endOfLine = false; + beforeSpace = true; + } + else if (p.charAt(0) != "<"){ + endOfLine = false; + beforeSpace = false; + } + } + // beginning of line is nbsp + for (i = 0; i < parts.length; i++){ + p = parts[i]; + if (p == " "){ + parts[i] = ' '; + break; + } + else if (p.charAt(0) != "<"){ + break; + } + } + } + else + { + for (i = 0; i < parts.length; i++){ + p = parts[i]; + if (p == " "){ + parts[i] = ' '; + } + } + } + return parts.join(''); +} \ No newline at end of file From 7cc7bb1abc4ec47e464bb4fbc01c8c33acd3eb1a Mon Sep 17 00:00:00 2001 From: ilmar Date: Tue, 24 Apr 2018 12:25:56 +0300 Subject: [PATCH 10/11] upgrade to 1.6.5 --- CHANGELOG.md | 33 ++++++-- CONTRIBUTING.md | 20 ++++- README.md | 110 +++++++++++++------------ bin/cleanRun.sh | 2 +- bin/dirty-db-cleaner.py | 2 +- bin/installOnWindows.bat | 12 ++- doc/api/hooks_server-side.md | 12 +++ src/locales/cs.json | 5 +- src/locales/diq.json | 4 +- src/locales/fi.json | 2 +- src/locales/fr.json | 5 +- src/locales/ku-latn.json | 4 +- src/locales/ru.json | 5 +- src/locales/te.json | 9 +- src/locales/zh-hant.json | 2 +- src/node/db/AuthorManager.js | 52 ++++++------ src/node/db/Pad.js | 7 +- src/node/db/SecurityManager.js | 9 ++ src/node/handler/APIHandler.js | 6 +- src/node/handler/ImportHandler.js | 2 +- src/node/hooks/express.js | 4 + src/node/hooks/express/admin.js | 2 +- src/node/hooks/express/apicalls.js | 10 +-- src/node/hooks/express/importexport.js | 20 ++++- src/node/hooks/express/webaccess.js | 20 +++-- src/node/utils/Cli.js | 10 +++ src/node/utils/ExportEtherpad.js | 27 +++--- src/node/utils/ExportHtml.js | 38 ++++----- src/node/utils/LibreOffice.js | 16 +++- src/node/utils/Settings.js | 5 +- src/package.json | 17 ++-- src/static/css/iframe_editor.css | 8 +- src/static/css/pad.css | 9 +- src/static/js/ace2_inner.js | 4 +- src/static/js/admin/plugins.js | 11 ++- src/static/js/pad_cookie.js | 2 +- src/static/js/pluginfw/plugins.js | 2 +- src/templates/export_html.html | 1 - 38 files changed, 318 insertions(+), 191 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74d06f45..df249c25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +# 1.6.5 + * SECURITY: Escape data when listing available plugins + * FIX: Fix typo in apicalls.js which prevented importing isValidJSONPName + * FIX: fixed plugin dependency issue + * FIX: Update iframe_editor.css + * FIX: unbreak Safari iOS line wrapping + +# 1.6.4 + * SECURITY: exploitable /admin access - CVE-2018-9845 + * SECURITY: DoS with pad exports - CVE-2018-9327 + * SECURITY: Remote Code Execution - CVE-2018-9326 + * SECURITY: Pad data leak - CVE-2018-9325 + * Fix: Admin redirect URL + * Fix: Various script Fixes + * Fix: Various CSS/Style/Layout fixes + * NEW: Improved Pad contents readability + * NEW: Hook: onAccessCheck + * NEW: SESSIONKEY and APIKey customizable path + * NEW: checkPads script + * NEW: Support "cluster mode" + # 1.6.3 * SECURITY: Update ejs * SECURITY: xss vulnerability when reading window.location.href @@ -56,7 +77,7 @@ * NEW: Allow LibreOffice to be used when exporting a pad * NEW: Create hook exportHtmlAdditionalTagsWithData * NEW: Improve DB migration performance - * NEW: allow settings to be applied from the filesystem + * NEW: allow settings to be applied from the filesystem * NEW: remove applySettings hook and allow credentials.json to be part of core * NEW: Use exec to switch to node process * NEW: Validate incoming color codes @@ -85,7 +106,7 @@ * Fix: switchToPad method * Fix: Dead keys * Fix: Preserve new lines in copy-pasted text - * Fix: Compatibility mode on IE + * Fix: Compatibility mode on IE * Fix: Content Collector to get the class of the DOM-node * Fix: Timeslider export links * Fix: Double prompt on file upload @@ -212,7 +233,7 @@ * Fix: Session Deletion error * Fix: Allow browser tabs to be cycled when focus is in editor * Fix: Various Editor issues with Easysync potentially entering forever loop on bad changeset - + # 1.4 * NEW: Disable toolbar items through settings.json * NEW: Internal stats/metrics engine @@ -244,7 +265,7 @@ # 1.3 * NEW: We now follow the semantic versioning scheme! * NEW: Option to disable IP logging - * NEW: Localisation updates from http://translatewiki.net. + * NEW: Localisation updates from http://translatewiki.net. * Fix: Fix readOnly group pads * Fix: don't fetch padList on every request @@ -337,7 +358,7 @@ * NEW: Add authorId to chat and userlist as a data attribute * NEW: Refactor and fix our frontend tests * NEW: Localisation updates - + # 1.2.81 * Fix: CtrlZ-Y for Undo Redo @@ -377,7 +398,7 @@ * Other: Change loading message asking user to please wait on first build * Other: Allow etherpad to use global npm installation (Safe since node 6.3) * Other: Better documentation for log rotation and log message handling - + # 1.2.7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09ddc286..66946080 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Developer Guidelines +# Contributor Guidelines (Please talk to people on the mailing list before you change this page, see our section on [how to get in touch](https://github.com/ether/etherpad-lite#get-in-touch)) ## How to write a bug report @@ -35,7 +35,7 @@ The logfile location is defined in startup script or the log is directly shown i To make sure everybody is going in the same direction: * easy to install for admins and easy to use for people * easy to integrate into other apps, but also usable as standalone -* using less resources on server side +* lightweight and scalable * extensible, as much functionality should be extendable with plugins so changes don't have to be done in core. Also, keep it maintainable. We don't wanna end up as the monster Etherpad was! @@ -92,3 +92,19 @@ You can build the docs e.g. produce html, using `make docs`. At some point in th ## Testing Front-end tests are found in the `tests/frontend/` folder in the repository. Run them by pointing your browser to `/tests/frontend`. + +## Things you can help with +Etherpad is much more than software. So if you aren't a developer then worry not, there is still a LOT you can do! A big part of what we do is community engagement. You can help in the following ways + * Triage bugs (applying labels) and confirming their existance + * Testing fixes (simply applying them and seeing if it fixes your issue or not) - Some git experience required + * Notifying large site admins of new releases + * Writing Changelogs for releases + * Creating Windows packages + * Creating releases + * Bumping dependencies periodically and checking they don't break anything + * Write proposals for grants + * Co-Author and Publish CVEs + * Work with SFC to maintain legal side of project + * Maintain TODO page - https://github.com/ether/etherpad-lite/wiki/TODO#IMPORTANT_TODOS + * Replying to messages on IRC / The Mailing list / Emails + diff --git a/README.md b/README.md index e54ae18e..d8d7b621 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,43 @@ +### This project is looking for a new project lead. If you wish to help steer Etherpad forward please email contact@etherpad.org + +[![Deps](https://david-dm.org/ether/etherpad-lite.svg?branch=develop)](https://david-dm.org/ether/etherpad-lite) +[![NSP Status](https://nodesecurity.io/orgs/etherpad/projects/635f6185-35c6-4ed7-931a-0bc62758ece7/badge)](https://nodesecurity.io/orgs/etherpad/projects/635f6185-35c6-4ed7-931a-0bc62758ece7) + # A really-real time collaborative word processor for the web -![alt text](https://i.imgur.com/zYrGkg3.gif "Etherpad in action on PrimaryPad") +![Demo Etherpad Animated Jif](https://i.imgur.com/zYrGkg3.gif "Etherpad in action on PrimaryPad") # About -Etherpad is a really-real time collaborative editor maintained by the Etherpad Community. +Etherpad is a really-real time collaborative editor scalable to thousands of simultanious real time users. Unlike all other collaborative tools Etherpad provides full fidelity data export and portability making it fully GDPR compliant. -Etherpad is written in JavaScript (99.9%) on both the server and client so it's easy for developers to maintain and add new features. Because of this Etherpad has tons of customizations that you can leverage. - -Etherpad is designed to be easily embeddable and provides a [HTTP API](https://github.com/ether/etherpad-lite/wiki/HTTP-API) -that allows your web application to manage pads, users and groups. It is recommended to use the [available client implementations](https://github.com/ether/etherpad-lite/wiki/HTTP-API-client-libraries) in order to interact with this API. - -There is also a [jQuery plugin](https://github.com/ether/etherpad-lite-jquery-plugin) that helps you to embed Pads into your website. - -There's also a full-featured plugin framework, allowing you to easily add your own features. By default your Etherpad is rather sparse and because Etherpad takes a lot of its inspiration from WordPress, plugins are really easy to install and update. Once you have Etherpad installed you should visit the plugin page and take control. - -Finally, Etherpad comes with translations into most languages! Users are automatically delivered the correct language for their local settings. - - -**Visit [beta.etherpad.org](http://beta.etherpad.org) to test it live.** - -Also, check out the **[FAQ](https://github.com/ether/etherpad-lite/wiki/FAQ)**, really! +**[Try it out](http://beta.etherpad.org)** # Installation -Etherpad works with node v0.10+ (except 6.0 and 6.1). +## Uber-Quick Ubuntu +``` +curl -sL https://deb.nodesource.com/setup_9.x | sudo -E bash - +sudo apt-get install -y nodejs +git clone https://github.com/ether/etherpad-lite.git && cd etherpad-lite && bin/run.sh +``` + +## GNU/Linux and other UNIX-like systems +You'll need gzip, git, curl, libssl develop libraries, python and gcc. +- *For Debian/Ubuntu*: `apt install gzip git curl python libssl-dev pkg-config build-essential` +- *For Fedora/CentOS*: `yum install gzip git curl python openssl-devel && yum groupinstall "Development Tools"` +- *For FreeBSD*: `portinstall node, npm, curl, git (optional)` + +Additionally, you'll need [node.js](https://nodejs.org) installed, Ideally the latest stable version, we recommend installing/compiling nodejs from source (avoiding apt). + +**As any user (we recommend creating a separate user called etherpad):** + +1. Move to a folder where you want to install Etherpad. Clone the git repository `git clone git://github.com/ether/etherpad-lite.git` +2. Change into the new directory containing the cloned source code `cd etherpad-lite` + +Now, run `bin/run.sh` and open in your browser. + +Update to the latest version with `git pull origin`. The next start with bin/run.sh will update the dependencies. + +[Next steps](#next-steps). ## Windows @@ -52,27 +67,6 @@ If cloning to a subdirectory within another project, you may need to do the foll 2. Edit the db `filename` in `settings.json` to the relative directory with the file (e.g. `application/lib/etherpad-lite/var/dirty.db`) 3. Add auto-generated files to the main project `.gitignore` -[Next steps](#next-steps). - -## GNU/Linux and other UNIX-like systems -You'll need gzip, git, curl, libssl develop libraries, python and gcc. -- *For Debian/Ubuntu*: `apt-get install gzip git curl python libssl-dev pkg-config build-essential` -- *For Fedora/CentOS*: `yum install gzip git curl python openssl-devel && yum groupinstall "Development Tools"` -- *For FreeBSD*: `portinstall node, npm, curl, git (optional)` - -Additionally, you'll need [node.js](https://nodejs.org) installed, Ideally the latest stable version, we recommend installing/compiling nodejs from source (avoiding apt). - -**As any user (we recommend creating a separate user called etherpad):** - -1. Move to a folder where you want to install Etherpad. Clone the git repository `git clone git://github.com/ether/etherpad-lite.git` -2. Change into the new directory containing the cloned source code `cd etherpad-lite` - -Now, run `bin/run.sh` and open in your browser. - -Update to the latest version with `git pull origin`. The next start with bin/run.sh will update the dependencies. - -You like it? [Next steps](#next-steps). - # Next Steps ## Tweak the settings @@ -85,9 +79,9 @@ You should use a dedicated database such as "mysql", if you are planning on usin Etherpad is very customizable through plugins. Instructions for installing themes and plugins can be found in [the plugin wiki article](https://github.com/ether/etherpad-lite/wiki/Available-Plugins). ## Helpful resources -The [wiki](https://github.com/ether/etherpad-lite/wiki) is your one-stop resource for Tutorials and How-to's, really check it out! Also, feel free to improve these wiki pages. +The [wiki](https://github.com/ether/etherpad-lite/wiki) is your one-stop resource for Tutorials and How-to's. -Documentation can be found in `docs/`. +Documentation can be found in `doc/`. # Development @@ -100,26 +94,38 @@ You can debug Etherpad using `bin/debugRun.sh`. If you want to find out how Etherpad's `Easysync` works (the library that makes it really realtime), start with this [PDF](https://github.com/ether/etherpad-lite/raw/master/doc/easysync/easysync-full-description.pdf) (complex, but worth reading). -## Getting started -You know all this and just want to know how you can help? - -Look at the [TODO list](https://github.com/ether/etherpad-lite/wiki/TODO) and our [Issue tracker](https://github.com/ether/etherpad-lite/issues). (Please consider using [jshint](http://www.jshint.com/about/), if you plan to contribute code.) - -Also, and most importantly, read our [**Developer Guidelines**](https://github.com/ether/etherpad-lite/blob/master/CONTRIBUTING.md), really! +## Contributing +Read our [**Developer Guidelines**](https://github.com/ether/etherpad-lite/blob/master/CONTRIBUTING.md) # Get in touch -Join the [mailinglist](https://groups.google.com/group/etherpad-lite-dev) and make some noise on our busy freenode irc channel [#etherpad-lite-dev](https://webchat.freenode.net?channels=#etherpad-lite-dev)! +[mailinglist](https://groups.google.com/group/etherpad-lite-dev) +[#etherpad-lite-dev freenode IRC](https://webchat.freenode.net?channels=#etherpad-lite-dev)! -# Modules created for this project +# Languages +Etherpad is written in JavaScript on both the server and client so it's easy for developers to maintain and add new features. -* [ueberDB](https://github.com/Pita/ueberDB) "transforms every database into a object key value store" - manages all database access -* [channels](https://github.com/Pita/channels) "Event channels in node.js" - ensures that ueberDB operations are atomic and in series for each key -* [async-stacktrace](https://github.com/Pita/async-stacktrace) "Improves node.js stacktraces and makes it easier to handle errors" +# HTTP API +Etherpad is designed to be easily embeddable and provides a [HTTP API](https://github.com/ether/etherpad-lite/wiki/HTTP-API) +that allows your web application to manage pads, users and groups. It is recommended to use the [available client implementations](https://github.com/ether/etherpad-lite/wiki/HTTP-API-client-libraries) in order to interact with this API. + +# jQuery plugin +There is a [jQuery plugin](https://github.com/ether/etherpad-lite-jquery-plugin) that helps you to embed Pads into your website. + +# Plugin Framework +Etherpad offers a plugin framework, allowing you to easily add your own features. By default your Etherpad is extremely light-weight and it's up to you to customize your experience. Once you have Etherpad installed you should visit the plugin page and take control. + +# Translations / Localizations (i18n / l10n) +Etherpad comes with translations into all languages thanks to the team at TranslateWiki. + +# FAQ +Visit the **[FAQ](https://github.com/ether/etherpad-lite/wiki/FAQ)**. # Donate! * [Flattr](https://flattr.com/thing/71378/Etherpad-Foundation) * Paypal - Press the donate button on [etherpad.org](http://etherpad.org) * [Bitcoin](https://coinbase.com/checkouts/1e572bf8a82e4663499f7f1f66c2d15a) +All donations go to the Etherpad foundation which is part of Software Freedom Conservency + # License [Apache License v2](http://www.apache.org/licenses/LICENSE-2.0.html) diff --git a/bin/cleanRun.sh b/bin/cleanRun.sh index ef802815..57325dd2 100755 --- a/bin/cleanRun.sh +++ b/bin/cleanRun.sh @@ -38,4 +38,4 @@ bin/installDeps.sh $* || exit 1 echo "Started Etherpad..." SCRIPTPATH=`pwd -P` -node $SCRIPTPATH/node_modules/ep_etherpad-lite/node/server.js $* +node "${$SCRIPTPATH}/node_modules/ep_etherpad-lite/node/server.js" $* diff --git a/bin/dirty-db-cleaner.py b/bin/dirty-db-cleaner.py index 8ed9c506..d3e49a0d 100755 --- a/bin/dirty-db-cleaner.py +++ b/bin/dirty-db-cleaner.py @@ -1,4 +1,4 @@ -#!/usr/bin/python -u +#!/usr/bin/env PYTHONUNBUFFERED=1 python2 # # Created by Bjarni R. Einarsson, placed in the public domain. Go wild! # diff --git a/bin/installOnWindows.bat b/bin/installOnWindows.bat index db679ef0..5ba05736 100644 --- a/bin/installOnWindows.bat +++ b/bin/installOnWindows.bat @@ -8,7 +8,15 @@ cmd /C node -e "" || ( echo "Please install node.js ( https://nodejs.org )" && e echo _ echo Ensure that all dependencies are up to date... If this is the first time you have run Etherpad please be patient. -cmd /C npm install src/ --loglevel warn || exit /B 1 + +mkdir node_modules +cd /D node_modules +mklink /D "ep_etherpad-lite" "..\src" + +cd /D "ep_etherpad-lite" +cmd /C npm install --loglevel warn || exit /B 1 + +cd /D "%~dp0\.." echo _ echo Copying custom templates... @@ -31,4 +39,4 @@ IF NOT EXIST settings.json ( ) echo _ -echo Installed Etherpad! To run Etherpad type start.bat +echo Installed Etherpad! To run Etherpad type start.bat \ No newline at end of file diff --git a/doc/api/hooks_server-side.md b/doc/api/hooks_server-side.md index 5dc8f094..edf2051a 100644 --- a/doc/api/hooks_server-side.md +++ b/doc/api/hooks_server-side.md @@ -108,6 +108,18 @@ Usage examples: * https://github.com/tiblu/ep_authorship_toggle +## onAccessCheck +Called from: src/node/db/SecurityManager.js + +Things in context: + +1. padID - the pad the user wants to access +2. password - the password the user has given to access the pad +3. token - the token of the author +4. sessionCookie - the session the use has + +This hook gets called when the access to the concrete pad is being checked. Return `false` to deny access. + ## padCreate Called from: src/node/db/Pad.js diff --git a/src/locales/cs.json b/src/locales/cs.json index cda8ccb0..10fc55e1 100644 --- a/src/locales/cs.json +++ b/src/locales/cs.json @@ -8,7 +8,8 @@ "Quinn", "Aktron", "Mormegil", - "Dvorapa" + "Dvorapa", + "Clon" ] }, "index.newPad": "Založ nový Pad", @@ -63,6 +64,8 @@ "pad.modals.connected": "Připojeno.", "pad.modals.reconnecting": "Znovupřipojování k Padu…", "pad.modals.forcereconnect": "Vynutit znovupřipojení", + "pad.modals.reconnecttimer": "Zkouším to znovu připojit", + "pad.modals.cancel": "Zrušit", "pad.modals.userdup": "Otevřeno v jiném okně", "pad.modals.userdup.explanation": "Zdá se, že tento Pad je na tomto počítači otevřen ve více než jednom okně.", "pad.modals.userdup.advice": "Pro použití tohoto okna je třeba se znovu připojit.", diff --git a/src/locales/diq.json b/src/locales/diq.json index 86f44314..a823232f 100644 --- a/src/locales/diq.json +++ b/src/locales/diq.json @@ -47,7 +47,7 @@ "pad.settings.fontType.monospaced": "Yewca", "pad.settings.globalView": "Asayışo Global", "pad.settings.language": "Zıwan:", - "pad.importExport.import_export": "Zeredayış/Teberdayış", + "pad.importExport.import_export": "Zerredayış/Teberdayış", "pad.importExport.import": "Dosya ya zi dokumanê meqaleyê de tesadufi bar ke", "pad.importExport.importSuccessful": "Mıwafaq biye", "pad.importExport.export": "Mewcud bloknoti ateberd:", @@ -95,7 +95,7 @@ "timeslider.toolbar.returnbutton": "Peyser şo ped", "timeslider.toolbar.authors": "Nuştoği:", "timeslider.toolbar.authorsList": "Nuştoği çıniyê", - "timeslider.toolbar.exportlink.title": "Teber de", + "timeslider.toolbar.exportlink.title": "Teberdayış", "timeslider.exportCurrent": "Versiyonê enewki teber de:", "timeslider.version": "Versiyonê {{version}}", "timeslider.saved": "{{day}} {{month}}, {{year}} de biyo qeyd", diff --git a/src/locales/fi.json b/src/locales/fi.json index e42847ee..0a8e43fb 100644 --- a/src/locales/fi.json +++ b/src/locales/fi.json @@ -65,7 +65,7 @@ "pad.importExport.exportword": "Microsoft Word", "pad.importExport.exportpdf": "PDF", "pad.importExport.exportopen": "ODF (Open Document Format)", - "pad.importExport.abiword.innerHTML": "Tuonti on tuettu vain HTML- ja raakatekstitiedostoista. Lisätietoja tuonnin lisäasetuksista on sivulla install abiword.", + "pad.importExport.abiword.innerHTML": "Tuonti on tuettu vain HTML- ja raakatekstitiedostoista. Monipuoliset tuontiominaisuudet ovat käytettävissä asentamalla AbiWord.", "pad.modals.connected": "Yhdistetty.", "pad.modals.reconnecting": "Muodostetaan yhteyttä muistioon uudelleen...", "pad.modals.forcereconnect": "Pakota yhdistämään uudelleen", diff --git a/src/locales/fr.json b/src/locales/fr.json index 5f6b664a..29583d09 100644 --- a/src/locales/fr.json +++ b/src/locales/fr.json @@ -23,7 +23,8 @@ "Fylip22", "C13m3n7", "Wladek92", - "Urhixidur" + "Urhixidur", + "Envlh" ] }, "index.newPad": "Nouveau pad", @@ -48,7 +49,7 @@ "pad.colorpicker.save": "Enregistrer", "pad.colorpicker.cancel": "Annuler", "pad.loading": "Chargement…", - "pad.noCookie": "Le témoin (cookie) n’a pas pu être trouvé. Veuillez autoriser les témoins dans votre navigateur !", + "pad.noCookie": "Le cookie n’a pas pu être trouvé. Veuillez autoriser les cookies dans votre navigateur !", "pad.passwordRequired": "Vous avez besoin d'un mot de passe pour accéder à ce pad", "pad.permissionDenied": "Vous n'avez pas la permission d’accéder à ce pad", "pad.wrongPassword": "Votre mot de passe est incorrect", diff --git a/src/locales/ku-latn.json b/src/locales/ku-latn.json index 725a84d7..b5edc68b 100644 --- a/src/locales/ku-latn.json +++ b/src/locales/ku-latn.json @@ -5,7 +5,8 @@ "Dilyaramude", "George Animal", "Gomada", - "Mehk63" + "Mehk63", + "Ghybu" ] }, "index.newPad": "Bloknota nû", @@ -40,6 +41,7 @@ "pad.importExport.exportpdf": "PDF", "pad.modals.connected": "Hate girêdan.", "pad.modals.reconnecting": "Ji bloknota te re dîsa tê girêdan...", + "pad.modals.cancel": "Betal bike", "pad.modals.userdup": "Di pencereyek din de vebû", "pad.modals.userdup.advice": "Ji bo di vê pencereye de bikarbînîy dîsa giredanek çeke.", "pad.modals.unauth": "Desthilatdar nîne", diff --git a/src/locales/ru.json b/src/locales/ru.json index b9fbc3c8..801ceaa3 100644 --- a/src/locales/ru.json +++ b/src/locales/ru.json @@ -7,7 +7,8 @@ "Okras", "Volkov", "Nzeemin", - "Facenapalm" + "Facenapalm", + "Patrick Star" ] }, "index.newPad": "Создать", @@ -58,7 +59,7 @@ "pad.importExport.exportword": "Microsoft Word", "pad.importExport.exportpdf": "PDF", "pad.importExport.exportopen": "ODF (документ OpenOffice)", - "pad.importExport.abiword.innerHTML": "Вы можете импортировать только из обычного текста или HTML. Для более продвинутых функций импорта, пожалуйста, установите AbiWord.", + "pad.importExport.abiword.innerHTML": "Вы можете импортировать только из обычного текста или HTML. Для более продвинутых функций импорта, пожалуйста, установите AbiWord.", "pad.modals.connected": "Подключен.", "pad.modals.reconnecting": "Повторное подключение к вашему документу", "pad.modals.forcereconnect": "Принудительное переподключение", diff --git a/src/locales/te.json b/src/locales/te.json index 846ced8e..13af3970 100644 --- a/src/locales/te.json +++ b/src/locales/te.json @@ -11,12 +11,12 @@ }, "index.newPad": "కొత్త పలక", "index.createOpenPad": "ఒక పేరుతో పలకని సృష్టించండి లేదా అదే పేరుతో ఉన్న పలకని తెరవండి", - "pad.toolbar.bold.title": "మందం", - "pad.toolbar.italic.title": "వాలు అక్షరాలు", + "pad.toolbar.bold.title": "బొద్దు (Ctrl+B)", + "pad.toolbar.italic.title": "వాలు (Ctrl+I)", "pad.toolbar.underline.title": "క్రిందగీత", "pad.toolbar.strikethrough.title": "కొట్టివేత (Ctrl+5)", - "pad.toolbar.ol.title": "నిర్ధేశింపబడిన జాబితా", - "pad.toolbar.ul.title": "అనిర్దేశిత జాబితా, ( క్రమపద్ధతి లేని జాబితా )", + "pad.toolbar.ol.title": "క్రమ జాబితా (Ctrl+Shift+N)", + "pad.toolbar.ul.title": "బిందు జాబితా (Ctrl+Shift+L)", "pad.toolbar.undo.title": "చేయవద్దు", "pad.toolbar.redo.title": "తిరిగిచెయ్యి", "pad.toolbar.clearAuthorship.title": "మూలకర్తపు వర్ణాలను తీసివేయండి", @@ -53,6 +53,7 @@ "pad.modals.connected": "సంబంధం కుదిరింది.", "pad.modals.reconnecting": "మీ పలకకు మరల సంబంధం కలుపుతుంది...", "pad.modals.forcereconnect": "బలవంతంగానైనా సంబంధం కుదిరించు", + "pad.modals.cancel": "రద్దుచేయి", "pad.modals.userdup.explanation": "ఈ పలక, ఈ కంప్యూటర్లో ఒకటికన్న ఎక్కువ గవాక్షములలో తెరుచుకున్నట్లు అనిపిస్తుంది.", "pad.modals.userdup.advice": "బదులుగా ఈ గవాక్షమును వాడడానికి మరల సంబంధం కలపండి", "pad.modals.unauth": "అధికారం లేదు", diff --git a/src/locales/zh-hant.json b/src/locales/zh-hant.json index 17591884..2a7d6d1e 100644 --- a/src/locales/zh-hant.json +++ b/src/locales/zh-hant.json @@ -117,7 +117,7 @@ "timeslider.month.october": "10月", "timeslider.month.november": "11月", "timeslider.month.december": "12月", - "timeslider.unnamedauthors": "{{num}}匿名{[plural(num) 作者]}", + "timeslider.unnamedauthors": "{{num}} 個匿名{[plural(num) one:作者, other:作者]}", "pad.savedrevs.marked": "標記此修訂版本為已儲存修訂版本。", "pad.savedrevs.timeslider": "您可使用時段滑標來查看先前保存的版本內容", "pad.userlist.entername": "輸入您的姓名", diff --git a/src/node/db/AuthorManager.js b/src/node/db/AuthorManager.js index 3e3b691a..1f2a736b 100644 --- a/src/node/db/AuthorManager.js +++ b/src/node/db/AuthorManager.js @@ -25,7 +25,7 @@ var customError = require("../utils/customError"); var randomString = require('ep_etherpad-lite/static/js/pad_utils').randomString; exports.getColorPalette = function(){ - return ["#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1", "#ff8f8f", "#ffe38f", "#c7ff8f", "#8fffab", "#8fffff", "#8fabff", "#c78fff", "#ff8fe3", "#d97979", "#d9c179", "#a9d979", "#79d991", "#79d9d9", "#7991d9", "#a979d9", "#d979c1", "#d9a9a9", "#d9cda9", "#c1d9a9", "#a9d9b5", "#a9d9d9", "#a9b5d9", "#c1a9d9", "#d9a9cd", "#4c9c82", "#12d1ad", "#2d8e80", "#7485c3", "#a091c7", "#3185ab", "#6818b4", "#e6e76d", "#a42c64", "#f386e5", "#4ecc0c", "#c0c236", "#693224", "#b5de6a", "#9b88fd", "#358f9b", "#496d2f", "#e267fe", "#d23056", "#1a1a64", "#5aa335", "#d722bb", "#86dc6c", "#b5a714", "#955b6a", "#9f2985", "#4b81c8", "#3d6a5b", "#434e16", "#d16084", "#af6a0e", "#8c8bd8"]; + return ["#ffc7c7", "#fff1c7", "#e3ffc7", "#c7ffd5", "#c7ffff", "#c7d5ff", "#e3c7ff", "#ffc7f1", "#ffa8a8", "#ffe699", "#cfff9e", "#99ffb3", "#a3ffff", "#99b3ff", "#cc99ff", "#ff99e5", "#e7b1b1", "#e9dcAf", "#cde9af", "#bfedcc", "#b1e7e7", "#c3cdee", "#d2b8ea", "#eec3e6", "#e9cece", "#e7e0ca", "#d3e5c7", "#bce1c5", "#c1e2e2", "#c1c9e2", "#cfc1e2", "#e0bdd9", "#baded3", "#a0f8eb", "#b1e7e0", "#c3c8e4", "#cec5e2", "#b1d5e7", "#cda8f0", "#f0f0a8", "#f2f2a6", "#f5a8eb", "#c5f9a9", "#ececbb", "#e7c4bc", "#daf0b2", "#b0a0fd", "#bce2e7", "#cce2bb", "#ec9afe", "#edabbd", "#aeaeea", "#c4e7b1", "#d722bb", "#f3a5e7", "#ffa8a8", "#d8c0c5", "#eaaedd", "#adc6eb", "#bedad1", "#dee9af", "#e9afc2", "#f8d2a0", "#b3b3e6"]; }; /** @@ -42,9 +42,9 @@ exports.doesAuthorExists = function (authorID, callback) } /** - * Returns the AuthorID for a token. - * @param {String} token The token - * @param {Function} callback callback (err, author) + * Returns the AuthorID for a token. + * @param {String} token The token + * @param {Function} callback callback (err, author) */ exports.getAuthor4Token = function (token, callback) { @@ -57,21 +57,21 @@ exports.getAuthor4Token = function (token, callback) } /** - * Returns the AuthorID for a mapper. + * Returns the AuthorID for a mapper. * @param {String} token The mapper * @param {String} name The name of the author (optional) - * @param {Function} callback callback (err, author) + * @param {Function} callback callback (err, author) */ exports.createAuthorIfNotExistsFor = function (authorMapper, name, callback) { mapAuthorWithDBKey("mapper2author", authorMapper, function(err, author) { if(ERR(err, callback)) return; - + //set the name of this author if(name) exports.setAuthorName(author.authorID, name); - + //return the authorID callback(null, author); }); @@ -80,27 +80,27 @@ exports.createAuthorIfNotExistsFor = function (authorMapper, name, callback) /** * Returns the AuthorID for a mapper. We can map using a mapperkey, * so far this is token2author and mapper2author - * @param {String} mapperkey The database key name for this mapper + * @param {String} mapperkey The database key name for this mapper * @param {String} mapper The mapper - * @param {Function} callback callback (err, author) + * @param {Function} callback callback (err, author) */ function mapAuthorWithDBKey (mapperkey, mapper, callback) -{ +{ //try to map to an author db.get(mapperkey + ":" + mapper, function (err, author) { if(ERR(err, callback)) return; - + //there is no author with this mapper, so create one if(author == null) { exports.createAuthor(null, function(err, author) { if(ERR(err, callback)) return; - + //create the token2author relation db.set(mapperkey + ":" + mapper, author.authorID); - + //return the author callback(null, author); }); @@ -110,7 +110,7 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) { //update the timestamp of this author db.setSub("globalAuthor:" + author, ["timestamp"], new Date().getTime()); - + //return the author callback(null, {authorID: author}); } @@ -118,20 +118,20 @@ function mapAuthorWithDBKey (mapperkey, mapper, callback) } /** - * Internal function that creates the database entry for an author - * @param {String} name The name of the author + * Internal function that creates the database entry for an author + * @param {String} name The name of the author */ exports.createAuthor = function(name, callback) { //create the new author name var author = "a." + randomString(16); - + //create the globalAuthors db entry var authorObj = {"colorId" : Math.floor(Math.random()*(exports.getColorPalette().length)), "name": name, "timestamp": new Date().getTime()}; - + //set the global author db entry db.set("globalAuthor:" + author, authorObj); - + callback(null, {authorID: author}); } @@ -212,7 +212,7 @@ exports.listPadsOfAuthor = function (authorID, callback) } //everything is fine, return the pad IDs else - { + { var pads = []; if(author.padIDs != null) { @@ -238,16 +238,16 @@ exports.addPad = function (authorID, padID) { if(ERR(err)) return; if(author == null) return; - + //the entry doesn't exist so far, let's create it if(author.padIDs == null) { author.padIDs = {}; } - + //add the entry for this pad author.padIDs[padID] = 1;// anything, because value is not used - + //save the new element back db.set("globalAuthor:" + authorID, author); }); @@ -264,11 +264,11 @@ exports.removePad = function (authorID, padID) { if(ERR(err)) return; if(author == null) return; - + if(author.padIDs != null) { //remove pad from author - delete author.padIDs[padID]; + delete author.padIDs[padID]; db.set("globalAuthor:" + authorID, author); } }); diff --git a/src/node/db/Pad.js b/src/node/db/Pad.js index d44cb7b3..0cb01cac 100644 --- a/src/node/db/Pad.js +++ b/src/node/db/Pad.js @@ -464,9 +464,10 @@ Pad.prototype.copy = function copy(destinationID, force, callback) { } else force = true; - //kick everyone from this pad - // TODO: this presents a message on the client saying that the pad was 'deleted'. Fix this? - padMessageHandler.kickSessionsFromPad(sourceID); + // Kick everyone from this pad. + // This was commented due to https://github.com/ether/etherpad-lite/issues/3183. + // Do we really need to kick everyone out? + // padMessageHandler.kickSessionsFromPad(sourceID); // flush the source pad: _this.saveToDatabase(); diff --git a/src/node/db/SecurityManager.js b/src/node/db/SecurityManager.js index bbd8cef4..98feafb3 100644 --- a/src/node/db/SecurityManager.js +++ b/src/node/db/SecurityManager.js @@ -22,6 +22,7 @@ var ERR = require("async-stacktrace"); var async = require("async"); var authorManager = require("./AuthorManager"); +var hooks = require("ep_etherpad-lite/static/js/pluginfw/hooks.js"); var padManager = require("./PadManager"); var sessionManager = require("./SessionManager"); var settings = require("../utils/Settings"); @@ -45,6 +46,14 @@ exports.checkAccess = function (padID, sessionCookie, token, password, callback) return; } + // allow plugins to deny access + var deniedByHook = hooks.callAll("onAccessCheck", {'padID': padID, 'password': password, 'token': token, 'sessionCookie': sessionCookie}).indexOf(false) > -1; + if(deniedByHook) + { + callback(null, {accessStatus: "deny"}); + return; + } + // a valid session is required (api-only mode) if(settings.requireSession) { diff --git a/src/node/handler/APIHandler.js b/src/node/handler/APIHandler.js index 179c2b40..05e14705 100644 --- a/src/node/handler/APIHandler.js +++ b/src/node/handler/APIHandler.js @@ -24,17 +24,19 @@ var fs = require("fs"); var api = require("../db/API"); var padManager = require("../db/PadManager"); var randomString = require("../utils/randomstring"); +var argv = require('../utils/Cli').argv; //ensure we have an apikey var apikey = null; +var apikeyFilename = argv.apikey || "./APIKEY.txt"; try { - apikey = fs.readFileSync("./APIKEY.txt","utf8"); + apikey = fs.readFileSync(apikeyFilename,"utf8"); } catch(e) { apikey = randomString(32); - fs.writeFileSync("./APIKEY.txt",apikey,"utf8"); + fs.writeFileSync(apikeyFilename,apikey,"utf8"); } //a list of all functions diff --git a/src/node/handler/ImportHandler.js b/src/node/handler/ImportHandler.js index 6aa94e64..3e3dc195 100644 --- a/src/node/handler/ImportHandler.js +++ b/src/node/handler/ImportHandler.js @@ -90,7 +90,7 @@ exports.doImport = function(req, res, padId) //this allows us to accept source code files like .c or .java function(callback) { var fileEnding = path.extname(srcFile).toLowerCase() - , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad"] + , knownFileEndings = [".txt", ".doc", ".docx", ".pdf", ".odt", ".html", ".htm", ".etherpad", ".rtf"] , fileEndingKnown = (knownFileEndings.indexOf(fileEnding) > -1); //if the file ending is known, continue as normal diff --git a/src/node/hooks/express.js b/src/node/hooks/express.js index 17910e4b..48dcf56c 100644 --- a/src/node/hooks/express.js +++ b/src/node/hooks/express.js @@ -25,6 +25,10 @@ exports.createServer = function () { else{ console.warn("Admin username and password not set in settings.json. To access admin please uncomment and edit 'users' in settings.json"); } + var env = process.env.NODE_ENV || 'development'; + if(env !== 'production'){ + console.warn("Etherpad is running in Development mode. This mode is slower for users and less secure than production mode. You should set the NODE_ENV environment variable to production by using: export NODE_ENV=production"); + } } exports.restartServer = function () { diff --git a/src/node/hooks/express/admin.js b/src/node/hooks/express/admin.js index 70539f0c..0884cde5 100644 --- a/src/node/hooks/express/admin.js +++ b/src/node/hooks/express/admin.js @@ -2,7 +2,7 @@ var eejs = require('ep_etherpad-lite/node/eejs'); exports.expressCreateServer = function (hook_name, args, cb) { args.app.get('/admin', function(req, res) { - if('/' != req.path[req.path.length-1]) return res.redirect('/admin/'); + if('/' != req.path[req.path.length-1]) return res.redirect('./admin/'); res.send( eejs.require("ep_etherpad-lite/templates/admin/index.html", {}) ); }); } diff --git a/src/node/hooks/express/apicalls.js b/src/node/hooks/express/apicalls.js index 4482fd84..d6011c97 100644 --- a/src/node/hooks/express/apicalls.js +++ b/src/node/hooks/express/apicalls.js @@ -3,7 +3,7 @@ var apiLogger = log4js.getLogger("API"); var clientLogger = log4js.getLogger("client"); var formidable = require('formidable'); var apiHandler = require('../../handler/APIHandler'); -var isVarName = require('is-var-name'); +var isValidJSONPName = require('./isValidJSONPName'); //This is for making an api call, collecting all post information and passing it to the apiHandler var apiCaller = function(req, res, fields) { @@ -19,7 +19,7 @@ var apiCaller = function(req, res, fields) { apiLogger.info("RESPONSE, " + req.params.func + ", " + response); //is this a jsonp call, if yes, add the function call - if(req.query.jsonp && isVarName(response)) + if(req.query.jsonp && isValidJSONPName.check(req.query.jsonp)) response = req.query.jsonp + "(" + response + ")"; res._____send(response); @@ -46,7 +46,7 @@ exports.expressCreateServer = function (hook_name, args, cb) { //The Etherpad client side sends information about how a disconnect happened args.app.post('/ep/pad/connection-diagnostic-info', function(req, res) { - new formidable.IncomingForm().parse(req, function(err, fields, files) { + new formidable.IncomingForm().parse(req, function(err, fields, files) { clientLogger.info("DIAGNOSTIC-INFO: " + fields.diagnosticInfo); res.end("OK"); }); @@ -54,7 +54,7 @@ exports.expressCreateServer = function (hook_name, args, cb) { //The Etherpad client side sends information about client side javscript errors args.app.post('/jserror', function(req, res) { - new formidable.IncomingForm().parse(req, function(err, fields, files) { + new formidable.IncomingForm().parse(req, function(err, fields, files) { try { var data = JSON.parse(fields.errorInfo) }catch(e){ @@ -64,7 +64,7 @@ exports.expressCreateServer = function (hook_name, args, cb) { res.end("OK"); }); }); - + //Provide a possibility to query the latest available API version args.app.get('/api', function (req, res) { res.json({"currentVersion" : apiHandler.latestApiVersion}); diff --git a/src/node/hooks/express/importexport.js b/src/node/hooks/express/importexport.js index 5ebac1db..a62942cc 100644 --- a/src/node/hooks/express/importexport.js +++ b/src/node/hooks/express/importexport.js @@ -2,6 +2,7 @@ var hasPadAccess = require("../../padaccess"); var settings = require('../../utils/Settings'); var exportHandler = require('../../handler/ExportHandler'); var importHandler = require('../../handler/ImportHandler'); +var padManager = require("../../db/PadManager"); exports.expressCreateServer = function (hook_name, args, cb) { args.app.get('/p/:pad/:rev?/export/:type', function(req, res, next) { @@ -22,14 +23,29 @@ exports.expressCreateServer = function (hook_name, args, cb) { res.header("Access-Control-Allow-Origin", "*"); hasPadAccess(req, res, function() { - exportHandler.doExport(req, res, req.params.pad, req.params.type); + console.log('req.params.pad', req.params.pad); + padManager.doesPadExists(req.params.pad, function(err, exists) + { + if(!exists) { + return next(); + } + + exportHandler.doExport(req, res, req.params.pad, req.params.type); + }); }); }); //handle import requests args.app.post('/p/:pad/import', function(req, res, next) { hasPadAccess(req, res, function() { - importHandler.doImport(req, res, req.params.pad); + padManager.doesPadExists(req.params.pad, function(err, exists) + { + if(!exists) { + return next(); + } + + importHandler.doImport(req, res, req.params.pad); + }); }); }); } diff --git a/src/node/hooks/express/webaccess.js b/src/node/hooks/express/webaccess.js index 190021a3..4cb4b9d3 100644 --- a/src/node/hooks/express/webaccess.js +++ b/src/node/hooks/express/webaccess.js @@ -20,7 +20,7 @@ exports.basicAuth = function (req, res, next) { // Do not require auth for static paths and the API...this could be a bit brittle if (req.path.match(/^\/(static|javascripts|pluginfw|api)/)) return cb(true); - if (req.path.indexOf('/admin') != 0) { + if (req.path.toLowerCase().indexOf('/admin') != 0) { if (!settings.requireAuthentication) return cb(true); if (!settings.requireAuthorization && req.session && req.session.user) return cb(true); } @@ -36,13 +36,16 @@ exports.basicAuth = function (req, res, next) { var userpass = new Buffer(req.headers.authorization.split(' ')[1], 'base64').toString().split(":") var username = userpass.shift(); var password = userpass.join(':'); - - if (settings.users[username] != undefined && settings.users[username].password == password) { - settings.users[username].username = username; - req.session.user = settings.users[username]; - return cb(true); - } - return hooks.aCallFirst("authenticate", {req: req, res:res, next:next, username: username, password: password}, hookResultMangle(cb)); + var fallback = function(success) { + if (success) return cb(true); + if (settings.users[username] != undefined && settings.users[username].password === password) { + settings.users[username].username = username; + req.session.user = settings.users[username]; + return cb(true); + } + return cb(false); + }; + return hooks.aCallFirst("authenticate", {req: req, res:res, next:next, username: username, password: password}, hookResultMangle(fallback)); } hooks.aCallFirst("authenticate", {req: req, res:res, next:next}, hookResultMangle(cb)); } @@ -126,4 +129,3 @@ exports.expressConfigure = function (hook_name, args, cb) { args.app.use(exports.basicAuth); } - diff --git a/src/node/utils/Cli.js b/src/node/utils/Cli.js index 9419ed26..154590dc 100644 --- a/src/node/utils/Cli.js +++ b/src/node/utils/Cli.js @@ -39,5 +39,15 @@ for ( var i = 0; i < argv.length; i++ ) { exports.argv.credentials = arg; } + // Override location of settings.json file + if ( prevArg == '--sessionkey' || prevArg == '-k' ) { + exports.argv.sessionkey = arg; + } + + // Override location of settings.json file + if ( prevArg == '--apikey' || prevArg == '-k' ) { + exports.argv.apikey = arg; + } + prevArg = arg; } diff --git a/src/node/utils/ExportEtherpad.js b/src/node/utils/ExportEtherpad.js index 46ae0d7a..a68ab0b2 100644 --- a/src/node/utils/ExportEtherpad.js +++ b/src/node/utils/ExportEtherpad.js @@ -22,25 +22,18 @@ var ERR = require("async-stacktrace"); exports.getPadRaw = function(padId, callback){ async.waterfall([ function(cb){ - - // Get the Pad - db.findKeys("pad:"+padId, null, function(err,padcontent){ - if(!err){ - cb(err, padcontent); - } - }) + db.get("pad:"+padId, cb); }, function(padcontent,cb){ + var records = ["pad:"+padId]; + for (var i = 0; i <= padcontent.head; i++) { + records.push("pad:"+padId+":revs:" + i); + } + + for (var i = 0; i <= padcontent.chatHead; i++) { + records.push("pad:"+padId+":chat:" + i); + } - // Get the Pad available content keys - db.findKeys("pad:"+padId+":*", null, function(err,records){ - if(!err){ - for (var key in padcontent) { records.push(padcontent[key]);} - cb(err, records); - } - }) - }, - function(records, cb){ var data = {}; async.forEachSeries(Object.keys(records), function(key, r){ @@ -69,7 +62,7 @@ exports.getPadRaw = function(padId, callback){ } r(null); // callback; }); - }, function(err){ + }, function(err){ cb(err, data); }) } diff --git a/src/node/utils/ExportHtml.js b/src/node/utils/ExportHtml.js index 5276553a..bd0ad12c 100644 --- a/src/node/utils/ExportHtml.js +++ b/src/node/utils/ExportHtml.js @@ -110,31 +110,27 @@ function getHTMLFromAtext(pad, atext, authorColors) css+="