Minify and compress JS & CSS before sending it

node/minify.js

@@ -0,0 +1,233 @@
+var settings = require('./settings');
+var async = require('async');
+var fs = require('fs');
+var cleanCSS = require('clean-css');
+var jsp = require("uglify-js").parser;
+var pro = require("uglify-js").uglify;
+var compress=require("compress");
+var path = require('path');
+var Buffer = require('buffer').Buffer;
+var gzip = require('gzip');
+
+/**
+ * Answers a http request for the pad javascript
+ */
+exports.padJS = function(req, res)
+{
+  res.header("Content-Type","text/javascript");
+
+  var jsFiles = ["plugins.js", "undo-xpopup.js", "json2.js", "pad_utils.js", "pad_cookie.js", "pad_editor.js", "pad_editbar.js", "pad_docbar.js", "pad_modals.js", "ace.js", "collab_client.js", "pad_userlist.js", "pad_impexp.js", "pad_savedrevs.js", "pad_connectionstatus.js", "pad2.js"];
+  
+  //minifying is enabled
+  if(settings.minify)
+  {
+    var fileValues = {};
+    var embeds = {};
+    var latestModification = 0;
+    
+    async.series([
+      //find out the highest modification date
+      function(callback)
+      {        
+        var folders2check = ["../static/css","../static/js"];
+        
+        //go trough this two folders
+        async.forEach(folders2check, function(path, callback)
+        {
+          //read the files in the folder
+          fs.readdir(path, function(err, files)
+          {
+            if(err) { callback(err); return; }
+            
+            //we wanna check the directory itself for changes too
+            files.push(".");
+            
+            //go trough all files in this folder
+            async.forEach(files, function(filename, callback) 
+            {
+              //get the stat data of this file
+              fs.stat(path + "/" + filename, function(err, stats)
+              {
+                if(err) { callback(err); return; }
+              
+                //get the modification time
+                var modificationTime = stats.mtime.getTime();
+              
+                //compare the modification time to the highest found
+                if(modificationTime > latestModification)
+                {
+                  latestModification = modificationTime;
+                }
+                
+                callback();
+              });
+            }, callback);
+          });
+        }, callback);
+      },
+      function(callback)
+      {
+        //check the modification time of the minified js
+        fs.stat("../var/minified_pad.js", function(err, stats)
+        {
+          if(err && err.code != "ENOENT") callback(err);
+        
+          //there is no minfied file or there new changes since this file was generated, so continue generating this file
+          if((err && err.code == "ENOENT") || stats.mtime.getTime() < latestModification)
+          {
+            callback();
+          }
+          //the minified file is still up to date, stop minifying
+          else
+          {
+            callback("stop");
+          }
+        });
+      }, 
+      //load all js files
+      function (callback)
+      {
+        async.forEach(jsFiles, function (item, callback)
+        {
+          fs.readFile("../static/js/" + item, "utf-8", function(err, data)
+          {            
+            fileValues[item] = data;
+            callback(err);
+          });
+        }, callback);
+      },
+      //find all includes in ace.js and embed them 
+      function(callback)
+      {        
+        var founds = fileValues["ace.js"].match(/\$\$INCLUDE_[a-zA-Z_]+\([a-zA-Z0-9.\/_"]+\)/gi);
+        
+        //go trough all includes
+        async.forEach(founds, function (item, callback)
+        {
+          var filename = item.match(/"[^"]*"/g)[0].substr(1);
+          filename = filename.substr(0,filename.length-1);
+        
+          var type = item.match(/INCLUDE_[A-Z]+/g)[0].substr("INCLUDE_".length);
+        
+          var quote = item.search("_Q") != -1;
+        
+          //read the included file
+          fs.readFile(".." + filename, "utf-8", function(err, data)
+          {         
+            //compress the file               
+            if(type == "JS")
+            {
+              embeds[item] = "<script>\n" + compressJS([data])+ "\n\\x3c/script>";
+            }
+            else
+            {
+              embeds[item] = "<style>" + compressCSS([data])+ "</style>";
+            }
+            
+            //do the first escape
+            embeds[item] = JSON.stringify(embeds[item]).replace(/'/g, "\\'").replace(/\\"/g, "\"");
+            embeds[item] = embeds[item].substr(1);
+            embeds[item] = embeds[item].substr(0, embeds[item].length-1);
+            
+            //add quotes, if wished
+            if(quote)
+            {
+              embeds[item] = "'" + embeds[item] + "'";
+            }
+            
+            //do the second escape
+            embeds[item] = JSON.stringify(embeds[item]).replace(/'/g, "\\'").replace(/\"/g, "\"");
+            embeds[item] = embeds[item].substr(1);
+            embeds[item] = embeds[item].substr(0, embeds[item].length-1);
+            embeds[item] = "'" + embeds[item] + "'";
+            
+            callback(err);
+          });
+        }, function(err)
+        {
+          //replace the include command with the include
+          for(var i in embeds)
+          {
+            fileValues["ace.js"]=fileValues["ace.js"].replace(i, embeds[i]);
+          }
+          
+          callback(err);
+        });
+      },
+      //put all together and write it into a file
+      function(callback)
+      {
+        //put all javascript files in an array
+        var values = [];
+        for(var i in fileValues)
+        {
+          values.push(fileValues[i]);
+        }
+        
+        //minify all javascript files to one
+        var result = compressJS(values);
+        
+        async.parallel([
+          //write the results plain in a file
+          function(callback)
+          {
+            fs.writeFile("../var/minified_pad.js", result, "utf8", callback);  
+          },
+          //write the results compressed in a file
+          function(callback)
+          {
+            gzip(result, 9, function(err, compressedResult){
+              if(err) {callback(err); return}
+              
+              fs.writeFile("../var/minified_pad.js.gz", compressedResult, callback);  
+            });
+          }
+        ],callback);
+      }
+    ], function(err)
+    {
+      if(err && err != "stop") throw err;
+      
+      //check if gzip is supported by this browser
+      var gzipSupport = req.header('Accept-Encoding', '').indexOf('gzip') != -1;
+      
+      var pathStr;
+      if(gzipSupport)
+      {
+        pathStr = path.normalize(__dirname + "/../var/minified_pad.js.gz");
+        res.header('Content-Encoding', 'gzip');
+      }
+      else
+      {
+        pathStr = path.normalize(__dirname + "/../var/minified_pad.js");
+      }
+      
+      res.sendfile(pathStr);
+    })
+  }
+  //minifying is disabled, so load the files with jquery
+  else
+  {
+    for(var i in jsFiles)
+    {
+      res.write("$.getScript('/static/js/" + jsFiles[i]+ "');\n");
+    }
+    
+    res.end();
+  }
+}
+
+function compressJS(values)
+{
+  var complete = values.join("\n");
+  var ast = jsp.parse(complete); // parse code and get the initial AST
+  ast = pro.ast_mangle(ast); // get a new AST with mangled names
+  ast = pro.ast_squeeze(ast); // get an AST with compression optimizations
+  return pro.gen_code(ast); // compressed code here
+}
+
+function compressCSS(values)
+{
+  var complete = values.join("\n");
+  return cleanCSS.process(complete);
+}

node/server.js

@@ -16,12 +16,13 @@
 
 require('joose');
 
-var socketio = require('socket.io')
+var socketio = require('socket.io');
-var settings = require('./settings')
+var settings = require('./settings');
-var db = require('./db')
+var db = require('./db');
 var async = require('async');
 var express = require('express');
 var path = require('path');
+var minify = require('./minify');
 
 var serverName = "Etherpad-Lite ( http://j.mp/ep-lite )";
 
@@ -45,10 +46,27 @@ async.waterfall([
     app.get('/static/*', function(req, res)
     { 
       res.header("Server", serverName);
-      var filePath = path.normalize(__dirname + "/.." + req.url);
+      var filePath = path.normalize(__dirname + "/.." + req.url.split("?")[0]);
       res.sendfile(filePath, { maxAge: 1000*60*60 });
     });
     
+    //serve minified files
+    app.get('/minified/:id', function(req, res)
+    { 
+      res.header("Server", serverName);
+      
+      var id = req.params.id;
+      
+      if(id == "pad.js")
+      {
+        minify.padJS(req,res);
+      }
+      else
+      {
+        res.send('404 - Not Found', 404);
+      }
+    });
+    
     //serve pad.html under /p
     app.get('/p/:pad', function(req, res)
     {

node/settings.js

@@ -22,6 +22,7 @@ exports.dbType = "sqlite";
 exports.dbSettings = { "filename" : "../var/sqlite.db" };
 exports.logHTTP = true;
 exports.defaultPadText = "Welcome to Etherpad Lite!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nEtherpad Lite on Github: http:\/\/j.mp/ep-lite\n";
+exports.minify = true;
 
 //read the settings sync
 var settingsStr = fs.readFileSync("../settings.json").toString();

package.json

@@ -1,15 +1,18 @@
 {
   "name"          : "ep-lite",
   "description"   : "A Etherpad based on node.js",
-  "url"           : "https://github.com/Pita/etherpad-lite",
+  "homepage"      : "https://github.com/Pita/etherpad-lite",
   "keywords"      : ["etherpad", "realtime", "collaborative", "editor"],
   "author"        : "Peter 'Pita' Martischka <petermartischka@googlemail.com>",
   "dependencies"  : {
-                      "socket.io"  : ">=0.6.18",
+                      "socket.io"  : "0.6.18",
-                      "ueberDB"    : ">=0.0.3",
+                      "ueberDB"    : "0.0.3",
-                      "async"      : ">=0.1.9",
+                      "async"      : "0.1.9",
-                      "joose"      : ">=3.18.0",
+                      "joose"      : "3.18.0",
-                      "express"    : ">=2.3.6"
+                      "express"    : "2.3.6",
+                      "clean-css"  : "0.2.3",
+                      "uglify-js"  : "1.0.2",
+                      "gzip"       : "0.1.0"
                     },
   "version"       : "0.0.3",
   "bin"           : { 

settings.json

@@ -18,12 +18,16 @@ This file must be valid JSON. But comments are allowed
                     "host"    : "localhost", 
                     "password": "", 
                     "database": "store"
-                  }
+                  },
   */
   
   //if true, every http request will be loged to stdout
   "logHTTP" : true,
   
   //the default text of a pad
-  "defaultPadText" : "Welcome to Etherpad Lite!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nEtherpad Lite on Github: http:\/\/j.mp/ep-lite\n"
+  "defaultPadText" : "Welcome to Etherpad Lite!\n\nThis pad text is synchronized as you type, so that everyone viewing this page sees the same text. This allows you to collaborate seamlessly on documents!\n\nEtherpad Lite on Github: http:\/\/j.mp/ep-lite\n",
+  
+  /* if true, all css & js will be minified before sending to the client. This will improve the loading performance massivly, 
+     but makes it impossible to debug the javascript/css */
+  "minify" : true
 }

static/js/ace.js

@@ -164,8 +164,7 @@ function Ace2Editor() {
     };
 
     (function() {
-      var doctype = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" '+
+      var doctype = "<!doctype html>";
-	'"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">';
      
       var iframeHTML = ["'"+doctype+"<html><head>'"];
 
@@ -173,18 +172,9 @@ function Ace2Editor() {
         "aceInitInnerdocbodyHead", {iframeHTML:iframeHTML});
   
       // these lines must conform to a specific format because they are passed by the build script:      
-      //iframeHTML.push($$INCLUDE_CSS_Q("editor.css syntax.css inner.css"));
-      
       iframeHTML.push($$INCLUDE_CSS_Q("/static/css/editor.css"));
       iframeHTML.push($$INCLUDE_CSS_Q("/static/css/syntax.css"));
       iframeHTML.push($$INCLUDE_CSS_Q("/static/css/inner.css"));
-      
-      //iframeHTML.push(INCLUDE_JS_Q_DEV("ace2_common_dev.js"));
-      //iframeHTML.push(INCLUDE_JS_Q_DEV("profiler.js"));
-      
-      //iframeHTML.push($$INCLUDE_JS_Q("ace2_common.js skiplist.js virtual_lines.js easysync2.js cssmanager.js colorutils.js undomodule.js contentcollector.js changesettracker.js linestylefilter.js domline.js"));
-      //iframeHTML.push($$INCLUDE_JS_Q("ace2_inner.js"));
-      
       iframeHTML.push($$INCLUDE_JS_Q("/static/js/ace2_common.js"));
       iframeHTML.push($$INCLUDE_JS_Q("/static/js/skiplist.js"));
       iframeHTML.push($$INCLUDE_JS_Q("/static/js/virtual_lines.js"));
@@ -212,8 +202,8 @@ function Ace2Editor() {
 	'outerdocbody.insertBefore(iframe, outerdocbody.firstChild); '+
 	'iframe.ace_outerWin = window; '+
 	'readyFunc = function() { editorInfo.onEditorReady(); readyFunc = null; editorInfo = null; }; '+
-	'var doc = iframe.contentWindow.document; doc.open(); doc.write('+
+	'var doc = iframe.contentWindow.document; doc.open(); var text = ('+
-	iframeHTML.join('+')+'); doc.close(); '+
+	iframeHTML.join('+')+').replace(/\\\\x3c/g, \'<\');doc.write(text); doc.close(); '+
 	'}, 0); }';
 
       var outerHTML = [doctype, '<html><head>',
@@ -221,10 +211,9 @@ function Ace2Editor() {
 	// bizarrely, in FF2, a file with no "external" dependencies won't finish loading properly
 	// (throbs busy while typing)
 	'<link rel="stylesheet" type="text/css" href="data:text/css,"/>',
-	'\x3cscript>', outerScript, '\x3c/script>',
+	'\x3cscript>\n', outerScript, '\n\x3c/script>',
 	'</head><body id="outerdocbody"><div id="sidediv"><!-- --></div><div id="linemetricsdiv">x</div><div id="overlaysdiv"><!-- --></div></body></html>'];
 
-
       if (!Array.prototype.map) Array.prototype.map = function(fun) { //needed for IE
         if (typeof fun != "function") throw new TypeError();
         var len = this.length;

static/pad.html

@@ -18,15 +18,13 @@
   // <![CDATA[
 var clientVars = {};  // ]]>
 </script>
-<!--    <script type="text/javascript" src="/static/js/client.js"></script>-->
+
-<script type="text/javascript" src="/static/js/plugins.js"></script>
-<script type="text/javascript" src="/static/js/undo-xpopup.js"></script>
 <script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.5.1/jquery.min.js"></script>
-<!--<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.4/jquery.min.js"></script>-->
+<script type="text/javascript" src="/socket.io/socket.io.js"></script>
-<!-- <script type="text/javascript" src="/static/js/json.js"></script> -->
+
+<!--<script type="text/javascript" src="/static/js/plugins.js"></script>
+<script type="text/javascript" src="/static/js/undo-xpopup.js"></script>
 <script type="text/javascript" src="/static/js/json2.js"></script>
-<!--<script type="text/javascript" src="/static/js/colorutils.js"></script>-->
-<!--<script type="text/javascript" src="/static/js/draggable.js"></script>-->
 <script type="text/javascript" src="/static/js/pad_utils.js"></script>
 <script type="text/javascript" src="/static/js/pad_cookie.js"></script>
 <script type="text/javascript" src="/static/js/pad_editor.js"></script>
@@ -40,7 +38,10 @@ var clientVars = {};  // ]]>
 <script type="text/javascript" src="/static/js/pad_savedrevs.js"></script>
 <script type="text/javascript" src="/static/js/pad_connectionstatus.js"></script>
 <script type="text/javascript" src="/static/js/pad2.js"></script>
-<script type="text/javascript" src="/socket.io/socket.io.js"></script>
+-->
+
+<script type="text/javascript" src="/minified/pad.js"></script>
+
 </head>
 
 <body>

var/.gitignore

@@ -1 +1,2 @@
 sqlite.db
+minified*