switch between s3/local storage in config

README.md

@@ -1,7 +1,5 @@
 # Spacedeck Open
 
-[![Greenkeeper badge](https://badges.greenkeeper.io/spacedeck/spacedeck-open.svg)](https://greenkeeper.io/)
-
 This is the free and open source version of Spacedeck, a web based, real time, collaborative whiteboard application with rich media support. Spacedeck was developed in 6 major releases during Autumn 2011 until the end of 2016 and was originally a commercial SaaS. The developers were Lukas F. Hartmann (mntmn) and Martin Güther (magegu). All icons and large parts of the CSS were designed by Thomas Helbig (dergraph).
 
 As we plan to retire the subscription based service at spacedeck.com in late 2017, we decided to open-source Spacedeck to allow educational and other organizations who currently rely on Spacedeck to migrate to a self-hosted version.
@@ -25,16 +23,16 @@ We appreciate filed issues, pull requests and general discussion.
 
 Spacedeck uses the following major building blocks:
 
-- Node.js 4.x (Backend / API)
+- Node.js 7.x (Backend / API)
 - MongoDB 3.x (Datastore)
 - Redis 3.x (Datastore for realtime channels)
 - Vue.js (Frontend)
 
 It also has some binary dependencies for media conversion and PDF export:
 
-- imagemagick
+- imagemagick, graphicsmagick, libav(+codecs, ffmpeg replacement), audiowaveform (https://github.com/bbcrd/audiowaveform), phantomjs (http://phantomjs.org/)
 
-Currently, media files are stored in Amazon S3, so you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails, Amazon SES is required.
+Currently, media files are stored in Amazon S3, so you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails in production, Amazon SES is required.
 
 To install Spacedeck, you need node.js 4.x and a running MongoDB instance. Then, to install all node dependencies, run
 
@@ -44,11 +42,17 @@ To rebuild the frontend CSS styles (you need to do this at least once):
 
     gulp styles
 
+# Configuration
+
+see: config/config.json
+
 # Run
 
     export NODE_ENV=development
     npm start
 
+    open http://localhost:9000
+
 # License
 
 Spacedeck Open is released under the GNU Affero General Public License Version 3 (GNU AGPLv3).

app.js

@@ -15,13 +15,16 @@ const favicon = require('serve-favicon');
 const logger = require('morgan');
 const cookieParser = require('cookie-parser');
 const bodyParser = require('body-parser');
+
 const mongoose = require('mongoose');
+
 const swig = require('swig');
 const i18n = require('i18n-2');
 const helmet = require('helmet');
 
 const express = require('express');
 const app = express();
+const serveStatic = require('serve-static');
 
 const isProduction = app.get('env') === 'production';
 
@@ -67,7 +70,6 @@ app.use(bodyParser.urlencoded({
 }));
 
 app.use(cookieParser());
-app.use(helmet.noCache())
 app.use(helmet.frameguard())
 app.use(helmet.xssFilter())
 app.use(helmet.hsts({
@@ -109,6 +111,12 @@ app.use('/api/teams', require('./routes/api/teams'));
 app.use('/api/webgrabber', require('./routes/api/webgrabber'));
 app.use('/', require('./routes/root'));
 
+if (config.get('storage_local_path')) {
+  app.use('/storage', serveStatic(config.get('storage_local_path')+"/"+config.get('storage_bucket'), {
+    maxAge: 24*3600
+  }));
+}
+
 // catch 404 and forward to error handler
 app.use(require('./middlewares/404'));
 if (app.get('env') == 'development') {
@@ -128,7 +136,7 @@ mongoose.connect('mongodb://' + mongoHost + '/spacedeck');
 const port = 9000;
 
 const server = http.Server(app).listen(port, () => {
-
+  
   if ("send" in process) {
     process.send('online');
   }

config/default.json

@@ -1,7 +1,9 @@
 {
   "endpoint": "http://localhost:9000",
-  "storage_bucket": "my_spacedeck_s3_bucket",
-  "storage_cdn": "xyz.cloudfront.net",
+  "storage_region": "eu-central-1",
+  "storage_bucket": "my_spacedeck_bucket",
+  "storage_cdn": "/storage",
+  "storage_local_path": "./storage",
   "google_access" : "",
   "google_secret" : "",
   "admin_pass": "very_secret_admin_password",

helpers/artifact_converter.js

@@ -7,6 +7,7 @@ const fs = require('fs');
 const Models = require('../models/schema');
 const uploader = require('../helpers/uploader');
 const path = require('path');
+const os = require('os');
 
 const fileExtensionMap = {
   ".amr" : "audio/AMR",
@@ -245,13 +246,12 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) {
   if (max>320 || size.width > max || size.height > max) {
     var resizedFileName = max + "_"+fileName;
     var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + resizedFileName;
-    var localResizedFilePath = "/tmp/"+resizedFileName;
+    var localResizedFilePath = os.tmpdir()+"/"+resizedFileName;
     gm(localFilePath).resize(max, max).autoOrient().write(localResizedFilePath, function (err) {
       if(!err) {
         uploader.uploadFile(s3Key, "image/jpeg", localResizedFilePath, function(err, url) {
           if (err) callback(err);
           else{
-            console.log(localResizedFilePath);
             fs.unlink(localResizedFilePath, function (err) {
               if (err) {
                 console.error(err);

helpers/redis.js

@@ -1,14 +1,104 @@
 'use strict';
 
-const RedisConnection = require('ioredis');
-const websockets = require('./websockets');
+// this is a mock version of the Redis API,
+// emulating Redis if it is not available locally
+var notRedis = {
+  state: {},
+  topics: {},
+  
+  publish: function(topic, msg, cb) {
+    if (!this.topics[topic]) {
+      this.topics[topic] = {
+        subscribers: []
+      };
+    }
+    var t=this.topics[topic];
+    for (var i=0; i<t.subscribers.length; i++) {
+      var s=t.subscribers[i];
+      if (s.handler) {
+        s.handler(topic, msg);
+      }
+    }
+    if (cb) cb(null);
+  },
+
+  subscribe: function(topics, cb) {
+    var handle = {
+      handler: null,
+      on: function(evt, cb) {
+        if (evt == "message") {
+          this.handler = cb;
+        }
+      }
+    };
+    
+    for (var i=0; i<topics.length; i++) {
+      var topic = topics[i];
+      if (!this.topics[topic]) {
+        this.topics[topic] = {
+          subscribers: []
+        };
+      }
+    
+      var t=this.topics[topic];
+      t.subscribers.push(handle);
+    }
+
+    cb(null, topics.length);
+    return handle;
+  },
+
+  get: function(key, cb) {
+    cb(null, this.state[key]);
+    return this.state[key];
+  },
+
+  set: function(key, val, cb) {
+    this.state[key] = val;
+    cb();
+  },
+
+  del: function(key, cb) {
+    delete this.state[key];
+    cb(null);
+  },
+
+  sadd: function(key, skey, cb) {
+    if (!this.state[key]) this.state[key] = {};
+    this.state[key][skey] = true;
+    cb(null);
+  },
+
+  srem: function(key, skey, cb) {
+    if (this.state[key]) {
+      delete this.state[key][skey];
+    }
+    cb(null);
+  },
+
+  smembers: function(key, cb) {
+    cb(null, Object.keys(this.state[key]));
+  },
+
+  incr: function(key, cb) {
+    if (!this.state[key]) this.state[key] = 0;
+    this.state[key]++;
+    cb(null, this.state[key]);
+  },
+
+  expire: function() {
+  },
+}
 
 module.exports = {
-  connectRedis(){
-    const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost';
-    this.connection = new RedisConnection(6379, redisHost);
+  connectRedis: function() {
+    this.connection = notRedis;
   },
-  sendMessage(action, model, attributes, channelId) {
+  getConnection: function() {
+    this.connectRedis();
+    return this.connection;
+  },
+  sendMessage: function(action, model, attributes, channelId) {
     const data = JSON.stringify({
       channel_id: channelId,
       action: action,
@@ -17,12 +107,12 @@ module.exports = {
     });
     this.connection.publish('updates', data);
   },
-  logIp(ip, cb) {
+  logIp: function(ip, cb) {
     this.connection.incr("ip_"+ ip, (err, socketCounter) => {
       cb();
     });
   },
-  rateLimit(namespace, ip, cb) {
+  rateLimit: function(namespace, ip, cb) {
     const key = "limit_"+ namespace + "_"+ ip;
     const redis = this.connection;
 
@@ -47,7 +137,7 @@ module.exports = {
       }
     });
   }, 
-  isOnlineInSpace(user, space, cb) {
+  isOnlineInSpace: function(user, space, cb) {
     this.connection.smembers("space_" + space._id.toString(), function(err, list) {
       if (err) cb(err);
       else {
@@ -59,3 +149,6 @@ module.exports = {
     });
   }
 };
+
+return module.exports;
+

helpers/uploader.js

@@ -1,15 +1,21 @@
 'use strict';
 
-var AWS = require('aws-sdk');
-AWS.config.region = 'eu-central-1';
-
 var fs = require('fs');
 var config = require('config');
 
+// use AWS S3 or local folder depending on config
+if (config.get("storage_local_path")) {
+  var AWS = require('mock-aws-s3');
+  AWS.config.basePath = config.get("storage_local_path");
+} else {
+  var AWS = require('aws-sdk');
+  AWS.config.region = config.get("storage_region");
+}
+
 module.exports = {
   removeFile: (path, callback) => {
     const s3 = new AWS.S3({
-      region: 'eu-central-1'
+      region: config.get("storage_region")
     });
     const bucket = config.get("storage_bucket");
     s3.deleteObject({
@@ -39,9 +45,8 @@ module.exports = {
       }
     });
     fileStream.on('open', function () {
-      // FIXME
       var s3 = new AWS.S3({
-        region: 'eu-central-1'
+        region: config.get("storage_region")
       });
 
       s3.putObject({
@@ -54,7 +59,7 @@ module.exports = {
           console.error(err);
           callback(err);
         }else {
-          const url = "https://"+ config.get("storage_cdn") + "/" + fileName;
+          const url = config.get("storage_cdn") + "/" + fileName;
           console.log("[s3]" + localFilePath + " to " + url);
           callback(null, url);
         }

helpers/websockets.js

@@ -3,18 +3,19 @@ require('../models/schema');
 
 const WebSocketServer = require('ws').Server;
 
-const Redis = require('ioredis');
 const async = require('async');
 const _ = require("underscore");
 const mongoose = require("mongoose");
 const crypto = require('crypto');
 
-module.exports = {
-  startWebsockets: function(server){
-    this.setupSubscription();
-    this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost');
+var redis = require("./redis.js");
 
-    if(!this.current_websockets){
+module.exports = {
+  startWebsockets: function(server) {
+    this.setupSubscription();
+    this.state = redis.getConnection();
+    
+    if(!this.current_websockets) {
       this.current_websockets = [];
     }
 
@@ -117,8 +118,7 @@ module.exports = {
   },
 
   setupSubscription: function() {
-    this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost');
-    this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) {
+    this.cursorSubscriber = redis.getConnection().subscribe(['cursors', 'users', 'updates'], function (err, count) {
       console.log("[redis] websockets to " + count + " topics." );
     });
     this.cursorSubscriber.on('message', function (channel, rawMessage) {
@@ -206,7 +206,7 @@ module.exports = {
       console.log("websocket not found to remove");
     }
 
-    this.state.del(ws.id, function(err, res) {
+    this.state.del(ws.id+"", function(err, res) {
       if (err) console.error(err, res);
       else {
         this.removeUserInSpace(ws.space_id, ws, (err) => {
@@ -221,7 +221,8 @@ module.exports = {
   
   addUserInSpace: function(username, space, ws, cb) {
     console.log("[websockets] user "+username+" in "+space.access_mode +" space " +  space._id + " with socket "  +  ws.id);
-    this.state.set(ws.id,  username, function(err, res) {
+    
+    this.state.set(ws.id+"", username+"", function(err, res) {
       if(err) console.error(err, res);
       else {
         this.state.sadd("space_" + space._id, ws.id, function(err, res) {
@@ -238,7 +239,7 @@ module.exports = {
     }.bind(this));
   },
   removeUserInSpace: function(spaceId, ws, cb) {
-    this.state.srem("space_" + spaceId, ws.id, function(err, res) {
+    this.state.srem("space_" + spaceId, ws.id+"", function(err, res) {
       if (err) cb(err);
       else {
         console.log("[websockets] socket "+  ws.id + " went offline in space " + spaceId);

models/artifact.js

@@ -3,7 +3,7 @@
 var mongoose = require('mongoose');
 var Schema = mongoose.Schema;
 
-module.exports.artifactSchema = mongoose.model('Artifact', {
+module.exports.artifactSchema = Schema({
   mime: String,
   thumbnail_uri: String,
   space_id: Schema.Types.ObjectId,

package.json

@@ -6,12 +6,15 @@
     "start": "nodemon -e .js,.html bin/www",
     "test": "mocha"
   },
+  "engines": {
+    "node": ">=7.8.0"
+  },
   "dependencies": {
     "archiver": "1.3.0",
     "async": "2.3.0",
     "aws-sdk": "2.39.0",
     "basic-auth": "1.1.0",
-    "bcrypt": "*",
+    "bcryptjs": "2.4.3",
     "body-parser": "~1.17.1",
     "cheerio": "0.22.0",
     "config": "1.25.1",
@@ -19,17 +22,16 @@
     "csurf": "1.9.0",
     "debug": "~2.6.3",
     "execSync": "latest",
-    "express": "~4.15.2",
+    "express": "~4.13.0",
     "glob": "7.1.1",
     "gm": "1.23.0",
     "googleapis": "18.0.0",
-    "gulp": "^3.9.0",
-    "gulp-concat": "2.6.1",
-    "gulp-eslint": "*",
-    "gulp-express": "0.3.5",
+    "gulp": "^3.9.1",
+    "gulp-concat": "2.6.0",
+    "gulp-express": "0.3.0",
     "gulp-nodemon": "*",
-    "gulp-sass": "^3.1.0",
-    "gulp-uglify": "^2.1.2",
+    "gulp-sass": "^2.0.3",
+    "gulp-uglify": "^1.5.1",
     "gulp-util": "^3.0.6",
     "helmet": "^3.5.0",
     "i18n-2": "0.6.3",
@@ -37,13 +39,13 @@
     "lodash": "^4.3.0",
     "log-timestamp": "latest",
     "md5": "2.2.1",
+    "mock-aws-s3": "^2.6.0",
     "moment": "2.18.1",
     "mongoose": "4.9.3",
     "morgan": "1.8.1",
+    "node-phantom-simple": "2.2.4",
     "node-sass-middleware": "0.11.0",
     "pdfkit": "0.8.0",
-    "validator": "7.0.0",
-    "node-phantom-simple": "2.2.4",
     "phantomjs-prebuilt": "2.1.14",
     "pm2": "latest",
     "qr-image": "3.2.0",
@@ -51,18 +53,19 @@
     "request": "2.81.0",
     "sanitize-html": "^1.11.1",
     "serve-favicon": "~2.4.2",
-    "swig": "1.4.2",
+    "serve-static": "^1.13.1",
     "slug": "0.9.1",
+    "swig": "1.4.2",
     "underscore": "1.8.3",
+    "validator": "7.0.0",
     "weak": "1.0.1",
     "ws": "2.2.3"
   },
   "devDependencies": {
     "express": "^4.13.3",
-    "gulp": "^3.9.0",
+    "gulp": "^3.9.1",
     "gulp-clean": "^0.3.2",
     "gulp-concat": "^2.6.0",
-    "gulp-eslint": "^3.0.1",
     "gulp-express": "^0.3.0",
     "gulp-fingerprint": "^0.3.2",
     "gulp-nodemon": "^2.0.4",
@@ -71,16 +74,14 @@
     "gulp-rev-replace": "^0.4.3",
     "gulp-sass": "^3.1.0",
     "gulp-uglify": "^2.1.2",
-    "mocha": "*",
-    "nodemon": "*",
+    "nodemon": "1.11.0",
     "should": "^11.2.1",
     "supertest": "^3.0.0",
     "winston": "^2.3.1"
   },
   "description": "",
   "main": "Gulpfile.js",
-  "directories": {
-  },
+  "directories": {},
   "repository": {
     "type": "git",
     "url": "https://github.com/spacedeck/spacedeck-open.git"

routes/api/sessions.js

@@ -3,7 +3,7 @@
 var config = require('config');
 require('../../models/schema');
 
-var bcrypt = require('bcrypt');
+var bcrypt = require('bcryptjs');
 var crypo = require('crypto');
 
 var express = require('express');

routes/api/teams.js

@@ -9,7 +9,7 @@ var mailer = require('../../helpers/mailer');
 var fs = require('fs');
 var _ = require('underscore');
 var crypto = require('crypto');
-var bcrypt = require('bcrypt');
+var bcrypt = require('bcryptjs');
 
 var express = require('express');
 var router = express.Router();

routes/api/users.js

@@ -6,7 +6,7 @@ require('../../models/schema');
 var mailer = require('../../helpers/mailer');
 var uploader = require('../../helpers/uploader');
 
-var bcrypt = require('bcrypt');
+var bcrypt = require('bcryptjs');
 var crypo = require('crypto');
 var swig = require('swig');
 var async = require('async');