switch between s3/local storage in config

README.md

@@ -23,16 +23,16 @@ We appreciate filed issues, pull requests and general discussion.
 
 Spacedeck uses the following major building blocks:
 
-- Node.js 4.x (Backend / API)
+- Node.js 7.x (Backend / API)
 - MongoDB 3.x (Datastore)
 - Redis 3.x (Datastore for realtime channels)
 - Vue.js (Frontend)
 
 It also has some binary dependencies for media conversion and PDF export:
 
-- imagemagick
+- imagemagick, graphicsmagick, libav(+codecs, ffmpeg replacement), audiowaveform (https://github.com/bbcrd/audiowaveform), phantomjs (http://phantomjs.org/)
 
-Currently, media files are stored in Amazon S3, so you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails, Amazon SES is required.
+Currently, media files are stored in Amazon S3, so you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails in production, Amazon SES is required.
 
 To install Spacedeck, you need node.js 4.x and a running MongoDB instance. Then, to install all node dependencies, run
 
@@ -42,11 +42,17 @@ To rebuild the frontend CSS styles (you need to do this at least once):
 
     gulp styles
 
+# Configuration
+
+see: config/config.json
+
 # Run
 
     export NODE_ENV=development
     npm start
 
+    open http://localhost:9000
+
 # License
 
 Spacedeck Open is released under the GNU Affero General Public License Version 3 (GNU AGPLv3).

app.js

@@ -15,13 +15,16 @@ const favicon = require('serve-favicon');
 const logger = require('morgan');
 const cookieParser = require('cookie-parser');
 const bodyParser = require('body-parser');
+
 const mongoose = require('mongoose');
+
 const swig = require('swig');
 const i18n = require('i18n-2');
 const helmet = require('helmet');
 
 const express = require('express');
 const app = express();
+const serveStatic = require('serve-static');
 
 const isProduction = app.get('env') === 'production';
 
@@ -67,7 +70,6 @@ app.use(bodyParser.urlencoded({
 }));
 
 app.use(cookieParser());
-app.use(helmet.noCache())
 app.use(helmet.frameguard())
 app.use(helmet.xssFilter())
 app.use(helmet.hsts({
@@ -109,6 +111,12 @@ app.use('/api/teams', require('./routes/api/teams'));
 app.use('/api/webgrabber', require('./routes/api/webgrabber'));
 app.use('/', require('./routes/root'));
 
+if (config.get('storage_local_path')) {
+  app.use('/storage', serveStatic(config.get('storage_local_path')+"/"+config.get('storage_bucket'), {
+    maxAge: 24*3600
+  }));
+}
+
 // catch 404 and forward to error handler
 app.use(require('./middlewares/404'));
 if (app.get('env') == 'development') {
@@ -128,7 +136,7 @@ mongoose.connect('mongodb://' + mongoHost + '/spacedeck');
 const port = 9000;
 
 const server = http.Server(app).listen(port, () => {
-
+  
   if ("send" in process) {
     process.send('online');
   }

config/default.json

@@ -1,7 +1,9 @@
 {
   "endpoint": "http://localhost:9000",
-  "storage_bucket": "my_spacedeck_s3_bucket",
-  "storage_cdn": "xyz.cloudfront.net",
+  "storage_region": "eu-central-1",
+  "storage_bucket": "my_spacedeck_bucket",
+  "storage_cdn": "/storage",
+  "storage_local_path": "./storage",
   "google_access" : "",
   "google_secret" : "",
   "admin_pass": "very_secret_admin_password",

helpers/artifact_converter.js

@@ -7,6 +7,7 @@ const fs = require('fs');
 const Models = require('../models/schema');
 const uploader = require('../helpers/uploader');
 const path = require('path');
+const os = require('os');
 
 const fileExtensionMap = {
   ".amr" : "audio/AMR",
@@ -245,13 +246,12 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) {
   if (max>320 || size.width > max || size.height > max) {
     var resizedFileName = max + "_"+fileName;
     var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + resizedFileName;
-    var localResizedFilePath = "/tmp/"+resizedFileName;
+    var localResizedFilePath = os.tmpdir()+"/"+resizedFileName;
     gm(localFilePath).resize(max, max).autoOrient().write(localResizedFilePath, function (err) {
       if(!err) {
         uploader.uploadFile(s3Key, "image/jpeg", localResizedFilePath, function(err, url) {
           if (err) callback(err);
           else{
-            console.log(localResizedFilePath);
             fs.unlink(localResizedFilePath, function (err) {
               if (err) {
                 console.error(err);

helpers/redis.js

@@ -1,14 +1,104 @@
 'use strict';
 
-const RedisConnection = require('ioredis');
-const websockets = require('./websockets');
+// this is a mock version of the Redis API,
+// emulating Redis if it is not available locally
+var notRedis = {
+  state: {},
+  topics: {},
+  
+  publish: function(topic, msg, cb) {
+    if (!this.topics[topic]) {
+      this.topics[topic] = {
+        subscribers: []
+      };
+    }
+    var t=this.topics[topic];
+    for (var i=0; i<t.subscribers.length; i++) {
+      var s=t.subscribers[i];
+      if (s.handler) {
+        s.handler(topic, msg);
+      }
+    }
+    if (cb) cb(null);
+  },
+
+  subscribe: function(topics, cb) {
+    var handle = {
+      handler: null,
+      on: function(evt, cb) {
+        if (evt == "message") {
+          this.handler = cb;
+        }
+      }
+    };
+    
+    for (var i=0; i<topics.length; i++) {
+      var topic = topics[i];
+      if (!this.topics[topic]) {
+        this.topics[topic] = {
+          subscribers: []
+        };
+      }
+    
+      var t=this.topics[topic];
+      t.subscribers.push(handle);
+    }
+
+    cb(null, topics.length);
+    return handle;
+  },
+
+  get: function(key, cb) {
+    cb(null, this.state[key]);
+    return this.state[key];
+  },
+
+  set: function(key, val, cb) {
+    this.state[key] = val;
+    cb();
+  },
+
+  del: function(key, cb) {
+    delete this.state[key];
+    cb(null);
+  },
+
+  sadd: function(key, skey, cb) {
+    if (!this.state[key]) this.state[key] = {};
+    this.state[key][skey] = true;
+    cb(null);
+  },
+
+  srem: function(key, skey, cb) {
+    if (this.state[key]) {
+      delete this.state[key][skey];
+    }
+    cb(null);
+  },
+
+  smembers: function(key, cb) {
+    cb(null, Object.keys(this.state[key]));
+  },
+
+  incr: function(key, cb) {
+    if (!this.state[key]) this.state[key] = 0;
+    this.state[key]++;
+    cb(null, this.state[key]);
+  },
+
+  expire: function() {
+  },
+}
 
 module.exports = {
-  connectRedis(){
-    const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost';
-    this.connection = new RedisConnection(6379, redisHost);
+  connectRedis: function() {
+    this.connection = notRedis;
   },
-  sendMessage(action, model, attributes, channelId) {
+  getConnection: function() {
+    this.connectRedis();
+    return this.connection;
+  },
+  sendMessage: function(action, model, attributes, channelId) {
     const data = JSON.stringify({
       channel_id: channelId,
       action: action,
@@ -17,12 +107,12 @@ module.exports = {
     });
     this.connection.publish('updates', data);
   },
-  logIp(ip, cb) {
+  logIp: function(ip, cb) {
     this.connection.incr("ip_"+ ip, (err, socketCounter) => {
       cb();
     });
   },
-  rateLimit(namespace, ip, cb) {
+  rateLimit: function(namespace, ip, cb) {
     const key = "limit_"+ namespace + "_"+ ip;
     const redis = this.connection;
 
@@ -47,7 +137,7 @@ module.exports = {
       }
     });
   }, 
-  isOnlineInSpace(user, space, cb) {
+  isOnlineInSpace: function(user, space, cb) {
     this.connection.smembers("space_" + space._id.toString(), function(err, list) {
       if (err) cb(err);
       else {
@@ -59,3 +149,6 @@ module.exports = {
     });
   }
 };
+
+return module.exports;
+

helpers/uploader.js

@@ -1,15 +1,21 @@
 'use strict';
 
-var AWS = require('aws-sdk');
-AWS.config.region = 'eu-central-1';
-
 var fs = require('fs');
 var config = require('config');
 
+// use AWS S3 or local folder depending on config
+if (config.get("storage_local_path")) {
+  var AWS = require('mock-aws-s3');
+  AWS.config.basePath = config.get("storage_local_path");
+} else {
+  var AWS = require('aws-sdk');
+  AWS.config.region = config.get("storage_region");
+}
+
 module.exports = {
   removeFile: (path, callback) => {
     const s3 = new AWS.S3({
-      region: 'eu-central-1'
+      region: config.get("storage_region")
     });
     const bucket = config.get("storage_bucket");
     s3.deleteObject({
@@ -39,9 +45,8 @@ module.exports = {
       }
     });
     fileStream.on('open', function () {
-      // FIXME
       var s3 = new AWS.S3({
-        region: 'eu-central-1'
+        region: config.get("storage_region")
       });
 
       s3.putObject({
@@ -54,7 +59,7 @@ module.exports = {
           console.error(err);
           callback(err);
         }else {
-          const url = "https://"+ config.get("storage_cdn") + "/" + fileName;
+          const url = config.get("storage_cdn") + "/" + fileName;
           console.log("[s3]" + localFilePath + " to " + url);
           callback(null, url);
         }

helpers/websockets.js

@@ -3,18 +3,19 @@ require('../models/schema');
 
 const WebSocketServer = require('ws').Server;
 
-const Redis = require('ioredis');
 const async = require('async');
 const _ = require("underscore");
 const mongoose = require("mongoose");
 const crypto = require('crypto');
 
-module.exports = {
-  startWebsockets: function(server){
-    this.setupSubscription();
-    this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost');
+var redis = require("./redis.js");
 
-    if(!this.current_websockets){
+module.exports = {
+  startWebsockets: function(server) {
+    this.setupSubscription();
+    this.state = redis.getConnection();
+    
+    if(!this.current_websockets) {
       this.current_websockets = [];
     }
 
@@ -117,8 +118,7 @@ module.exports = {
   },
 
   setupSubscription: function() {
-    this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost');
-    this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) {
+    this.cursorSubscriber = redis.getConnection().subscribe(['cursors', 'users', 'updates'], function (err, count) {
       console.log("[redis] websockets to " + count + " topics." );
     });
     this.cursorSubscriber.on('message', function (channel, rawMessage) {
@@ -206,7 +206,7 @@ module.exports = {
       console.log("websocket not found to remove");
     }
 
-    this.state.del(ws.id, function(err, res) {
+    this.state.del(ws.id+"", function(err, res) {
       if (err) console.error(err, res);
       else {
         this.removeUserInSpace(ws.space_id, ws, (err) => {
@@ -221,7 +221,8 @@ module.exports = {
   
   addUserInSpace: function(username, space, ws, cb) {
     console.log("[websockets] user "+username+" in "+space.access_mode +" space " +  space._id + " with socket "  +  ws.id);
-    this.state.set(ws.id,  username, function(err, res) {
+    
+    this.state.set(ws.id+"", username+"", function(err, res) {
       if(err) console.error(err, res);
       else {
         this.state.sadd("space_" + space._id, ws.id, function(err, res) {
@@ -238,7 +239,7 @@ module.exports = {
     }.bind(this));
   },
   removeUserInSpace: function(spaceId, ws, cb) {
-    this.state.srem("space_" + spaceId, ws.id, function(err, res) {
+    this.state.srem("space_" + spaceId, ws.id+"", function(err, res) {
       if (err) cb(err);
       else {
         console.log("[websockets] socket "+  ws.id + " went offline in space " + spaceId);

models/artifact.js

@@ -3,7 +3,7 @@
 var mongoose = require('mongoose');
 var Schema = mongoose.Schema;
 
-module.exports.artifactSchema = mongoose.model('Artifact', {
+module.exports.artifactSchema = Schema({
   mime: String,
   thumbnail_uri: String,
   space_id: Schema.Types.ObjectId,

package.json

@@ -6,81 +6,82 @@
     "start": "nodemon -e .js,.html bin/www",
     "test": "mocha"
   },
+  "engines": {
+    "node": ">=7.8.0"
+  },
   "dependencies": {
-    "archiver": "0.14.4",
-    "async": "1.3.0",
-    "aws-sdk": "2.1.39",
-    "basic-auth": "1.0.3",
-    "bcrypt": "*",
-    "body-parser": "~1.13.1",
-    "cheerio": "0.19.0",
-    "config": "1.14.0",
-    "cookie-parser": "~1.3.5",
-    "csurf": "1.8.3",
-    "debug": "~2.2.0",
+    "archiver": "1.3.0",
+    "async": "2.3.0",
+    "aws-sdk": "2.39.0",
+    "basic-auth": "1.1.0",
+    "bcryptjs": "2.4.3",
+    "body-parser": "~1.17.1",
+    "cheerio": "0.22.0",
+    "config": "1.25.1",
+    "cookie-parser": "~1.4.3",
+    "csurf": "1.9.0",
+    "debug": "~2.6.3",
     "execSync": "latest",
     "express": "~4.13.0",
-    "glob": "5.0.14",
-    "gm": "1.18.1",
-    "googleapis": "2.1.3",
-    "gulp": "^3.9.0",
+    "glob": "7.1.1",
+    "gm": "1.23.0",
+    "googleapis": "18.0.0",
+    "gulp": "^3.9.1",
     "gulp-concat": "2.6.0",
-    "gulp-eslint": "*",
     "gulp-express": "0.3.0",
     "gulp-nodemon": "*",
     "gulp-sass": "^2.0.3",
     "gulp-uglify": "^1.5.1",
     "gulp-util": "^3.0.6",
-    "helmet": "^1.1.0",
-    "i18n-2": "0.4.6",
-    "ioredis": "1.6.1",
+    "helmet": "^3.5.0",
+    "i18n-2": "0.6.3",
+    "ioredis": "2.5.0",
     "lodash": "^4.3.0",
     "log-timestamp": "latest",
-    "md5": "2.0.0",
-    "moment": "2.10.6",
-    "mongoose": "4.4.3",
-    "morgan": "1.6.1",
-    "node-sass-middleware": "0.8.0",
-    "pdfkit": "0.7.1",
-    "validator": "5.2.0",
+    "md5": "2.2.1",
+    "mock-aws-s3": "^2.6.0",
+    "moment": "2.18.1",
+    "mongoose": "4.9.3",
+    "morgan": "1.8.1",
     "node-phantom-simple": "2.2.4",
-    "phantomjs-prebuilt": "2.1.7",
+    "node-sass-middleware": "0.11.0",
+    "pdfkit": "0.8.0",
+    "phantomjs-prebuilt": "2.1.14",
     "pm2": "latest",
-    "qr-image": "3.1.0",
-    "raven": "0.8.1",
-    "request": "2.60.0",
+    "qr-image": "3.2.0",
+    "raven": "1.2.0",
+    "request": "2.81.0",
     "sanitize-html": "^1.11.1",
-    "serve-favicon": "~2.3.0",
-    "swig": "1.4.2",
+    "serve-favicon": "~2.4.2",
+    "serve-static": "^1.13.1",
     "slug": "0.9.1",
+    "swig": "1.4.2",
     "underscore": "1.8.3",
-    "weak": "1.0.0",
-    "ws": "0.7.2"
+    "validator": "7.0.0",
+    "weak": "1.0.1",
+    "ws": "2.2.3"
   },
   "devDependencies": {
     "express": "^4.13.3",
-    "gulp": "^3.9.0",
+    "gulp": "^3.9.1",
     "gulp-clean": "^0.3.2",
     "gulp-concat": "^2.6.0",
-    "gulp-eslint": "^1.1.0",
     "gulp-express": "^0.3.0",
     "gulp-fingerprint": "^0.3.2",
     "gulp-nodemon": "^2.0.4",
-    "gulp-rev": "^6.0.1",
-    "gulp-rev-all": "^0.8.22",
+    "gulp-rev": "^7.1.2",
+    "gulp-rev-all": "^0.9.7",
     "gulp-rev-replace": "^0.4.3",
-    "gulp-sass": "^2.1.0",
-    "gulp-uglify": "^1.4.2",
-    "mocha": "*",
-    "nodemon": "*",
-    "should": "^7.1.0",
-    "supertest": "^1.1.0",
-    "winston": "^1.0.1"
+    "gulp-sass": "^3.1.0",
+    "gulp-uglify": "^2.1.2",
+    "nodemon": "1.11.0",
+    "should": "^11.2.1",
+    "supertest": "^3.0.0",
+    "winston": "^2.3.1"
   },
   "description": "",
   "main": "Gulpfile.js",
-  "directories": {
-  },
+  "directories": {},
   "repository": {
     "type": "git",
     "url": "https://github.com/spacedeck/spacedeck-open.git"

routes/api/sessions.js

@@ -3,7 +3,7 @@
 var config = require('config');
 require('../../models/schema');
 
-var bcrypt = require('bcrypt');
+var bcrypt = require('bcryptjs');
 var crypo = require('crypto');
 
 var express = require('express');

routes/api/teams.js

@@ -9,7 +9,7 @@ var mailer = require('../../helpers/mailer');
 var fs = require('fs');
 var _ = require('underscore');
 var crypto = require('crypto');
-var bcrypt = require('bcrypt');
+var bcrypt = require('bcryptjs');
 
 var express = require('express');
 var router = express.Router();

routes/api/users.js

@@ -6,7 +6,7 @@ require('../../models/schema');
 var mailer = require('../../helpers/mailer');
 var uploader = require('../../helpers/uploader');
 
-var bcrypt = require('bcrypt');
+var bcrypt = require('bcryptjs');
 var crypo = require('crypto');
 var swig = require('swig');
 var async = require('async');