From a3e2129b79d72f57edc564e08763e1b0ecc7660c Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 10:50:48 +0200 Subject: [PATCH 01/14] added experiemental docker support --- Dockerfile | 15 +++++++++++++++ README.md | 13 ++++++++++--- docker-compose.yml | 27 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb8022d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM spacedeck/docker-baseimage:latest +ENV NODE_ENV production + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY package.json /usr/src/app/ +RUN npm install +COPY . /usr/src/app +RUN npm cache clean + +CMD [ "npm", "start" ] + +EXPOSE 9666 + diff --git a/README.md b/README.md index 03431ae..0f42318 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,14 @@ We appreciate filed issues, pull requests and general discussion. Spacedeck uses the following major building blocks: -- Node.js 4.x (Backend / API) +- Vue.js (Frontend) +- Node.js 7.x (Backend / API) - MongoDB 3.x (Datastore) - Redis 3.x (Datastore for realtime channels) -- Vue.js (Frontend) It also has some binary dependencies for media conversion and PDF export: -- imagemagick +- imagemagick, graphicsmagick, libav(+codecs, ffmpeg replacement), audiowaveform (https://github.com/bbcrd/audiowaveform), phantomjs (http://phantomjs.org/) Currently, media files are stored in Amazon S3, so you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails, Amazon SES is required. @@ -47,6 +47,13 @@ To rebuild the frontend CSS styles (you need to do this at least once): export NODE_ENV=development npm start +#experimental docker support + +We have a docker base image at https://github.com/spacedeck/docker-baseimage that includes all required binaries. Based on this image we can use Docker-Compose to bootstrap a Spacedeck including data storages. + +docker-compose build +docker-compose run -e ENV=development -p 9666:9666 -e NODE_ENV=development spacedeck + # License Spacedeck Open is released under the GNU Affero General Public License Version 3 (GNU AGPLv3). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..084118f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: '2' +services: + sync: + image: redis + storage: + image: minio/minio + environment: + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + ports: + - 9123:9000 + db: + image: mongo + spacedeck: + environment: + - env=development + build: . + volumes: + - ./:/usr/src/app + - /usr/src/app/node_modules + command: npm start + ports: + - 9666:9666 + depends_on: + - db + - sync + - storage \ No newline at end of file From f9cf8ba7e8ae8f54b2ba38a3d8fbe5283a84bd5c Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 11:55:07 +0200 Subject: [PATCH 02/14] update ports and variables for docker compose bootstrap --- README.md | 2 +- app.js | 4 ++-- config/default.json | 4 ++-- helpers/phantom.js | 47 ++++++++++++++++++++++++------------------- helpers/redis.js | 2 +- helpers/uploader.js | 9 +++++++++ helpers/websockets.js | 6 +++--- models/team.js | 2 +- views/spacedeck.html | 8 ++++---- 9 files changed, 49 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 0f42318..08ce9f9 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ To rebuild the frontend CSS styles (you need to do this at least once): export NODE_ENV=development npm start -#experimental docker support +# experimental docker support We have a docker base image at https://github.com/spacedeck/docker-baseimage that includes all required binaries. Based on this image we can use Docker-Compose to bootstrap a Spacedeck including data storages. diff --git a/app.js b/app.js index 24b4fb5..be1ffb9 100644 --- a/app.js +++ b/app.js @@ -121,11 +121,11 @@ if (app.get('env') == 'development') { module.exports = app; // CONNECT TO DATABASE -const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || 'localhost'; +const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || 'db'; mongoose.connect('mongodb://' + mongoHost + '/spacedeck'); // START WEBSERVER -const port = 9000; +const port = 9666; const server = http.Server(app).listen(port, () => { diff --git a/config/default.json b/config/default.json index 445fa23..51317c2 100644 --- a/config/default.json +++ b/config/default.json @@ -1,7 +1,7 @@ { - "endpoint": "http://localhost:9000", + "endpoint": "http://localhost:9666", "storage_bucket": "my_spacedeck_s3_bucket", - "storage_cdn": "xyz.cloudfront.net", + "storage_cdn": "http://localhost:9666", "google_access" : "", "google_secret" : "", "admin_pass": "very_secret_admin_password", diff --git a/helpers/phantom.js b/helpers/phantom.js index c195a52..2d6fa1d 100644 --- a/helpers/phantom.js +++ b/helpers/phantom.js @@ -32,31 +32,36 @@ module.exports = { }; phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) { - return browser.createPage(function (err, page) { - console.log("page created, opening ",space_url); + if(err){ + console.err(err); + }else{ + return browser.createPage(function (err, page) { + console.log("page created, opening ",space_url); - if (type=="pdf") { - var psz = { - width: space.advanced.width+"px", - height: space.advanced.height+"px" - }; - page.set('paperSize', psz); - } + if (type=="pdf") { + var psz = { + width: space.advanced.width+"px", + height: space.advanced.height+"px" + }; + page.set('paperSize', psz); + } - page.set('settings.resourceTimeout',timeout); - page.set('settings.javascriptEnabled',false); + page.set('settings.resourceTimeout',timeout); + page.set('settings.javascriptEnabled',false); - return page.open(space_url, function (err,status) { - page.render(export_path, function() { - on_success_called = true; - if (on_success) { - on_success(export_path); - } - page.close(); - browser.exit(); + return page.open(space_url, function (err,status) { + page.render(export_path, function() { + on_success_called = true; + if (on_success) { + on_success(export_path); + } + page.close(); + browser.exit(); + }); }); - }); - }); + }); + } + }, { onExit: on_exit }); diff --git a/helpers/redis.js b/helpers/redis.js index 90b440a..0198d38 100644 --- a/helpers/redis.js +++ b/helpers/redis.js @@ -5,7 +5,7 @@ const websockets = require('./websockets'); module.exports = { connectRedis(){ - const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'; + const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'sync'; this.connection = new RedisConnection(6379, redisHost); }, sendMessage(action, model, attributes, channelId) { diff --git a/helpers/uploader.js b/helpers/uploader.js index c314062..95c2b32 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -6,6 +6,15 @@ AWS.config.region = 'eu-central-1'; var fs = require('fs'); var config = require('config'); +var cdn = config.get("storage_cdn") +var storage_endpoint = 'http://storage:9000'; +const bucketName = "sdeck-fresh-development"; +const ep = new AWS.Endpoint(storage_endpoint); +const s3 = new AWS.S3({ + endpoint: ep +}); + + module.exports = { removeFile: (path, callback) => { const s3 = new AWS.S3({ diff --git a/helpers/websockets.js b/helpers/websockets.js index 26fcd1c..48a8222 100644 --- a/helpers/websockets.js +++ b/helpers/websockets.js @@ -3,7 +3,7 @@ require('../models/schema'); const WebSocketServer = require('ws').Server; -const Redis = require('ioredis'); +const RedisConnection = require('ioredis'); const async = require('async'); const _ = require("underscore"); const mongoose = require("mongoose"); @@ -12,7 +12,7 @@ const crypto = require('crypto'); module.exports = { startWebsockets: function(server){ this.setupSubscription(); - this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); + this.state = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'sync'); if(!this.current_websockets){ this.current_websockets = []; @@ -117,7 +117,7 @@ module.exports = { }, setupSubscription: function() { - this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); + this.cursorSubscriber = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'sync'); this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) { console.log("[redis] websockets to " + count + " topics." ); }); diff --git a/models/team.js b/models/team.js index a1f3cc7..b35942c 100644 --- a/models/team.js +++ b/models/team.js @@ -45,7 +45,7 @@ module.exports.teamSchema.index({ module.exports.teamSchema.statics.getTeamForHost = (host, cb) => { - if (host != "127.0.0.1:9000") { //phantomjs check + if (host != "127.0.0.1:9666") { //phantomjs check let subDomainParts = host.split('.'); if (subDomainParts.length > 2) { diff --git a/views/spacedeck.html b/views/spacedeck.html index 54144dd..bc93fbd 100644 --- a/views/spacedeck.html +++ b/views/spacedeck.html @@ -23,10 +23,10 @@ {% if process.env.NODE_ENV != "production" %} var ENV = { name: 'development', - webHost: "localhost:9000", - webEndpoint:"http://localhost:9000", - apiEndpoint: "http://localhost:9000", - websocketsEndpoint: "ws://localhost:9000" + webHost: "localhost:9666", + webEndpoint:"http://localhost:9666", + apiEndpoint: "http://localhost:9666", + websocketsEndpoint: "ws://localhost:9666" }; {% endif %} From af5335025f53afe0c2609bff483b5f4b256f8d93 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 11:58:29 +0200 Subject: [PATCH 03/14] fixed crash on console --- helpers/phantom.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/phantom.js b/helpers/phantom.js index 2d6fa1d..ed897f5 100644 --- a/helpers/phantom.js +++ b/helpers/phantom.js @@ -33,7 +33,7 @@ module.exports = { phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) { if(err){ - console.err(err); + console.log(err); }else{ return browser.createPage(function (err, page) { console.log("page created, opening ",space_url); From bd0471dad6196e483bcdf61b400f39ec6a89e861 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 12:03:58 +0200 Subject: [PATCH 04/14] added docker ignore --- .dockerignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..979bf6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.DS_Store +.git +logs +*.log +scripts +pids +*.pid +*.seed +lib-cov +coverage +.grunt +.lock-wscript +build/Release +node_modules \ No newline at end of file From 1426bc9c24e9cea5848eccd3245f916ad8d65130 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 13:07:09 +0200 Subject: [PATCH 05/14] updated storage config --- config/default.json | 5 +++-- docker-compose.yml | 9 ++++++++- helpers/uploader.js | 31 ++++++++++++++++++++----------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/config/default.json b/config/default.json index 51317c2..dc18811 100644 --- a/config/default.json +++ b/config/default.json @@ -1,7 +1,8 @@ { "endpoint": "http://localhost:9666", - "storage_bucket": "my_spacedeck_s3_bucket", - "storage_cdn": "http://localhost:9666", + "storage_bucket": "sdeck-development", + "storage_cdn": "http://localhost:9123/sdeck-development", + "storage_endpoint": "http://storage:9000", "google_access" : "", "google_secret" : "", "admin_pass": "very_secret_admin_password", diff --git a/docker-compose.yml b/docker-compose.yml index 5584bd9..4d1bf00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,11 +9,14 @@ services: - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ports: - 9123:9000 + command: server /export db: image: mongo spacedeck-open: environment: - env=development + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY build: . volumes: - ./:/usr/src/app @@ -24,4 +27,8 @@ services: depends_on: - db - sync - - storage \ No newline at end of file + - storage + links: + - storage + - db + - sync \ No newline at end of file diff --git a/helpers/uploader.js b/helpers/uploader.js index 95c2b32..62f64d7 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -6,20 +6,29 @@ AWS.config.region = 'eu-central-1'; var fs = require('fs'); var config = require('config'); -var cdn = config.get("storage_cdn") -var storage_endpoint = 'http://storage:9000'; +var cdn = config.get("storage_cdn"); +var storage_endpoint = config.get("storage_endpoint"); + const bucketName = "sdeck-fresh-development"; const ep = new AWS.Endpoint(storage_endpoint); + +AWS.config.update(new AWS.Config({ + accessKeyId: process.env.MINIO_ACCESS_KEY, + secretAccessKey: process.env.MINIO_SECRET_KEY, + region: 'us-east-1', + s3ForcePathStyle: true, + signatureVersion: 'v4' +})); + const s3 = new AWS.S3({ endpoint: ep }); - module.exports = { removeFile: (path, callback) => { - const s3 = new AWS.S3({ - region: 'eu-central-1' - }); + // const s3 = new AWS.S3({ + // region: 'eu-central-1' + // }); const bucket = config.get("storage_bucket"); s3.deleteObject({ Bucket: bucket, Key: path @@ -37,7 +46,7 @@ module.exports = { callback({error:"missing path"}, null); return; } - console.log("[s3] uploading", localFilePath, " to ", fileName); + console.log("[storage] uploading", localFilePath, " to ", fileName); const bucket = config.get("storage_bucket"); const fileStream = fs.createReadStream(localFilePath); @@ -49,9 +58,9 @@ module.exports = { }); fileStream.on('open', function () { // FIXME - var s3 = new AWS.S3({ - region: 'eu-central-1' - }); + // var s3 = new AWS.S3({ + // region: 'eu-central-1' + // }); s3.putObject({ Bucket: bucket, @@ -63,7 +72,7 @@ module.exports = { console.error(err); callback(err); }else { - const url = "https://"+ config.get("storage_cdn") + "/" + fileName; + const url = cdn + "/" + fileName; console.log("[s3]" + localFilePath + " to " + url); callback(null, url); } From 03059b67f19a4b04b02c96bacd070b049a200378 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 15:34:33 +0200 Subject: [PATCH 06/14] add gulp stuff to build process --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index bb8022d..17a36a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,12 @@ WORKDIR /usr/src/app COPY package.json /usr/src/app/ RUN npm install +RUN npm install gulp-rev-replace gulp-clean gulp-fingerprint gulp-rev gulp-rev-all gulp-rev-replace +RUN npm install -g --save-dev gulp + COPY . /usr/src/app +RUN gulp styles + RUN npm cache clean CMD [ "npm", "start" ] From 820203625ced99356957c0c582fa8fa6234e2a61 Mon Sep 17 00:00:00 2001 From: "Lukas F. Hartmann" Date: Fri, 7 Apr 2017 18:01:23 +0200 Subject: [PATCH 07/14] don't default to app volume mount; create s3 bucket on boot; fix revAll gulp step --- Dockerfile | 1 - Gulpfile.js | 3 +-- app.js | 2 +- docker-compose.yml | 4 ++-- helpers/uploader.js | 16 ++++++++-------- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 17a36a2..5666914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ RUN npm install -g --save-dev gulp COPY . /usr/src/app RUN gulp styles - RUN npm cache clean CMD [ "npm", "start" ] diff --git a/Gulpfile.js b/Gulpfile.js index af47a5d..e4aa47e 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -12,10 +12,9 @@ var uglify = require('gulp-uglify'); var fingerprint = require('gulp-fingerprint'); var rev = require('gulp-rev'); -var RevAll = require('gulp-rev-all'); +var revAll = require('gulp-rev-all'); gulp.task('rev', () => { - var revAll = new RevAll(); return gulp.src(['public/**']) .pipe(gulp.dest('build/assets')) .pipe(revAll.revision()) diff --git a/app.js b/app.js index be1ffb9..433af65 100644 --- a/app.js +++ b/app.js @@ -47,7 +47,7 @@ swig.setFilter('cdn', function(input, idx) { app.engine('html', swig.renderFile); app.set('view engine', 'html'); -if (app.get('env') != 'development') { +if (isProduction) { app.set('views', path.join(__dirname, 'build', 'views')); app.use(favicon(path.join(__dirname, 'build', 'assets', 'images', 'favicon.png'))); app.use(express.static(path.join(__dirname, 'build', 'assets'))); diff --git a/docker-compose.yml b/docker-compose.yml index 4d1bf00..688f381 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY build: . volumes: - - ./:/usr/src/app + # - ./:/usr/src/app - /usr/src/app/node_modules command: npm start ports: @@ -31,4 +31,4 @@ services: links: - storage - db - - sync \ No newline at end of file + - sync diff --git a/helpers/uploader.js b/helpers/uploader.js index 62f64d7..b39b6ee 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -24,11 +24,16 @@ const s3 = new AWS.S3({ endpoint: ep }); +s3.createBucket({ + Bucket: config.get("storage_bucket"), + ACL: "public-read", + GrantRead: "*" +}, (err,res) => { + console.log("createBucket",err,res); +}); + module.exports = { removeFile: (path, callback) => { - // const s3 = new AWS.S3({ - // region: 'eu-central-1' - // }); const bucket = config.get("storage_bucket"); s3.deleteObject({ Bucket: bucket, Key: path @@ -57,11 +62,6 @@ module.exports = { } }); fileStream.on('open', function () { - // FIXME - // var s3 = new AWS.S3({ - // region: 'eu-central-1' - // }); - s3.putObject({ Bucket: bucket, Key: fileName, From 6ad97ac5c2167484dcda8407919b6aee292c1be2 Mon Sep 17 00:00:00 2001 From: mntmn Date: Sun, 7 Jan 2018 22:54:31 +0100 Subject: [PATCH 08/14] fix port number in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 73a2846..723f75f 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ see: config/config.json export NODE_ENV=development npm start - open http://localhost:90000 + open http://localhost:9000 # License From 7e8a27e140ddea52e2697028651bd1cfef4a9e36 Mon Sep 17 00:00:00 2001 From: mntmn Date: Mon, 8 Jan 2018 00:08:42 +0100 Subject: [PATCH 09/14] replace redis by in-memory object --- helpers/redis.js | 110 ++++++++++++++++++++++++++++++++++++++---- helpers/websockets.js | 23 ++++----- 2 files changed, 113 insertions(+), 20 deletions(-) diff --git a/helpers/redis.js b/helpers/redis.js index 90b440a..610742c 100644 --- a/helpers/redis.js +++ b/helpers/redis.js @@ -1,14 +1,103 @@ 'use strict'; -const RedisConnection = require('ioredis'); -const websockets = require('./websockets'); +var notRedis = { + state: {}, + topics: {}, + + publish: function(topic, msg, cb) { + //console.log("[notredis] publish",topic,msg); + if (!this.topics[topic]) { + this.topics[topic] = { + subscribers: [] + }; + } + var t=this.topics[topic]; + for (var i=0; i { cb(); }); }, - rateLimit(namespace, ip, cb) { + rateLimit: function(namespace, ip, cb) { const key = "limit_"+ namespace + "_"+ ip; const redis = this.connection; @@ -47,7 +136,7 @@ module.exports = { } }); }, - isOnlineInSpace(user, space, cb) { + isOnlineInSpace: function(user, space, cb) { this.connection.smembers("space_" + space._id.toString(), function(err, list) { if (err) cb(err); else { @@ -59,3 +148,6 @@ module.exports = { }); } }; + +return module.exports; + diff --git a/helpers/websockets.js b/helpers/websockets.js index 26fcd1c..9394913 100644 --- a/helpers/websockets.js +++ b/helpers/websockets.js @@ -3,18 +3,19 @@ require('../models/schema'); const WebSocketServer = require('ws').Server; -const Redis = require('ioredis'); const async = require('async'); const _ = require("underscore"); const mongoose = require("mongoose"); const crypto = require('crypto'); -module.exports = { - startWebsockets: function(server){ - this.setupSubscription(); - this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); +var redis = require("./redis.js"); - if(!this.current_websockets){ +module.exports = { + startWebsockets: function(server) { + this.setupSubscription(); + this.state = redis.getConnection(); + + if(!this.current_websockets) { this.current_websockets = []; } @@ -117,8 +118,7 @@ module.exports = { }, setupSubscription: function() { - this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); - this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) { + this.cursorSubscriber = redis.getConnection().subscribe(['cursors', 'users', 'updates'], function (err, count) { console.log("[redis] websockets to " + count + " topics." ); }); this.cursorSubscriber.on('message', function (channel, rawMessage) { @@ -206,7 +206,7 @@ module.exports = { console.log("websocket not found to remove"); } - this.state.del(ws.id, function(err, res) { + this.state.del(ws.id+"", function(err, res) { if (err) console.error(err, res); else { this.removeUserInSpace(ws.space_id, ws, (err) => { @@ -221,7 +221,8 @@ module.exports = { addUserInSpace: function(username, space, ws, cb) { console.log("[websockets] user "+username+" in "+space.access_mode +" space " + space._id + " with socket " + ws.id); - this.state.set(ws.id, username, function(err, res) { + + this.state.set(ws.id+"", username+"", function(err, res) { if(err) console.error(err, res); else { this.state.sadd("space_" + space._id, ws.id, function(err, res) { @@ -238,7 +239,7 @@ module.exports = { }.bind(this)); }, removeUserInSpace: function(spaceId, ws, cb) { - this.state.srem("space_" + spaceId, ws.id, function(err, res) { + this.state.srem("space_" + spaceId, ws.id+"", function(err, res) { if (err) cb(err); else { console.log("[websockets] socket "+ ws.id + " went offline in space " + spaceId); From efb7970ecb35041892dc5578a59400144d28df9d Mon Sep 17 00:00:00 2001 From: mntmn Date: Mon, 8 Jan 2018 12:40:11 +0100 Subject: [PATCH 10/14] fix express caching --- app.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app.js b/app.js index 24b4fb5..a6e67e1 100644 --- a/app.js +++ b/app.js @@ -15,13 +15,16 @@ const favicon = require('serve-favicon'); const logger = require('morgan'); const cookieParser = require('cookie-parser'); const bodyParser = require('body-parser'); + const mongoose = require('mongoose'); + const swig = require('swig'); const i18n = require('i18n-2'); const helmet = require('helmet'); const express = require('express'); const app = express(); +const serveStatic = require('serve-static'); const isProduction = app.get('env') === 'production'; @@ -67,7 +70,6 @@ app.use(bodyParser.urlencoded({ })); app.use(cookieParser()); -app.use(helmet.noCache()) app.use(helmet.frameguard()) app.use(helmet.xssFilter()) app.use(helmet.hsts({ @@ -109,6 +111,12 @@ app.use('/api/teams', require('./routes/api/teams')); app.use('/api/webgrabber', require('./routes/api/webgrabber')); app.use('/', require('./routes/root')); +if (config.get('storage_local_path')) { + app.use('/storage', serveStatic(config.get('storage_local_path')+"/"+config.get('storage_bucket'), { + maxAge: 24*3600 + })); +} + // catch 404 and forward to error handler app.use(require('./middlewares/404')); if (app.get('env') == 'development') { @@ -128,7 +136,7 @@ mongoose.connect('mongodb://' + mongoHost + '/spacedeck'); const port = 9000; const server = http.Server(app).listen(port, () => { - + if ("send" in process) { process.send('online'); } From 0d92343d55cf35ef3f718ad7a07f0c8591759bc1 Mon Sep 17 00:00:00 2001 From: mntmn Date: Mon, 8 Jan 2018 12:41:12 +0100 Subject: [PATCH 11/14] fix redis mock incr() --- helpers/redis.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/helpers/redis.js b/helpers/redis.js index 610742c..9866ba4 100644 --- a/helpers/redis.js +++ b/helpers/redis.js @@ -1,11 +1,12 @@ 'use strict'; +// this is a mock version of the Redis API, +// emulating Redis if it is not available locally var notRedis = { state: {}, topics: {}, publish: function(topic, msg, cb) { - //console.log("[notredis] publish",topic,msg); if (!this.topics[topic]) { this.topics[topic] = { subscribers: [] @@ -43,7 +44,7 @@ var notRedis = { t.subscribers.push(handle); } - cb(null, handle, topics.length); + cb(null, topics.length); return handle; }, @@ -82,7 +83,7 @@ var notRedis = { incr: function(key, cb) { if (!this.state[key]) this.state[key] = 0; this.state[key]++; - cb(); + cb(null, this.state[key]); }, expire: function() { From 8ee338647003e870ffc986b7e99e1674a149ede0 Mon Sep 17 00:00:00 2001 From: mntmn Date: Mon, 8 Jan 2018 12:41:38 +0100 Subject: [PATCH 12/14] fix tmp dir in artifact uploader --- helpers/artifact_converter.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/artifact_converter.js b/helpers/artifact_converter.js index 3605a90..d6f468d 100644 --- a/helpers/artifact_converter.js +++ b/helpers/artifact_converter.js @@ -7,6 +7,7 @@ const fs = require('fs'); const Models = require('../models/schema'); const uploader = require('../helpers/uploader'); const path = require('path'); +const os = require('os'); const fileExtensionMap = { ".amr" : "audio/AMR", @@ -245,13 +246,12 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) { if (max>320 || size.width > max || size.height > max) { var resizedFileName = max + "_"+fileName; var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + resizedFileName; - var localResizedFilePath = "/tmp/"+resizedFileName; + var localResizedFilePath = os.tmpdir()+"/"+resizedFileName; gm(localFilePath).resize(max, max).autoOrient().write(localResizedFilePath, function (err) { if(!err) { uploader.uploadFile(s3Key, "image/jpeg", localResizedFilePath, function(err, url) { if (err) callback(err); else{ - console.log(localResizedFilePath); fs.unlink(localResizedFilePath, function (err) { if (err) { console.error(err); From ffb7f301339e1ec7c1a06ee1d93585cf1ef53371 Mon Sep 17 00:00:00 2001 From: mntmn Date: Mon, 8 Jan 2018 12:43:47 +0100 Subject: [PATCH 13/14] switch between s3/local storage in config --- config/default.json | 6 ++++-- helpers/uploader.js | 19 ++++++++++++------- package.json | 17 ++++++++--------- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/config/default.json b/config/default.json index 445fa23..6fd9f66 100644 --- a/config/default.json +++ b/config/default.json @@ -1,7 +1,9 @@ { "endpoint": "http://localhost:9000", - "storage_bucket": "my_spacedeck_s3_bucket", - "storage_cdn": "xyz.cloudfront.net", + "storage_region": "eu-central-1", + "storage_bucket": "my_spacedeck_bucket", + "storage_cdn": "/storage", + "storage_local_path": "./storage", "google_access" : "", "google_secret" : "", "admin_pass": "very_secret_admin_password", diff --git a/helpers/uploader.js b/helpers/uploader.js index c314062..7fe1df1 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -1,15 +1,21 @@ 'use strict'; -var AWS = require('aws-sdk'); -AWS.config.region = 'eu-central-1'; - var fs = require('fs'); var config = require('config'); +// use AWS S3 or local folder depending on config +if (config.get("storage_local_path")) { + var AWS = require('mock-aws-s3'); + AWS.config.basePath = config.get("storage_local_path"); +} else { + var AWS = require('aws-sdk'); + AWS.config.region = config.get("storage_region"); +} + module.exports = { removeFile: (path, callback) => { const s3 = new AWS.S3({ - region: 'eu-central-1' + region: config.get("storage_region") }); const bucket = config.get("storage_bucket"); s3.deleteObject({ @@ -39,9 +45,8 @@ module.exports = { } }); fileStream.on('open', function () { - // FIXME var s3 = new AWS.S3({ - region: 'eu-central-1' + region: config.get("storage_region") }); s3.putObject({ @@ -54,7 +59,7 @@ module.exports = { console.error(err); callback(err); }else { - const url = "https://"+ config.get("storage_cdn") + "/" + fileName; + const url = config.get("storage_cdn") + "/" + fileName; console.log("[s3]" + localFilePath + " to " + url); callback(null, url); } diff --git a/package.json b/package.json index 78b7025..e6b38d1 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "start": "nodemon -e .js,.html bin/www", "test": "mocha" }, - "engines" : { - "node" : ">=7.8.0" + "engines": { + "node": ">=7.8.0" }, "dependencies": { "archiver": "1.3.0", @@ -28,7 +28,6 @@ "googleapis": "18.0.0", "gulp": "^3.9.1", "gulp-concat": "2.6.0", - "gulp-eslint": "*", "gulp-express": "0.3.0", "gulp-nodemon": "*", "gulp-sass": "^2.0.3", @@ -40,13 +39,13 @@ "lodash": "^4.3.0", "log-timestamp": "latest", "md5": "2.2.1", + "mock-aws-s3": "^2.6.0", "moment": "2.18.1", "mongoose": "4.9.3", "morgan": "1.8.1", + "node-phantom-simple": "2.2.4", "node-sass-middleware": "0.11.0", "pdfkit": "0.8.0", - "validator": "7.0.0", - "node-phantom-simple": "2.2.4", "phantomjs-prebuilt": "2.1.14", "pm2": "latest", "qr-image": "3.2.0", @@ -54,9 +53,11 @@ "request": "2.81.0", "sanitize-html": "^1.11.1", "serve-favicon": "~2.4.2", - "swig": "1.4.2", + "serve-static": "^1.13.1", "slug": "0.9.1", + "swig": "1.4.2", "underscore": "1.8.3", + "validator": "7.0.0", "weak": "1.0.1", "ws": "2.2.3" }, @@ -65,7 +66,6 @@ "gulp": "^3.9.1", "gulp-clean": "^0.3.2", "gulp-concat": "^2.6.0", - "gulp-eslint": "^3.0.1", "gulp-express": "^0.3.0", "gulp-fingerprint": "^0.3.2", "gulp-nodemon": "^2.0.4", @@ -81,8 +81,7 @@ }, "description": "", "main": "Gulpfile.js", - "directories": { - }, + "directories": {}, "repository": { "type": "git", "url": "https://github.com/spacedeck/spacedeck-open.git" From 462e9edaab7e1bd0354dca673565853ec3755f63 Mon Sep 17 00:00:00 2001 From: mntmn Date: Mon, 8 Jan 2018 15:57:59 +0100 Subject: [PATCH 14/14] first importer WIP --- helpers/importer.js | 103 ++++++++++++++++++++++++++++++++++ package.json | 1 + routes/api/space_artifacts.js | 4 +- routes/api/users.js | 10 ++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 helpers/importer.js diff --git a/helpers/importer.js b/helpers/importer.js new file mode 100644 index 0000000..90eb2a9 --- /dev/null +++ b/helpers/importer.js @@ -0,0 +1,103 @@ +'use strict'; + +const extract = require('extract-zip') +const config = require('config') +const fs = require('fs') +const path = require('path') + +require('../models/schema') + +module.exports = { + importZIP: function(user, zipPath) { + + // 1. extract zip to local storage folder + // 2. read spaces.json from this folder + // 3. iterate through spaces and read all their artifact jsons + // 4. fixup storage paths + // 5. replace creator id by user._id + + let relativeImportDir = 'import_'+user._id + let importDir = path.resolve(config.get('storage_local_path')+'/'+config.get('storage_bucket')+'/'+relativeImportDir) + + if (!fs.existsSync(importDir)) { + fs.mkdirSync(importDir) + } + + extract(zipPath, {dir: importDir}, function(err) { + if (err) { + console.log(err) + return + } + console.log('[import] extracted to',importDir) + + let spacesJson = fs.readFileSync(importDir+'/spaces.json') + let spaces = JSON.parse(spacesJson) + var homeFolderId = null + + console.log('[import] spaces:',spaces.length) + + // pass 1: find homefolder + for (var i=0; i { "nickname": 1, "email": 1 }).exec((err, user) => { - a['user'] = user.toObject(); + if (user) { + a['user'] = user.toObject(); + } cb(err, a); }); } else { diff --git a/routes/api/users.js b/routes/api/users.js index 8140bcc..d8e995c 100644 --- a/routes/api/users.js +++ b/routes/api/users.js @@ -5,6 +5,7 @@ require('../../models/schema'); var mailer = require('../../helpers/mailer'); var uploader = require('../../helpers/uploader'); +var importer = require('../../helpers/importer'); var bcrypt = require('bcryptjs'); var crypo = require('crypto'); @@ -467,4 +468,13 @@ router.post('/:user_id/confirm', function(req, res, next) { res.sendStatus(201); }); +router.get('/:user_id/import', function(req, res, next) { + if (req.query.zip) { + res.send("importing"); + importer.importZIP(req.user, req.query.zip); + } else { + res.sendStatus(400); + } +}); + module.exports = router;