From a3e2129b79d72f57edc564e08763e1b0ecc7660c Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 10:50:48 +0200 Subject: [PATCH 1/7] added experiemental docker support --- Dockerfile | 15 +++++++++++++++ README.md | 13 ++++++++++--- docker-compose.yml | 27 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bb8022d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM spacedeck/docker-baseimage:latest +ENV NODE_ENV production + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY package.json /usr/src/app/ +RUN npm install +COPY . /usr/src/app +RUN npm cache clean + +CMD [ "npm", "start" ] + +EXPOSE 9666 + diff --git a/README.md b/README.md index 03431ae..0f42318 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,14 @@ We appreciate filed issues, pull requests and general discussion. Spacedeck uses the following major building blocks: -- Node.js 4.x (Backend / API) +- Vue.js (Frontend) +- Node.js 7.x (Backend / API) - MongoDB 3.x (Datastore) - Redis 3.x (Datastore for realtime channels) -- Vue.js (Frontend) It also has some binary dependencies for media conversion and PDF export: -- imagemagick +- imagemagick, graphicsmagick, libav(+codecs, ffmpeg replacement), audiowaveform (https://github.com/bbcrd/audiowaveform), phantomjs (http://phantomjs.org/) Currently, media files are stored in Amazon S3, so you need an Amazon AWS account and have the ```AWS_ACCESS_KEY_ID``` and ```AWS_SECRET_ACCESS_KEY``` environment variables defined. For sending emails, Amazon SES is required. @@ -47,6 +47,13 @@ To rebuild the frontend CSS styles (you need to do this at least once): export NODE_ENV=development npm start +#experimental docker support + +We have a docker base image at https://github.com/spacedeck/docker-baseimage that includes all required binaries. Based on this image we can use Docker-Compose to bootstrap a Spacedeck including data storages. + +docker-compose build +docker-compose run -e ENV=development -p 9666:9666 -e NODE_ENV=development spacedeck + # License Spacedeck Open is released under the GNU Affero General Public License Version 3 (GNU AGPLv3). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..084118f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +version: '2' +services: + sync: + image: redis + storage: + image: minio/minio + environment: + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + ports: + - 9123:9000 + db: + image: mongo + spacedeck: + environment: + - env=development + build: . + volumes: + - ./:/usr/src/app + - /usr/src/app/node_modules + command: npm start + ports: + - 9666:9666 + depends_on: + - db + - sync + - storage \ No newline at end of file From f9cf8ba7e8ae8f54b2ba38a3d8fbe5283a84bd5c Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 11:55:07 +0200 Subject: [PATCH 2/7] update ports and variables for docker compose bootstrap --- README.md | 2 +- app.js | 4 ++-- config/default.json | 4 ++-- helpers/phantom.js | 47 ++++++++++++++++++++++++------------------- helpers/redis.js | 2 +- helpers/uploader.js | 9 +++++++++ helpers/websockets.js | 6 +++--- models/team.js | 2 +- views/spacedeck.html | 8 ++++---- 9 files changed, 49 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 0f42318..08ce9f9 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ To rebuild the frontend CSS styles (you need to do this at least once): export NODE_ENV=development npm start -#experimental docker support +# experimental docker support We have a docker base image at https://github.com/spacedeck/docker-baseimage that includes all required binaries. Based on this image we can use Docker-Compose to bootstrap a Spacedeck including data storages. diff --git a/app.js b/app.js index 24b4fb5..be1ffb9 100644 --- a/app.js +++ b/app.js @@ -121,11 +121,11 @@ if (app.get('env') == 'development') { module.exports = app; // CONNECT TO DATABASE -const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || 'localhost'; +const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || 'db'; mongoose.connect('mongodb://' + mongoHost + '/spacedeck'); // START WEBSERVER -const port = 9000; +const port = 9666; const server = http.Server(app).listen(port, () => { diff --git a/config/default.json b/config/default.json index 445fa23..51317c2 100644 --- a/config/default.json +++ b/config/default.json @@ -1,7 +1,7 @@ { - "endpoint": "http://localhost:9000", + "endpoint": "http://localhost:9666", "storage_bucket": "my_spacedeck_s3_bucket", - "storage_cdn": "xyz.cloudfront.net", + "storage_cdn": "http://localhost:9666", "google_access" : "", "google_secret" : "", "admin_pass": "very_secret_admin_password", diff --git a/helpers/phantom.js b/helpers/phantom.js index c195a52..2d6fa1d 100644 --- a/helpers/phantom.js +++ b/helpers/phantom.js @@ -32,31 +32,36 @@ module.exports = { }; phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) { - return browser.createPage(function (err, page) { - console.log("page created, opening ",space_url); + if(err){ + console.err(err); + }else{ + return browser.createPage(function (err, page) { + console.log("page created, opening ",space_url); - if (type=="pdf") { - var psz = { - width: space.advanced.width+"px", - height: space.advanced.height+"px" - }; - page.set('paperSize', psz); - } + if (type=="pdf") { + var psz = { + width: space.advanced.width+"px", + height: space.advanced.height+"px" + }; + page.set('paperSize', psz); + } - page.set('settings.resourceTimeout',timeout); - page.set('settings.javascriptEnabled',false); + page.set('settings.resourceTimeout',timeout); + page.set('settings.javascriptEnabled',false); - return page.open(space_url, function (err,status) { - page.render(export_path, function() { - on_success_called = true; - if (on_success) { - on_success(export_path); - } - page.close(); - browser.exit(); + return page.open(space_url, function (err,status) { + page.render(export_path, function() { + on_success_called = true; + if (on_success) { + on_success(export_path); + } + page.close(); + browser.exit(); + }); }); - }); - }); + }); + } + }, { onExit: on_exit }); diff --git a/helpers/redis.js b/helpers/redis.js index 90b440a..0198d38 100644 --- a/helpers/redis.js +++ b/helpers/redis.js @@ -5,7 +5,7 @@ const websockets = require('./websockets'); module.exports = { connectRedis(){ - const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'; + const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'sync'; this.connection = new RedisConnection(6379, redisHost); }, sendMessage(action, model, attributes, channelId) { diff --git a/helpers/uploader.js b/helpers/uploader.js index c314062..95c2b32 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -6,6 +6,15 @@ AWS.config.region = 'eu-central-1'; var fs = require('fs'); var config = require('config'); +var cdn = config.get("storage_cdn") +var storage_endpoint = 'http://storage:9000'; +const bucketName = "sdeck-fresh-development"; +const ep = new AWS.Endpoint(storage_endpoint); +const s3 = new AWS.S3({ + endpoint: ep +}); + + module.exports = { removeFile: (path, callback) => { const s3 = new AWS.S3({ diff --git a/helpers/websockets.js b/helpers/websockets.js index 26fcd1c..48a8222 100644 --- a/helpers/websockets.js +++ b/helpers/websockets.js @@ -3,7 +3,7 @@ require('../models/schema'); const WebSocketServer = require('ws').Server; -const Redis = require('ioredis'); +const RedisConnection = require('ioredis'); const async = require('async'); const _ = require("underscore"); const mongoose = require("mongoose"); @@ -12,7 +12,7 @@ const crypto = require('crypto'); module.exports = { startWebsockets: function(server){ this.setupSubscription(); - this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); + this.state = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'sync'); if(!this.current_websockets){ this.current_websockets = []; @@ -117,7 +117,7 @@ module.exports = { }, setupSubscription: function() { - this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); + this.cursorSubscriber = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'sync'); this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) { console.log("[redis] websockets to " + count + " topics." ); }); diff --git a/models/team.js b/models/team.js index a1f3cc7..b35942c 100644 --- a/models/team.js +++ b/models/team.js @@ -45,7 +45,7 @@ module.exports.teamSchema.index({ module.exports.teamSchema.statics.getTeamForHost = (host, cb) => { - if (host != "127.0.0.1:9000") { //phantomjs check + if (host != "127.0.0.1:9666") { //phantomjs check let subDomainParts = host.split('.'); if (subDomainParts.length > 2) { diff --git a/views/spacedeck.html b/views/spacedeck.html index 54144dd..bc93fbd 100644 --- a/views/spacedeck.html +++ b/views/spacedeck.html @@ -23,10 +23,10 @@ {% if process.env.NODE_ENV != "production" %} var ENV = { name: 'development', - webHost: "localhost:9000", - webEndpoint:"http://localhost:9000", - apiEndpoint: "http://localhost:9000", - websocketsEndpoint: "ws://localhost:9000" + webHost: "localhost:9666", + webEndpoint:"http://localhost:9666", + apiEndpoint: "http://localhost:9666", + websocketsEndpoint: "ws://localhost:9666" }; {% endif %} From af5335025f53afe0c2609bff483b5f4b256f8d93 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 11:58:29 +0200 Subject: [PATCH 3/7] fixed crash on console --- helpers/phantom.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/phantom.js b/helpers/phantom.js index 2d6fa1d..ed897f5 100644 --- a/helpers/phantom.js +++ b/helpers/phantom.js @@ -33,7 +33,7 @@ module.exports = { phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) { if(err){ - console.err(err); + console.log(err); }else{ return browser.createPage(function (err, page) { console.log("page created, opening ",space_url); From bd0471dad6196e483bcdf61b400f39ec6a89e861 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 12:03:58 +0200 Subject: [PATCH 4/7] added docker ignore --- .dockerignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..979bf6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.DS_Store +.git +logs +*.log +scripts +pids +*.pid +*.seed +lib-cov +coverage +.grunt +.lock-wscript +build/Release +node_modules \ No newline at end of file From 1426bc9c24e9cea5848eccd3245f916ad8d65130 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 13:07:09 +0200 Subject: [PATCH 5/7] updated storage config --- config/default.json | 5 +++-- docker-compose.yml | 9 ++++++++- helpers/uploader.js | 31 ++++++++++++++++++++----------- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/config/default.json b/config/default.json index 51317c2..dc18811 100644 --- a/config/default.json +++ b/config/default.json @@ -1,7 +1,8 @@ { "endpoint": "http://localhost:9666", - "storage_bucket": "my_spacedeck_s3_bucket", - "storage_cdn": "http://localhost:9666", + "storage_bucket": "sdeck-development", + "storage_cdn": "http://localhost:9123/sdeck-development", + "storage_endpoint": "http://storage:9000", "google_access" : "", "google_secret" : "", "admin_pass": "very_secret_admin_password", diff --git a/docker-compose.yml b/docker-compose.yml index 5584bd9..4d1bf00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,11 +9,14 @@ services: - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY ports: - 9123:9000 + command: server /export db: image: mongo spacedeck-open: environment: - env=development + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY build: . volumes: - ./:/usr/src/app @@ -24,4 +27,8 @@ services: depends_on: - db - sync - - storage \ No newline at end of file + - storage + links: + - storage + - db + - sync \ No newline at end of file diff --git a/helpers/uploader.js b/helpers/uploader.js index 95c2b32..62f64d7 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -6,20 +6,29 @@ AWS.config.region = 'eu-central-1'; var fs = require('fs'); var config = require('config'); -var cdn = config.get("storage_cdn") -var storage_endpoint = 'http://storage:9000'; +var cdn = config.get("storage_cdn"); +var storage_endpoint = config.get("storage_endpoint"); + const bucketName = "sdeck-fresh-development"; const ep = new AWS.Endpoint(storage_endpoint); + +AWS.config.update(new AWS.Config({ + accessKeyId: process.env.MINIO_ACCESS_KEY, + secretAccessKey: process.env.MINIO_SECRET_KEY, + region: 'us-east-1', + s3ForcePathStyle: true, + signatureVersion: 'v4' +})); + const s3 = new AWS.S3({ endpoint: ep }); - module.exports = { removeFile: (path, callback) => { - const s3 = new AWS.S3({ - region: 'eu-central-1' - }); + // const s3 = new AWS.S3({ + // region: 'eu-central-1' + // }); const bucket = config.get("storage_bucket"); s3.deleteObject({ Bucket: bucket, Key: path @@ -37,7 +46,7 @@ module.exports = { callback({error:"missing path"}, null); return; } - console.log("[s3] uploading", localFilePath, " to ", fileName); + console.log("[storage] uploading", localFilePath, " to ", fileName); const bucket = config.get("storage_bucket"); const fileStream = fs.createReadStream(localFilePath); @@ -49,9 +58,9 @@ module.exports = { }); fileStream.on('open', function () { // FIXME - var s3 = new AWS.S3({ - region: 'eu-central-1' - }); + // var s3 = new AWS.S3({ + // region: 'eu-central-1' + // }); s3.putObject({ Bucket: bucket, @@ -63,7 +72,7 @@ module.exports = { console.error(err); callback(err); }else { - const url = "https://"+ config.get("storage_cdn") + "/" + fileName; + const url = cdn + "/" + fileName; console.log("[s3]" + localFilePath + " to " + url); callback(null, url); } From 03059b67f19a4b04b02c96bacd070b049a200378 Mon Sep 17 00:00:00 2001 From: Martin Guether Date: Fri, 7 Apr 2017 15:34:33 +0200 Subject: [PATCH 6/7] add gulp stuff to build process --- Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Dockerfile b/Dockerfile index bb8022d..17a36a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,12 @@ WORKDIR /usr/src/app COPY package.json /usr/src/app/ RUN npm install +RUN npm install gulp-rev-replace gulp-clean gulp-fingerprint gulp-rev gulp-rev-all gulp-rev-replace +RUN npm install -g --save-dev gulp + COPY . /usr/src/app +RUN gulp styles + RUN npm cache clean CMD [ "npm", "start" ] From 820203625ced99356957c0c582fa8fa6234e2a61 Mon Sep 17 00:00:00 2001 From: "Lukas F. Hartmann" Date: Fri, 7 Apr 2017 18:01:23 +0200 Subject: [PATCH 7/7] don't default to app volume mount; create s3 bucket on boot; fix revAll gulp step --- Dockerfile | 1 - Gulpfile.js | 3 +-- app.js | 2 +- docker-compose.yml | 4 ++-- helpers/uploader.js | 16 ++++++++-------- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 17a36a2..5666914 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ RUN npm install -g --save-dev gulp COPY . /usr/src/app RUN gulp styles - RUN npm cache clean CMD [ "npm", "start" ] diff --git a/Gulpfile.js b/Gulpfile.js index af47a5d..e4aa47e 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -12,10 +12,9 @@ var uglify = require('gulp-uglify'); var fingerprint = require('gulp-fingerprint'); var rev = require('gulp-rev'); -var RevAll = require('gulp-rev-all'); +var revAll = require('gulp-rev-all'); gulp.task('rev', () => { - var revAll = new RevAll(); return gulp.src(['public/**']) .pipe(gulp.dest('build/assets')) .pipe(revAll.revision()) diff --git a/app.js b/app.js index be1ffb9..433af65 100644 --- a/app.js +++ b/app.js @@ -47,7 +47,7 @@ swig.setFilter('cdn', function(input, idx) { app.engine('html', swig.renderFile); app.set('view engine', 'html'); -if (app.get('env') != 'development') { +if (isProduction) { app.set('views', path.join(__dirname, 'build', 'views')); app.use(favicon(path.join(__dirname, 'build', 'assets', 'images', 'favicon.png'))); app.use(express.static(path.join(__dirname, 'build', 'assets'))); diff --git a/docker-compose.yml b/docker-compose.yml index 4d1bf00..688f381 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY build: . volumes: - - ./:/usr/src/app + # - ./:/usr/src/app - /usr/src/app/node_modules command: npm start ports: @@ -31,4 +31,4 @@ services: links: - storage - db - - sync \ No newline at end of file + - sync diff --git a/helpers/uploader.js b/helpers/uploader.js index 62f64d7..b39b6ee 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -24,11 +24,16 @@ const s3 = new AWS.S3({ endpoint: ep }); +s3.createBucket({ + Bucket: config.get("storage_bucket"), + ACL: "public-read", + GrantRead: "*" +}, (err,res) => { + console.log("createBucket",err,res); +}); + module.exports = { removeFile: (path, callback) => { - // const s3 = new AWS.S3({ - // region: 'eu-central-1' - // }); const bucket = config.get("storage_bucket"); s3.deleteObject({ Bucket: bucket, Key: path @@ -57,11 +62,6 @@ module.exports = { } }); fileStream.on('open', function () { - // FIXME - // var s3 = new AWS.S3({ - // region: 'eu-central-1' - // }); - s3.putObject({ Bucket: bucket, Key: fileName,