diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..979bf6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.DS_Store +.git +logs +*.log +scripts +pids +*.pid +*.seed +lib-cov +coverage +.grunt +.lock-wscript +build/Release +node_modules \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a614e97 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM spacedeck/docker-baseimage:latest +ENV NODE_ENV production + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY package.json /usr/src/app/ +RUN npm install +RUN npm install gulp-rev-replace gulp-clean gulp-fingerprint gulp-rev gulp-rev-all gulp-rev-replace +RUN npm install -g --save-dev gulp + +COPY app.js Dockerfile Gulpfile.js LICENSE /usr/src/app/ +COPY config /usr/src/app/config +COPY helpers /usr/src/app/helpers +COPY locales /usr/src/app/locales +COPY middlewares /usr/src/app/middlewares +COPY models /usr/src/app/models +COPY public /usr/src/app/public +COPY routes /usr/src/app/routes +COPY styles /usr/src/app/styles +COPY views /usr/src/app/views + +RUN gulp all +RUN npm cache clean + +CMD [ "node", "app.js" ] + +EXPOSE 9666 + diff --git a/Gulpfile.js b/Gulpfile.js index af47a5d..e4aa47e 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -12,10 +12,9 @@ var uglify = require('gulp-uglify'); var fingerprint = require('gulp-fingerprint'); var rev = require('gulp-rev'); -var RevAll = require('gulp-rev-all'); +var revAll = require('gulp-rev-all'); gulp.task('rev', () => { - var revAll = new RevAll(); return gulp.src(['public/**']) .pipe(gulp.dest('build/assets')) .pipe(revAll.revision()) diff --git a/README.md b/README.md index 73a2846..60ad14a 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,10 @@ We appreciate filed issues, pull requests and general discussion. Spacedeck uses the following major building blocks: +- Vue.js (Frontend) - Node.js 7.x (Backend / API) - MongoDB 3.x (Datastore) - Redis 3.x (Datastore for realtime channels) -- Vue.js (Frontend) It also has some binary dependencies for media conversion and PDF export: @@ -50,8 +50,14 @@ see: config/config.json export NODE_ENV=development npm start + open http://localhost:9666 - open http://localhost:90000 +# experimental docker(compose) support + +We have a docker base image at https://github.com/spacedeck/docker-baseimage that includes all required binaries. Based on this image we can use Docker Compose to bootstrap a Spacedeck including data storage. + + docker-compose build + docker-compose run -e ENV=development -p 9666:9666 -e NODE_ENV=development spacedeck-open # License diff --git a/app.js b/app.js index 24b4fb5..004fb5a 100644 --- a/app.js +++ b/app.js @@ -15,13 +15,16 @@ const favicon = require('serve-favicon'); const logger = require('morgan'); const cookieParser = require('cookie-parser'); const bodyParser = require('body-parser'); + const mongoose = require('mongoose'); + const swig = require('swig'); const i18n = require('i18n-2'); const helmet = require('helmet'); const express = require('express'); const app = express(); +const serveStatic = require('serve-static'); const isProduction = app.get('env') === 'production'; @@ -47,7 +50,7 @@ swig.setFilter('cdn', function(input, idx) { app.engine('html', swig.renderFile); app.set('view engine', 'html'); -if (app.get('env') != 'development') { +if (isProduction) { app.set('views', path.join(__dirname, 'build', 'views')); app.use(favicon(path.join(__dirname, 'build', 'assets', 'images', 'favicon.png'))); app.use(express.static(path.join(__dirname, 'build', 'assets'))); @@ -67,7 +70,6 @@ app.use(bodyParser.urlencoded({ })); app.use(cookieParser()); -app.use(helmet.noCache()) app.use(helmet.frameguard()) app.use(helmet.xssFilter()) app.use(helmet.hsts({ @@ -82,7 +84,6 @@ app.use(helmet.noSniff()) app.use(require("./middlewares/templates")); app.use(require("./middlewares/error_helpers")); app.use(require("./middlewares/setuser")); -app.use(require("./middlewares/subdomain")); app.use(require("./middlewares/cors")); app.use(require("./middlewares/i18n")); app.use("/api", require("./middlewares/api_helpers")); @@ -109,6 +110,12 @@ app.use('/api/teams', require('./routes/api/teams')); app.use('/api/webgrabber', require('./routes/api/webgrabber')); app.use('/', require('./routes/root')); +if (config.get('storage_local_path')) { + app.use('/storage', serveStatic(config.get('storage_local_path')+"/"+config.get('storage_bucket'), { + maxAge: 24*3600 + })); +} + // catch 404 and forward to error handler app.use(require('./middlewares/404')); if (app.get('env') == 'development') { @@ -121,14 +128,14 @@ if (app.get('env') == 'development') { module.exports = app; // CONNECT TO DATABASE -const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || 'localhost'; +const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || config.get('mongodb_host'); mongoose.connect('mongodb://' + mongoHost + '/spacedeck'); // START WEBSERVER -const port = 9000; +const port = 9666; const server = http.Server(app).listen(port, () => { - + if ("send" in process) { process.send('online'); } diff --git a/config/default.json b/config/default.json index 445fa23..37a00aa 100644 --- a/config/default.json +++ b/config/default.json @@ -1,7 +1,20 @@ { - "endpoint": "http://localhost:9000", - "storage_bucket": "my_spacedeck_s3_bucket", - "storage_cdn": "xyz.cloudfront.net", + //"endpoint": "http://localhost:9000", + "endpoint": "http://localhost:9666", + "storage_region": "eu-central-1", + + //"storage_bucket": "sdeck-development", + //"storage_cdn": "http://localhost:9123/sdeck-development", + //"storage_endpoint": "http://storage:9000", + + "storage_bucket": "my_spacedeck_bucket", + "storage_cdn": "/storage", + "storage_local_path": "./storage", + + "redis_mock": true, + "mongodb_host": "localhost", + "redis_host": "localhost", + "google_access" : "", "google_secret" : "", "admin_pass": "very_secret_admin_password", diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..688f381 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,34 @@ +version: '2' +services: + sync: + image: redis + storage: + image: minio/minio + environment: + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + ports: + - 9123:9000 + command: server /export + db: + image: mongo + spacedeck-open: + environment: + - env=development + - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE + - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + build: . + volumes: + # - ./:/usr/src/app + - /usr/src/app/node_modules + command: npm start + ports: + - 9666:9666 + depends_on: + - db + - sync + - storage + links: + - storage + - db + - sync diff --git a/helpers/artifact_converter.js b/helpers/artifact_converter.js index 3605a90..d6f468d 100644 --- a/helpers/artifact_converter.js +++ b/helpers/artifact_converter.js @@ -7,6 +7,7 @@ const fs = require('fs'); const Models = require('../models/schema'); const uploader = require('../helpers/uploader'); const path = require('path'); +const os = require('os'); const fileExtensionMap = { ".amr" : "audio/AMR", @@ -245,13 +246,12 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) { if (max>320 || size.width > max || size.height > max) { var resizedFileName = max + "_"+fileName; var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + resizedFileName; - var localResizedFilePath = "/tmp/"+resizedFileName; + var localResizedFilePath = os.tmpdir()+"/"+resizedFileName; gm(localFilePath).resize(max, max).autoOrient().write(localResizedFilePath, function (err) { if(!err) { uploader.uploadFile(s3Key, "image/jpeg", localResizedFilePath, function(err, url) { if (err) callback(err); else{ - console.log(localResizedFilePath); fs.unlink(localResizedFilePath, function (err) { if (err) { console.error(err); diff --git a/helpers/importer.js b/helpers/importer.js new file mode 100644 index 0000000..90eb2a9 --- /dev/null +++ b/helpers/importer.js @@ -0,0 +1,103 @@ +'use strict'; + +const extract = require('extract-zip') +const config = require('config') +const fs = require('fs') +const path = require('path') + +require('../models/schema') + +module.exports = { + importZIP: function(user, zipPath) { + + // 1. extract zip to local storage folder + // 2. read spaces.json from this folder + // 3. iterate through spaces and read all their artifact jsons + // 4. fixup storage paths + // 5. replace creator id by user._id + + let relativeImportDir = 'import_'+user._id + let importDir = path.resolve(config.get('storage_local_path')+'/'+config.get('storage_bucket')+'/'+relativeImportDir) + + if (!fs.existsSync(importDir)) { + fs.mkdirSync(importDir) + } + + extract(zipPath, {dir: importDir}, function(err) { + if (err) { + console.log(err) + return + } + console.log('[import] extracted to',importDir) + + let spacesJson = fs.readFileSync(importDir+'/spaces.json') + let spaces = JSON.parse(spacesJson) + var homeFolderId = null + + console.log('[import] spaces:',spaces.length) + + // pass 1: find homefolder + for (var i=0; i { cb(); }); }, - rateLimit(namespace, ip, cb) { + rateLimit: function(namespace, ip, cb) { const key = "limit_"+ namespace + "_"+ ip; const redis = this.connection; @@ -47,7 +144,7 @@ module.exports = { } }); }, - isOnlineInSpace(user, space, cb) { + isOnlineInSpace: function(user, space, cb) { this.connection.smembers("space_" + space._id.toString(), function(err, list) { if (err) cb(err); else { @@ -59,3 +156,6 @@ module.exports = { }); } }; + +return module.exports; + diff --git a/helpers/uploader.js b/helpers/uploader.js index c314062..958d12c 100644 --- a/helpers/uploader.js +++ b/helpers/uploader.js @@ -1,16 +1,41 @@ 'use strict'; -var AWS = require('aws-sdk'); -AWS.config.region = 'eu-central-1'; - var fs = require('fs'); var config = require('config'); +var s3 = null; + +// use AWS S3 or local folder depending on config +if (config.get("storage_local_path")) { + var AWS = require('mock-aws-s3'); + AWS.config.basePath = config.get("storage_local_path"); + s3 = new AWS.S3(); +} else { + var AWS = require('aws-sdk'); + var storage_endpoint = config.get("storage_endpoint"); + const ep = new AWS.Endpoint(storage_endpoint); + + AWS.config.update(new AWS.Config({ + accessKeyId: process.env.MINIO_ACCESS_KEY, + secretAccessKey: process.env.MINIO_SECRET_KEY, + region: config.get("storage_region"), + s3ForcePathStyle: true, + signatureVersion: 'v4' + })); + s3 = new AWS.S3({ + endpoint: ep + }); +} + +s3.createBucket({ + Bucket: config.get("storage_bucket"), + ACL: "public-read", + GrantRead: "*" +}, (err,res) => { + console.log("createBucket",err,res); +}); module.exports = { removeFile: (path, callback) => { - const s3 = new AWS.S3({ - region: 'eu-central-1' - }); const bucket = config.get("storage_bucket"); s3.deleteObject({ Bucket: bucket, Key: path @@ -28,7 +53,7 @@ module.exports = { callback({error:"missing path"}, null); return; } - console.log("[s3] uploading", localFilePath, " to ", fileName); + console.log("[storage] uploading", localFilePath, " to ", fileName); const bucket = config.get("storage_bucket"); const fileStream = fs.createReadStream(localFilePath); @@ -39,11 +64,6 @@ module.exports = { } }); fileStream.on('open', function () { - // FIXME - var s3 = new AWS.S3({ - region: 'eu-central-1' - }); - s3.putObject({ Bucket: bucket, Key: fileName, @@ -53,8 +73,8 @@ module.exports = { if (err){ console.error(err); callback(err); - }else { - const url = "https://"+ config.get("storage_cdn") + "/" + fileName; + } else { + const url = config.get("storage_cdn") + "/" + fileName; console.log("[s3]" + localFilePath + " to " + url); callback(null, url); } diff --git a/helpers/websockets.js b/helpers/websockets.js index 26fcd1c..d541ede 100644 --- a/helpers/websockets.js +++ b/helpers/websockets.js @@ -1,20 +1,28 @@ 'use strict'; require('../models/schema'); +const config = require('config'); + const WebSocketServer = require('ws').Server; -const Redis = require('ioredis'); +const RedisConnection = require('ioredis'); const async = require('async'); const _ = require("underscore"); const mongoose = require("mongoose"); const crypto = require('crypto'); -module.exports = { - startWebsockets: function(server){ - this.setupSubscription(); - this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); +const redisMock = require("./redis.js"); - if(!this.current_websockets){ +module.exports = { + startWebsockets: function(server) { + this.setupSubscription(); + + if (!this.current_websockets) { + if (config.get("redis_mock")) { + this.state = redisMock.getConnection(); + } else { + this.state = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || config.get("redis_host")); + } this.current_websockets = []; } @@ -117,10 +125,17 @@ module.exports = { }, setupSubscription: function() { - this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost'); - this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) { - console.log("[redis] websockets to " + count + " topics." ); - }); + if (config.get("redis_mock")) { + this.cursorSubscriber = redisMock.getConnection().subscribe(['cursors', 'users', 'updates'], function (err, count) { + console.log("[redis-mock] websockets subscribed to " + count + " topics." ); + }); + } else { + this.cursorSubscriber = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || config.get("redis_host")); + this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) { + console.log("[redis] websockets subscribed to " + count + " topics." ); + }); + } + this.cursorSubscriber.on('message', function (channel, rawMessage) { const msg = JSON.parse(rawMessage); const spaceId = msg.space_id; @@ -206,7 +221,7 @@ module.exports = { console.log("websocket not found to remove"); } - this.state.del(ws.id, function(err, res) { + this.state.del(ws.id+"", function(err, res) { if (err) console.error(err, res); else { this.removeUserInSpace(ws.space_id, ws, (err) => { @@ -221,7 +236,8 @@ module.exports = { addUserInSpace: function(username, space, ws, cb) { console.log("[websockets] user "+username+" in "+space.access_mode +" space " + space._id + " with socket " + ws.id); - this.state.set(ws.id, username, function(err, res) { + + this.state.set(ws.id+"", username+"", function(err, res) { if(err) console.error(err, res); else { this.state.sadd("space_" + space._id, ws.id, function(err, res) { @@ -238,7 +254,7 @@ module.exports = { }.bind(this)); }, removeUserInSpace: function(spaceId, ws, cb) { - this.state.srem("space_" + spaceId, ws.id, function(err, res) { + this.state.srem("space_" + spaceId, ws.id+"", function(err, res) { if (err) cb(err); else { console.log("[websockets] socket "+ ws.id + " went offline in space " + spaceId); diff --git a/middlewares/setuser.js b/middlewares/setuser.js index 349aebf..56bcd78 100644 --- a/middlewares/setuser.js +++ b/middlewares/setuser.js @@ -5,27 +5,24 @@ var config = require('config'); module.exports = (req, res, next) => { const token = req.cookies["sdsession"]; + if (token && token != "null" && token !== null) { User.findOne({ "sessions.token": token }).populate('team').exec((err, user) => { + if (err) console.error("session.token lookup error:",err); if (!user) { - // FIXME - var domain = "localhost"; - res.clearCookie('sdsession', { - domain: domain - }); + res.clearCookie('sdsession'); if (req.accepts("text/html")) { - res.redirect("/"); + res.send("Please clear your cookies and try again."); } else if (req.accepts('application/json')) { res.status(403).json({ "error": "token_not_found" }); } else { - res.redirect("/"); + res.send("Please clear your cookies and try again."); } - } else { req["token"] = token; req["user"] = user; diff --git a/models/team.js b/models/team.js index a1f3cc7..b35942c 100644 --- a/models/team.js +++ b/models/team.js @@ -45,7 +45,7 @@ module.exports.teamSchema.index({ module.exports.teamSchema.statics.getTeamForHost = (host, cb) => { - if (host != "127.0.0.1:9000") { //phantomjs check + if (host != "127.0.0.1:9666") { //phantomjs check let subDomainParts = host.split('.'); if (subDomainParts.length > 2) { diff --git a/package.json b/package.json index 78b7025..efe0f87 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,8 @@ "start": "nodemon -e .js,.html bin/www", "test": "mocha" }, - "engines" : { - "node" : ">=7.8.0" + "engines": { + "node": ">=7.8.0" }, "dependencies": { "archiver": "1.3.0", @@ -23,12 +23,12 @@ "debug": "~2.6.3", "execSync": "latest", "express": "~4.13.0", + "extract-zip": "^1.6.6", "glob": "7.1.1", "gm": "1.23.0", "googleapis": "18.0.0", "gulp": "^3.9.1", "gulp-concat": "2.6.0", - "gulp-eslint": "*", "gulp-express": "0.3.0", "gulp-nodemon": "*", "gulp-sass": "^2.0.3", @@ -40,13 +40,13 @@ "lodash": "^4.3.0", "log-timestamp": "latest", "md5": "2.2.1", + "mock-aws-s3": "^2.6.0", "moment": "2.18.1", "mongoose": "4.9.3", "morgan": "1.8.1", + "node-phantom-simple": "2.2.4", "node-sass-middleware": "0.11.0", "pdfkit": "0.8.0", - "validator": "7.0.0", - "node-phantom-simple": "2.2.4", "phantomjs-prebuilt": "2.1.14", "pm2": "latest", "qr-image": "3.2.0", @@ -54,9 +54,11 @@ "request": "2.81.0", "sanitize-html": "^1.11.1", "serve-favicon": "~2.4.2", - "swig": "1.4.2", + "serve-static": "^1.13.1", "slug": "0.9.1", + "swig": "1.4.2", "underscore": "1.8.3", + "validator": "7.0.0", "weak": "1.0.1", "ws": "2.2.3" }, @@ -65,7 +67,6 @@ "gulp": "^3.9.1", "gulp-clean": "^0.3.2", "gulp-concat": "^2.6.0", - "gulp-eslint": "^3.0.1", "gulp-express": "^0.3.0", "gulp-fingerprint": "^0.3.2", "gulp-nodemon": "^2.0.4", @@ -81,8 +82,7 @@ }, "description": "", "main": "Gulpfile.js", - "directories": { - }, + "directories": {}, "repository": { "type": "git", "url": "https://github.com/spacedeck/spacedeck-open.git" diff --git a/routes/api/sessions.js b/routes/api/sessions.js index 3b63042..054e49c 100644 --- a/routes/api/sessions.js +++ b/routes/api/sessions.js @@ -5,6 +5,7 @@ require('../../models/schema'); var bcrypt = require('bcryptjs'); var crypo = require('crypto'); +var URL = require('url').URL; var express = require('express'); var router = express.Router(); @@ -40,11 +41,11 @@ router.post('/', function(req, res) { user.sessions.push(session); user.save(function(err, result) { - // FIXME - var secure = process.env.NODE_ENV == "production" || process.env.NODE_ENV == "staging"; - var domain = (process.env.NODE_ENV == "production") ? ".example.org" : "localhost"; + if (err) console.error("Error saving user:",err); + + var domain = (process.env.NODE_ENV == "production") ? new URL(config.get('endpoint')).hostname : "localhost"; - res.cookie('sdsession', token, { domain: domain, httpOnly: true, secure: secure}); + res.cookie('sdsession', token, { domain: domain, httpOnly: true }); res.status(201).json(session); }); }); @@ -69,8 +70,7 @@ router.delete('/current', function(req, res, next) { }); user.sessions = newSessions; user.save(function(err, result) { - // FIXME - var domain = (process.env.NODE_ENV == "production") ? ".example.org" : "localhost"; + var domain = new URL(config.get('endpoint')).hostname; res.clearCookie('sdsession', { domain: domain }); res.sendStatus(204); }); diff --git a/routes/api/space_artifacts.js b/routes/api/space_artifacts.js index e714456..d3e5006 100644 --- a/routes/api/space_artifacts.js +++ b/routes/api/space_artifacts.js @@ -59,7 +59,9 @@ router.get('/', (req, res) => { "nickname": 1, "email": 1 }).exec((err, user) => { - a['user'] = user.toObject(); + if (user) { + a['user'] = user.toObject(); + } cb(err, a); }); } else { diff --git a/routes/api/users.js b/routes/api/users.js index 8140bcc..7e1a1aa 100644 --- a/routes/api/users.js +++ b/routes/api/users.js @@ -5,6 +5,7 @@ require('../../models/schema'); var mailer = require('../../helpers/mailer'); var uploader = require('../../helpers/uploader'); +var importer = require('../../helpers/importer'); var bcrypt = require('bcryptjs'); var crypo = require('crypto'); @@ -15,6 +16,7 @@ var fs = require('fs'); var request = require('request'); var gm = require('gm'); var validator = require('validator'); +var URL = require('url').URL; var express = require('express'); var router = express.Router(); @@ -181,8 +183,7 @@ router.get('/loginorsignupviagoogle', function(req, res) { var apiUrl = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=" + tokens.access_token; var finalizeLogin = function(session){ - var secure = process.env.NODE_ENV == "production" || process.env.NODE_ENV == "staging"; - res.cookie('sdsession', session.token, { httpOnly: true, secure: secure}); + res.cookie('sdsession', session.token, { httpOnly: true }); res.status(201).json(session); }; @@ -467,4 +468,13 @@ router.post('/:user_id/confirm', function(req, res, next) { res.sendStatus(201); }); +router.get('/:user_id/import', function(req, res, next) { + if (req.query.zip) { + res.send("importing"); + importer.importZIP(req.user, req.query.zip); + } else { + res.sendStatus(400); + } +}); + module.exports = router; diff --git a/views/layouts/outer.html b/views/layouts/outer.html index 8753582..603d96a 100644 --- a/views/layouts/outer.html +++ b/views/layouts/outer.html @@ -53,7 +53,7 @@

[[ __("contact") ]] - © 2011–2017 The Spacedeck Open Developers + © 2011–2018 The Spacedeck Open Developers https://github.com/spacedeck/spacedeck-open

diff --git a/views/spacedeck.html b/views/spacedeck.html index 54144dd..8cf7ec4 100644 --- a/views/spacedeck.html +++ b/views/spacedeck.html @@ -23,10 +23,18 @@ {% if process.env.NODE_ENV != "production" %} var ENV = { name: 'development', - webHost: "localhost:9000", - webEndpoint:"http://localhost:9000", - apiEndpoint: "http://localhost:9000", - websocketsEndpoint: "ws://localhost:9000" + webHost: "localhost:9666", + webEndpoint:"http://localhost:9666", + apiEndpoint: "http://localhost:9666", + websocketsEndpoint: "ws://localhost:9666" + }; + {% else %} + var ENV = { + name: 'production', + webHost: location.host, + webEndpoint: location.origin, + apiEndpoint: location.origin, + websocketsEndpoint: location.origin.replace("https:","wss:").replace("http:","ws:") }; {% endif %}