Merge pull request #10 from spacedeck/importer

WIP Importer for spacedeck.com exports Alternative (to S3) local file storage Alternative mocked redis (run without redis for single process instances) Experimental Docker support
2018-03-30 20:40:11 +00:00 · 2018-03-30 20:40:11 +00:00 · 256d2b8cbf
commit 256d2b8cbf
parent 9877977750 6d2d2310b6
22 changed files with 468 additions and 105 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -0,0 +1,14 @@
+.DS_Store
+.git
+logs
+*.log
+scripts
+pids
+*.pid
+*.seed
+lib-cov
+coverage
+.grunt
+.lock-wscript
+build/Release
+node_modules
--- a/29
+++ b/29
@ -0,0 +1,29 @@
+FROM spacedeck/docker-baseimage:latest
+ENV NODE_ENV production
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+COPY package.json /usr/src/app/
+RUN npm install
+RUN npm install gulp-rev-replace gulp-clean gulp-fingerprint gulp-rev gulp-rev-all gulp-rev-replace
+RUN npm install -g --save-dev gulp
+
+COPY app.js Dockerfile Gulpfile.js LICENSE /usr/src/app/
+COPY config /usr/src/app/config
+COPY helpers /usr/src/app/helpers
+COPY locales /usr/src/app/locales
+COPY middlewares /usr/src/app/middlewares
+COPY models /usr/src/app/models
+COPY public /usr/src/app/public
+COPY routes /usr/src/app/routes
+COPY styles /usr/src/app/styles
+COPY views /usr/src/app/views
+
+RUN gulp all
+RUN npm cache clean
+
+CMD [ "node", "app.js" ]
+
+EXPOSE 9666
+
--- a/Gulpfile.js
+++ b/Gulpfile.js
@ -12,10 +12,9 @@ var uglify = require('gulp-uglify');
 var fingerprint = require('gulp-fingerprint');
 var rev = require('gulp-rev');

-var RevAll = require('gulp-rev-all');
+var revAll = require('gulp-rev-all');

 gulp.task('rev', () => {
-  var revAll = new RevAll();
  return gulp.src(['public/**'])
    .pipe(gulp.dest('build/assets'))
    .pipe(revAll.revision())
--- a/README.md
+++ b/README.md
@ -23,10 +23,10 @@ We appreciate filed issues, pull requests and general discussion.

 Spacedeck uses the following major building blocks:

+- Vue.js (Frontend)
 - Node.js 7.x (Backend / API)
 - MongoDB 3.x (Datastore)
 - Redis 3.x (Datastore for realtime channels)
- Vue.js (Frontend)

 It also has some binary dependencies for media conversion and PDF export:

@ -50,8 +50,14 @@ see: config/config.json

    export NODE_ENV=development
    npm start
+    open http://localhost:9666

-    open http://localhost:90000
+# experimental docker(compose) support
+
+We have a docker base image at https://github.com/spacedeck/docker-baseimage that includes all required binaries. Based on this image we can use Docker Compose to bootstrap a Spacedeck including data storage.
+
+    docker-compose build
+    docker-compose run -e ENV=development -p 9666:9666 -e NODE_ENV=development spacedeck-open

 # License

--- a/app.js
+++ b/app.js
@ -15,13 +15,16 @@ const favicon = require('serve-favicon');
 const logger = require('morgan');
 const cookieParser = require('cookie-parser');
 const bodyParser = require('body-parser');
+
 const mongoose = require('mongoose');
+
 const swig = require('swig');
 const i18n = require('i18n-2');
 const helmet = require('helmet');

 const express = require('express');
 const app = express();
+const serveStatic = require('serve-static');

 const isProduction = app.get('env') === 'production';

@ -47,7 +50,7 @@ swig.setFilter('cdn', function(input, idx) {
 app.engine('html', swig.renderFile);
 app.set('view engine', 'html');

-if (app.get('env') != 'development') {
+if (isProduction) {
  app.set('views', path.join(__dirname, 'build', 'views'));
  app.use(favicon(path.join(__dirname, 'build', 'assets', 'images', 'favicon.png')));
  app.use(express.static(path.join(__dirname, 'build', 'assets')));
@ -67,7 +70,6 @@ app.use(bodyParser.urlencoded({
 }));

 app.use(cookieParser());
-app.use(helmet.noCache())
 app.use(helmet.frameguard())
 app.use(helmet.xssFilter())
 app.use(helmet.hsts({
@ -82,7 +84,6 @@ app.use(helmet.noSniff())
 app.use(require("./middlewares/templates"));
 app.use(require("./middlewares/error_helpers"));
 app.use(require("./middlewares/setuser"));
-app.use(require("./middlewares/subdomain"));
 app.use(require("./middlewares/cors"));
 app.use(require("./middlewares/i18n"));
 app.use("/api", require("./middlewares/api_helpers"));
@ -109,6 +110,12 @@ app.use('/api/teams', require('./routes/api/teams'));
 app.use('/api/webgrabber', require('./routes/api/webgrabber'));
 app.use('/', require('./routes/root'));

+if (config.get('storage_local_path')) {
+  app.use('/storage', serveStatic(config.get('storage_local_path')+"/"+config.get('storage_bucket'), {
+    maxAge: 24*3600
+  }));
+}
+
 // catch 404 and forward to error handler
 app.use(require('./middlewares/404'));
 if (app.get('env') == 'development') {
@ -121,11 +128,11 @@ if (app.get('env') == 'development') {
 module.exports = app;

 // CONNECT TO DATABASE
-const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || 'localhost';
+const mongoHost = process.env.MONGO_PORT_27017_TCP_ADDR || config.get('mongodb_host');
 mongoose.connect('mongodb://' + mongoHost + '/spacedeck');

 // START WEBSERVER
-const port = 9000;
+const port = 9666;

 const server = http.Server(app).listen(port, () => {
  
--- a/config/default.json
+++ b/config/default.json
@ -1,7 +1,20 @@
 {
-  "endpoint": "http://localhost:9000",
-  "storage_bucket": "my_spacedeck_s3_bucket",
-  "storage_cdn": "xyz.cloudfront.net",
+  //"endpoint": "http://localhost:9000",
+  "endpoint": "http://localhost:9666",
+  "storage_region": "eu-central-1",
+
+  //"storage_bucket": "sdeck-development",
+  //"storage_cdn": "http://localhost:9123/sdeck-development",
+  //"storage_endpoint": "http://storage:9000",
+
+  "storage_bucket": "my_spacedeck_bucket",
+  "storage_cdn": "/storage",
+  "storage_local_path": "./storage",
+
+  "redis_mock": true,
+  "mongodb_host": "localhost",
+  "redis_host": "localhost",
+
  "google_access" : "",
  "google_secret" : "",
  "admin_pass": "very_secret_admin_password",
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,34 @@
+version: '2'
+services:
+  sync:
+    image: redis
+  storage:
+    image: minio/minio
+    environment:
+      - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
+      - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+    ports:
+      - 9123:9000
+    command: server /export
+  db:
+    image: mongo
+  spacedeck-open:
+    environment:
+      - env=development
+      - MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE
+      - MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+    build: .
+    volumes:
+      # - ./:/usr/src/app
+      - /usr/src/app/node_modules
+    command: npm start
+    ports:
+      - 9666:9666
+    depends_on:
+      - db
+      - sync
+      - storage
+    links:
+      - storage
+      - db
+      - sync
--- a/helpers/artifact_converter.js
+++ b/helpers/artifact_converter.js
@ -7,6 +7,7 @@ const fs = require('fs');
 const Models = require('../models/schema');
 const uploader = require('../helpers/uploader');
 const path = require('path');
+const os = require('os');

 const fileExtensionMap = {
  ".amr" : "audio/AMR",
@ -245,13 +246,12 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) {
  if (max>320 || size.width > max || size.height > max) {
    var resizedFileName = max + "_"+fileName;
    var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + resizedFileName;
-    var localResizedFilePath = "/tmp/"+resizedFileName;
+    var localResizedFilePath = os.tmpdir()+"/"+resizedFileName;
    gm(localFilePath).resize(max, max).autoOrient().write(localResizedFilePath, function (err) {
      if(!err) {
        uploader.uploadFile(s3Key, "image/jpeg", localResizedFilePath, function(err, url) {
          if (err) callback(err);
          else{
-            console.log(localResizedFilePath);
            fs.unlink(localResizedFilePath, function (err) {
              if (err) {
                console.error(err);
--- a/helpers/importer.js
+++ b/helpers/importer.js
@ -0,0 +1,103 @@
+'use strict';
+
+const extract = require('extract-zip')
+const config = require('config')
+const fs = require('fs')
+const path = require('path')
+
+require('../models/schema')
+
+module.exports = {
+  importZIP: function(user, zipPath) {
+
+    // 1. extract zip to local storage folder
+    // 2. read spaces.json from this folder
+    // 3. iterate through spaces and read all their artifact jsons
+    // 4. fixup storage paths
+    // 5. replace creator id by user._id
+
+    let relativeImportDir = 'import_'+user._id
+    let importDir = path.resolve(config.get('storage_local_path')+'/'+config.get('storage_bucket')+'/'+relativeImportDir)
+    
+    if (!fs.existsSync(importDir)) {
+      fs.mkdirSync(importDir)
+    }
+  
+    extract(zipPath, {dir: importDir}, function(err) {
+      if (err) {
+        console.log(err)
+        return
+      }
+      console.log('[import] extracted to',importDir)
+
+      let spacesJson = fs.readFileSync(importDir+'/spaces.json')
+      let spaces = JSON.parse(spacesJson)
+      var homeFolderId = null
+
+      console.log('[import] spaces:',spaces.length)
+
+      // pass 1: find homefolder
+      for (var i=0; i<spaces.length; i++) {
+        let space = spaces[i]
+        if (!space.parent_space_id) {
+          homeFolderId = space._id
+          break
+        }
+      }
+
+      console.log("[import] homeFolderId:",homeFolderId)
+
+      for (var i=0; i<spaces.length; i++) {
+        let space = spaces[i]
+
+        if (space.parent_space_id) {
+          let artifacts = JSON.parse(fs.readFileSync(importDir+'/'+space._id+'_artifacts.json'))
+          console.log('[import] space',space._id,'artifacts:',artifacts.length)
+
+          let q = {_id: space._id}
+          space.creator = user._id
+          delete space.__v
+          
+          // transplant homefolder
+          console.log("parent:",space.parent_space_id)
+          if (space.parent_space_id+"" == homeFolderId+"") {
+            space.parent_space_id = user.home_folder_id
+          }
+
+          Space.findOneAndUpdate(q, space, {upsert: true}, function(err,res) {
+            if (err) console.log("[import] space upsert err:",err)
+          })
+          
+          for (var j=0; j<artifacts.length; j++) {
+            let a = artifacts[j]
+            
+            let q = {_id: a._id}
+            a.creator = user._id
+            delete a.__v
+            delete a.payload_thumbnail_big_uri
+
+            let prefix = "/storage/"+relativeImportDir+"/"+space._id+"_files/"
+            if (a.thumbnail_uri && a.thumbnail_uri[0]!='/') a.thumbnail_uri = prefix + a.thumbnail_uri
+            if (a.payload_uri && a.payload_uri[0]!='/') a.payload_uri = prefix + a.payload_uri
+            if (a.payload_thumbnail_web_uri && a.payload_thumbnail_web_uri[0]!='/') a.payload_thumbnail_web_uri = prefix + a.payload_thumbnail_web_uri
+            if (a.payload_thumbnail_medium_uri && a.payload_thumbnail_medium_uri[0]!='/') a.payload_thumbnail_medium_uri = prefix + a.payload_thumbnail_medium_uri
+
+            if (a.payload_alternatives) {
+              for (var k=0; k<a.payload_alternatives.length; k++) {
+                let alt = a.payload_alternatives[k]
+                
+                if (alt.payload_uri && alt.payload_uri[0]!='/') alt.payload_uri = prefix + alt.payload_uri
+                if (alt.payload_thumbnail_web_uri && alt.payload_thumbnail_web_uri[0]!='/') alt.payload_thumbnail_web_uri = prefix + alt.payload_thumbnail_web_uri
+                if (alt.payload_thumbnail_medium_uri && alt.payload_thumbnail_medium_uri[0]!='/') alt.payload_thumbnail_medium_uri = prefix + alt.payload_thumbnail_medium_uri
+              }
+            }
+
+            Artifact.findOneAndUpdate(q, a, {upsert: true}, function(err,res) {
+              if (err) console.log("[import] artifact upsert err:",err)
+            })
+          }
+        }
+      }
+    })
+  }
+}
--- a/helpers/mailer.js
+++ b/helpers/mailer.js
@ -53,7 +53,7 @@ module.exports = {
          }
        }
      }, function(err, data) {
-        if(err) console.log('Email not sent:', err);
+        if (err) console.error("Error sending email:", err);
        else console.log("Email sent.");
      });
    }
--- a/helpers/phantom.js
+++ b/helpers/phantom.js
@ -32,6 +32,9 @@ module.exports = {
    };

    phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) {
+      if(err){
+        console.log(err);
+      }else{
        return browser.createPage(function (err, page) {
          console.log("page created, opening ",space_url);

@ -57,6 +60,8 @@ module.exports = {
            });
          });
        });        
+      }
+
    }, {
      onExit: on_exit
    });
--- a/helpers/redis.js
+++ b/helpers/redis.js
@ -1,14 +1,111 @@
 'use strict';

-const RedisConnection = require('ioredis');
-const websockets = require('./websockets');
+const config = require('config');
+
+// this is a mock version of the Redis API,
+// emulating Redis if it is not available locally
+var notRedis = {
+  state: {},
+  topics: {},
+  
+  publish: function(topic, msg, cb) {
+    if (!this.topics[topic]) {
+      this.topics[topic] = {
+        subscribers: []
+      };
+    }
+    var t=this.topics[topic];
+    for (var i=0; i<t.subscribers.length; i++) {
+      var s=t.subscribers[i];
+      if (s.handler) {
+        s.handler(topic, msg);
+      }
+    }
+    if (cb) cb(null);
+  },
+
+  subscribe: function(topics, cb) {
+    var handle = {
+      handler: null,
+      on: function(evt, cb) {
+        if (evt == "message") {
+          this.handler = cb;
+        }
+      }
+    };
+    
+    for (var i=0; i<topics.length; i++) {
+      var topic = topics[i];
+      if (!this.topics[topic]) {
+        this.topics[topic] = {
+          subscribers: []
+        };
+      }
+    
+      var t=this.topics[topic];
+      t.subscribers.push(handle);
+    }
+
+    cb(null, topics.length);
+    return handle;
+  },
+
+  get: function(key, cb) {
+    cb(null, this.state[key]);
+    return this.state[key];
+  },
+
+  set: function(key, val, cb) {
+    this.state[key] = val;
+    cb();
+  },
+
+  del: function(key, cb) {
+    delete this.state[key];
+    cb(null);
+  },
+
+  sadd: function(key, skey, cb) {
+    if (!this.state[key]) this.state[key] = {};
+    this.state[key][skey] = true;
+    cb(null);
+  },
+
+  srem: function(key, skey, cb) {
+    if (this.state[key]) {
+      delete this.state[key][skey];
+    }
+    cb(null);
+  },
+
+  smembers: function(key, cb) {
+    cb(null, Object.keys(this.state[key]));
+  },
+
+  incr: function(key, cb) {
+    if (!this.state[key]) this.state[key] = 0;
+    this.state[key]++;
+    cb(null, this.state[key]);
+  },
+
+  expire: function() {
+  },
+}

 module.exports = {
-  connectRedis(){
-    const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost';
+  connectRedis: function() {
+    if (config.get("redis_mock")) {
+      this.connection = notRedis;
+    } else {
+      const redisHost = process.env.REDIS_PORT_6379_TCP_ADDR || 'sync';
      this.connection = new RedisConnection(6379, redisHost);
+    }
  },
-  sendMessage(action, model, attributes, channelId) {
+  getConnection: function() {
+    this.connectRedis();
+    return this.connection;
+  },
+  sendMessage: function(action, model, attributes, channelId) {
    const data = JSON.stringify({
      channel_id: channelId,
      action: action,
@ -17,12 +114,12 @@ module.exports = {
    });
    this.connection.publish('updates', data);
  },
-  logIp(ip, cb) {
+  logIp: function(ip, cb) {
    this.connection.incr("ip_"+ ip, (err, socketCounter) => {
      cb();
    });
  },
-  rateLimit(namespace, ip, cb) {
+  rateLimit: function(namespace, ip, cb) {
    const key = "limit_"+ namespace + "_"+ ip;
    const redis = this.connection;

@ -47,7 +144,7 @@ module.exports = {
      }
    });
  }, 
-  isOnlineInSpace(user, space, cb) {
+  isOnlineInSpace: function(user, space, cb) {
    this.connection.smembers("space_" + space._id.toString(), function(err, list) {
      if (err) cb(err);
      else {
@ -59,3 +156,6 @@ module.exports = {
    });
  }
 };
+
+return module.exports;
+
--- a/helpers/uploader.js
+++ b/helpers/uploader.js
@ -1,16 +1,41 @@
 'use strict';

-var AWS = require('aws-sdk');
-AWS.config.region = 'eu-central-1';
-
 var fs = require('fs');
 var config = require('config');
+var s3 = null;
+
+// use AWS S3 or local folder depending on config
+if (config.get("storage_local_path")) {
+  var AWS = require('mock-aws-s3');
+  AWS.config.basePath = config.get("storage_local_path");
+  s3 = new AWS.S3();
+} else {
+  var AWS = require('aws-sdk');
+  var storage_endpoint = config.get("storage_endpoint");
+  const ep = new AWS.Endpoint(storage_endpoint);
+
+  AWS.config.update(new AWS.Config({
+    accessKeyId: process.env.MINIO_ACCESS_KEY, 
+    secretAccessKey: process.env.MINIO_SECRET_KEY, 
+    region: config.get("storage_region"),
+    s3ForcePathStyle: true,
+    signatureVersion: 'v4'
+  }));
+  s3 = new AWS.S3({
+    endpoint: ep
+  });
+}
+
+s3.createBucket({
+  Bucket: config.get("storage_bucket"),
+  ACL: "public-read",
+  GrantRead: "*"
+}, (err,res) => {
+  console.log("createBucket",err,res);
+});

 module.exports = {
  removeFile: (path, callback) => {
-    const s3 = new AWS.S3({
-      region: 'eu-central-1'
-    });
    const bucket = config.get("storage_bucket");
    s3.deleteObject({
      Bucket: bucket, Key: path
@ -28,7 +53,7 @@ module.exports = {
      callback({error:"missing path"}, null);
      return;
    }
-    console.log("[s3] uploading", localFilePath, " to ", fileName);
+    console.log("[storage] uploading", localFilePath, " to ", fileName);

    const bucket = config.get("storage_bucket");
    const fileStream = fs.createReadStream(localFilePath);
@ -39,11 +64,6 @@ module.exports = {
      }
    });
    fileStream.on('open', function () {
-      // FIXME
-      var s3 = new AWS.S3({
-        region: 'eu-central-1'
-      });
-
      s3.putObject({
        Bucket: bucket,
        Key: fileName,
@ -54,7 +74,7 @@ module.exports = {
          console.error(err);
          callback(err);
        } else {
-          const url = "https://"+ config.get("storage_cdn") + "/" + fileName;
+          const url = config.get("storage_cdn") + "/" + fileName;
          console.log("[s3]" + localFilePath + " to " + url);
          callback(null, url);
        }
--- a/helpers/websockets.js
+++ b/helpers/websockets.js
@ -1,20 +1,28 @@
 'use strict';
 require('../models/schema');

+const config = require('config');
+
 const WebSocketServer = require('ws').Server;

-const Redis = require('ioredis');
+const RedisConnection = require('ioredis');
 const async = require('async');
 const _ = require("underscore");
 const mongoose = require("mongoose");
 const crypto = require('crypto');

+const redisMock = require("./redis.js");
+
 module.exports = {
  startWebsockets: function(server) {
    this.setupSubscription();
-    this.state = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost');
    
    if (!this.current_websockets) {
+      if (config.get("redis_mock")) {
+        this.state = redisMock.getConnection();
+      } else {
+        this.state = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || config.get("redis_host"));
+      }
      this.current_websockets = [];
    }

@ -117,10 +125,17 @@ module.exports = {
  },

  setupSubscription: function() {
-    this.cursorSubscriber = new Redis(6379, process.env.REDIS_PORT_6379_TCP_ADDR || 'localhost');
-    this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) {
-      console.log("[redis] websockets to " + count + " topics." );
+    if (config.get("redis_mock")) {
+      this.cursorSubscriber = redisMock.getConnection().subscribe(['cursors', 'users', 'updates'], function (err, count) {
+        console.log("[redis-mock] websockets subscribed to " + count + " topics." );
      });
+    } else {
+      this.cursorSubscriber = new RedisConnection(6379, process.env.REDIS_PORT_6379_TCP_ADDR || config.get("redis_host"));
+      this.cursorSubscriber.subscribe(['cursors', 'users', 'updates'], function (err, count) {
+        console.log("[redis] websockets subscribed to " + count + " topics." );
+      });
+    }
+    
    this.cursorSubscriber.on('message', function (channel, rawMessage) {
      const msg = JSON.parse(rawMessage);
      const spaceId = msg.space_id;
@ -206,7 +221,7 @@ module.exports = {
      console.log("websocket not found to remove");
    }

-    this.state.del(ws.id, function(err, res) {
+    this.state.del(ws.id+"", function(err, res) {
      if (err) console.error(err, res);
      else {
        this.removeUserInSpace(ws.space_id, ws, (err) => {
@ -221,7 +236,8 @@ module.exports = {
  
  addUserInSpace: function(username, space, ws, cb) {
    console.log("[websockets] user "+username+" in "+space.access_mode +" space " +  space._id + " with socket "  +  ws.id);
-    this.state.set(ws.id,  username, function(err, res) {
+    
+    this.state.set(ws.id+"", username+"", function(err, res) {
      if(err) console.error(err, res);
      else {
        this.state.sadd("space_" + space._id, ws.id, function(err, res) {
@ -238,7 +254,7 @@ module.exports = {
    }.bind(this));
  },
  removeUserInSpace: function(spaceId, ws, cb) {
-    this.state.srem("space_" + spaceId, ws.id, function(err, res) {
+    this.state.srem("space_" + spaceId, ws.id+"", function(err, res) {
      if (err) cb(err);
      else {
        console.log("[websockets] socket "+  ws.id + " went offline in space " + spaceId);
--- a/middlewares/setuser.js
+++ b/middlewares/setuser.js
@ -5,27 +5,24 @@ var config = require('config');

 module.exports = (req, res, next) => {
  const token = req.cookies["sdsession"];
+  
  if (token && token != "null" && token !== null) {
    User.findOne({
      "sessions.token": token
    }).populate('team').exec((err, user) => {
+      if (err) console.error("session.token lookup error:",err);
      if (!user) {
-        // FIXME
-        var domain = "localhost";
-        res.clearCookie('sdsession', {
-          domain: domain
-        });
+        res.clearCookie('sdsession');

        if (req.accepts("text/html")) {
-          res.redirect("/");
+          res.send("Please clear your cookies and try again.");
        } else if (req.accepts('application/json')) {
          res.status(403).json({
            "error": "token_not_found"
          });
        } else {
-          res.redirect("/");
+          res.send("Please clear your cookies and try again.");
        }
-
      } else {
        req["token"] = token;
        req["user"] = user;
--- a/models/team.js
+++ b/models/team.js
@ -45,7 +45,7 @@ module.exports.teamSchema.index({

 module.exports.teamSchema.statics.getTeamForHost = (host, cb) => {

-  if (host != "127.0.0.1:9000") { //phantomjs check
+  if (host != "127.0.0.1:9666") { //phantomjs check
    let subDomainParts = host.split('.');

    if (subDomainParts.length > 2) {
--- a/package.json
+++ b/package.json
@ -23,12 +23,12 @@
    "debug": "~2.6.3",
    "execSync": "latest",
    "express": "~4.13.0",
+    "extract-zip": "^1.6.6",
    "glob": "7.1.1",
    "gm": "1.23.0",
    "googleapis": "18.0.0",
    "gulp": "^3.9.1",
    "gulp-concat": "2.6.0",
-    "gulp-eslint": "*",
    "gulp-express": "0.3.0",
    "gulp-nodemon": "*",
    "gulp-sass": "^2.0.3",
@ -40,13 +40,13 @@
    "lodash": "^4.3.0",
    "log-timestamp": "latest",
    "md5": "2.2.1",
+    "mock-aws-s3": "^2.6.0",
    "moment": "2.18.1",
    "mongoose": "4.9.3",
    "morgan": "1.8.1",
+    "node-phantom-simple": "2.2.4",
    "node-sass-middleware": "0.11.0",
    "pdfkit": "0.8.0",
-    "validator": "7.0.0",
-    "node-phantom-simple": "2.2.4",
    "phantomjs-prebuilt": "2.1.14",
    "pm2": "latest",
    "qr-image": "3.2.0",
@ -54,9 +54,11 @@
    "request": "2.81.0",
    "sanitize-html": "^1.11.1",
    "serve-favicon": "~2.4.2",
-    "swig": "1.4.2",
+    "serve-static": "^1.13.1",
    "slug": "0.9.1",
+    "swig": "1.4.2",
    "underscore": "1.8.3",
+    "validator": "7.0.0",
    "weak": "1.0.1",
    "ws": "2.2.3"
  },
@ -65,7 +67,6 @@
    "gulp": "^3.9.1",
    "gulp-clean": "^0.3.2",
    "gulp-concat": "^2.6.0",
-    "gulp-eslint": "^3.0.1",
    "gulp-express": "^0.3.0",
    "gulp-fingerprint": "^0.3.2",
    "gulp-nodemon": "^2.0.4",
@ -81,8 +82,7 @@
  },
  "description": "",
  "main": "Gulpfile.js",
-  "directories": {
-  },
+  "directories": {},
  "repository": {
    "type": "git",
    "url": "https://github.com/spacedeck/spacedeck-open.git"
--- a/routes/api/sessions.js
+++ b/routes/api/sessions.js
@ -5,6 +5,7 @@ require('../../models/schema');

 var bcrypt = require('bcryptjs');
 var crypo = require('crypto');
+var URL = require('url').URL;

 var express = require('express');
 var router = express.Router();
@ -40,11 +41,11 @@ router.post('/', function(req, res) {
              user.sessions.push(session);

              user.save(function(err, result) {
-                // FIXME
-                var secure = process.env.NODE_ENV == "production" || process.env.NODE_ENV == "staging";
-                var domain = (process.env.NODE_ENV == "production") ? ".example.org" : "localhost";
+                if (err) console.error("Error saving user:",err);
                
-                res.cookie('sdsession', token, { domain: domain, httpOnly: true, secure: secure});
+                var domain = (process.env.NODE_ENV == "production") ? new URL(config.get('endpoint')).hostname : "localhost";
+
+                res.cookie('sdsession', token, { domain: domain, httpOnly: true });
                res.status(201).json(session);
              });
            });
@ -69,8 +70,7 @@ router.delete('/current', function(req, res, next) {
    });
    user.sessions = newSessions;
    user.save(function(err, result) {
-      // FIXME
-      var domain = (process.env.NODE_ENV == "production") ? ".example.org" : "localhost";
+      var domain = new URL(config.get('endpoint')).hostname;
      res.clearCookie('sdsession', { domain: domain });
      res.sendStatus(204);
    });
--- a/routes/api/space_artifacts.js
+++ b/routes/api/space_artifacts.js
@ -59,7 +59,9 @@ router.get('/', (req, res) => {
          "nickname": 1,
          "email": 1
        }).exec((err, user) => {
+          if (user) {
            a['user'] = user.toObject();
+          }
          cb(err, a);
        });
      } else {
--- a/routes/api/users.js
+++ b/routes/api/users.js
@ -5,6 +5,7 @@ require('../../models/schema');

 var mailer = require('../../helpers/mailer');
 var uploader = require('../../helpers/uploader');
+var importer = require('../../helpers/importer');

 var bcrypt = require('bcryptjs');
 var crypo = require('crypto');
@ -15,6 +16,7 @@ var fs = require('fs');
 var request = require('request');
 var gm = require('gm');
 var validator = require('validator');
+var URL = require('url').URL;

 var express = require('express');
 var router = express.Router();
@ -181,8 +183,7 @@ router.get('/loginorsignupviagoogle', function(req, res) {
      var apiUrl = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=" + tokens.access_token;
      
      var finalizeLogin = function(session){
-        var secure = process.env.NODE_ENV == "production" || process.env.NODE_ENV == "staging";
-        res.cookie('sdsession', session.token, { httpOnly: true, secure: secure});
+        res.cookie('sdsession', session.token, { httpOnly: true });
        res.status(201).json(session);
      };

@ -467,4 +468,13 @@ router.post('/:user_id/confirm', function(req, res, next) {
  res.sendStatus(201);
 });

+router.get('/:user_id/import', function(req, res, next) {
+  if (req.query.zip) {
+    res.send("importing");
+    importer.importZIP(req.user, req.query.zip);
+  } else {
+    res.sendStatus(400);
+  }
+});
+
 module.exports = router;
--- a/views/layouts/outer.html
+++ b/views/layouts/outer.html
@ -53,7 +53,7 @@
      <p>
        <div class="col-xs-6">
          <a href="/contact">[[ __("contact") ]]</a>
-          <span style="color:#888">&copy; 2011–2017 The Spacedeck Open Developers</span>
+          <span style="color:#888">&copy; 2011–2018 The Spacedeck Open Developers <a href="https://github.com/spacedeck/spacedeck-open">https://github.com/spacedeck/spacedeck-open</a></span>
        </div>
      </p>
    </div>
--- a/views/spacedeck.html
+++ b/views/spacedeck.html
@ -23,10 +23,18 @@
      {% if process.env.NODE_ENV != "production" %}
        var ENV = {
          name: 'development',
-          webHost: "localhost:9000",
-          webEndpoint:"http://localhost:9000",
-          apiEndpoint: "http://localhost:9000",
-          websocketsEndpoint: "ws://localhost:9000"
+          webHost: "localhost:9666",
+          webEndpoint:"http://localhost:9666",
+          apiEndpoint: "http://localhost:9666",
+          websocketsEndpoint: "ws://localhost:9666"
+        };
+      {% else %}
+        var ENV = {
+          name: 'production',
+          webHost: location.host,
+          webEndpoint: location.origin,
+          apiEndpoint: location.origin,
+          websocketsEndpoint: location.origin.replace("https:","wss:").replace("http:","ws:")
        };
      {% endif %}