WebDev/spacedeck-open
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Port Backend to SQLite/Sequelize (removes MongoDB), Support Electron (#14)

Browse Source 
* The MongoDB/Mongoose data storage is removed in favor of Sequelize. This abstracts over SQLite or RDBMs like PostgreSQL and MSSQL. The default is SQLite, which significantly simplifies deployments in end-user environments.

* As Spacedeck now has no more mandatory server dependencies, we can wrap it in Electron and ship it as a desktop application.

* Removes docker-compose.yml

* First version of import UI
This commit is contained in:
mntmn
2018-04-12 16:40:58 +00:00
committed by GitHub
parent 8e0bc69a11
commit ebac854da8
62 changed files with 1725 additions and 3024 deletions
Download Patch File Download Diff File Expand all files Collapse all files

215
helpers/artifact_converter.js
View File

@@ -4,52 +4,18 @@ const exec = require('child_process');
const gm = require('gm');
const async = require('async');
const fs = require('fs');
const Models = require('../models/schema');
const Models = require('../models/db');
const uploader = require('../helpers/uploader');
const path = require('path');
const os = require('os');
const fileExtensionMap = {
".amr" : "audio/AMR",
".ogg" : "audio/ogg",
".aac" : "audio/aac",
".mp3" : "audio/mpeg",
".mpg" : "video/mpeg",
".3ga" : "audio/3ga",
".mp4" : "video/mp4",
".wav" : "audio/wav",
".mov" : "video/quicktime",
".doc" : "application/msword",
".dot" : "application/msword",
".docx" : "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".dotx" : "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
".docm" : "application/vnd.ms-word.document.macroEnabled.12",
".dotm" : "application/vnd.ms-word.template.macroEnabled.12",
".xls" : "application/vnd.ms-excel",
".xlt" : "application/vnd.ms-excel",
".xla" : "application/vnd.ms-excel",
".xlsx" : "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".xltx" : "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
".xlsm" : "application/vnd.ms-excel.sheet.macroEnabled.12",
".xltm" : "application/vnd.ms-excel.template.macroEnabled.12",
".xlam" : "application/vnd.ms-excel.addin.macroEnabled.12",
".xlsb" : "application/vnd.ms-excel.sheet.binary.macroEnabled.12",
".ppt" : "application/vnd.ms-powerpoint",
".pot" : "application/vnd.ms-powerpoint",
".pps" : "application/vnd.ms-powerpoint",
".ppa" : "application/vnd.ms-powerpoint",
".pptx" : "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".potx" : "application/vnd.openxmlformats-officedocument.presentationml.template",
".ppsx" : "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
".ppam" : "application/vnd.ms-powerpoint.addin.macroEnabled.12",
".pptm" : "application/vnd.ms-powerpoint.presentation.macroEnabled.12",
".potm" : "application/vnd.ms-powerpoint.template.macroEnabled.12",
".ppsm" : "application/vnd.ms-powerpoint.slideshow.macroEnabled.12",
".key" : "application/x-iwork-keynote-sffkey",
".pages" : "application/x-iwork-pages-sffpages",
".numbers" : "application/x-iwork-numbers-sffnumbers",
".ttf" : "application/x-font-ttf"
};
const db = require('../models/db');
const Sequelize = require('sequelize');
const Op = Sequelize.Op;
const mime = require('mime-types');
const fileType = require('file-type');
const readChunk = require('read-chunk');
const convertableImageTypes = [
"image/png",
@@ -69,7 +35,7 @@ const convertableVideoTypes = [
const convertableAudioTypes = [
"application/ogg",
"audio/AMR",
"audio/amr",
"audio/3ga",
"audio/wav",
"audio/3gpp",
@@ -128,7 +94,7 @@ function createWaveform(fileName, localFilePath, callback){
function convertVideo(fileName, filePath, codec, callback, progress_callback) {
var ext = path.extname(fileName);
var presetMime = fileExtensionMap[ext];
var presetMime = mime.lookup(fileName);
var newExt = codec == "mp4" ? "mp4" : "ogv";
var convertedPath = filePath + "." + newExt;
@@ -186,7 +152,7 @@ function convertVideo(fileName, filePath, codec, callback, progress_callback) {
function convertAudio(fileName, filePath, codec, callback) {
var ext = path.extname(fileName);
var presetMime = fileExtensionMap[ext];
var presetMime = mime.lookup(fileName);
var newExt = codec == "mp3" ? "mp3" : "ogg";
var convertedPath = filePath + "." + newExt;
@@ -223,22 +189,14 @@ function createThumbnailForVideo(fileName, filePath, callback) {
function getMime(fileName, filePath, callback) {
var ext = path.extname(fileName);
var presetMime = fileExtensionMap[ext];
var presetMime = mime.lookup(fileName);
if (presetMime) {
callback(null, presetMime);
} else {
exec.execFile("file", ["-b","--mime-type", filePath], {}, function(error, stdout, stderr) {
console.log("file stdout: ",stdout);
if (stderr === '' && error == null) {
//filter special chars from commandline
var mime = stdout.replace(/[^a-zA-Z0-9\/\-]/g,'');
callback(null, mime);
} else {
console.log("getMime file error: ", error);
callback(error, null);
}
});
const buffer = readChunk.sync(filePath, 0, 4100);
var mimeType = fileType(buffer);
callback(null, mimeType);
}
}
@@ -272,7 +230,7 @@ function resizeAndUpload(a, size, max, fileName, localFilePath, callback) {
}
var resizeAndUploadImage = function(a, mime, size, fileName, fileNameOrg, imageFilePath, originalFilePath, payloadCallback) {
var resizeAndUploadImage = function(a, mimeType, size, fileName, fileNameOrg, imageFilePath, originalFilePath, payloadCallback) {
async.parallel({
small: function(callback){
resizeAndUpload(a, size, 320, fileName, imageFilePath, callback);
@@ -285,13 +243,13 @@ var resizeAndUploadImage = function(a, mime, size, fileName, fileNameOrg, imageF
},
original: function(callback){
var s3Key = "s"+ a.space_id.toString() + "/a" + a._id + "/" + fileNameOrg;
uploader.uploadFile(s3Key, mime, originalFilePath, function(err, url){
uploader.uploadFile(s3Key, mimeType, originalFilePath, function(err, url){
callback(null, url);
});
}
}, function(err, results) {
a.state = "idle";
a.mime = mime;
a.mime = mimeType;
var stats = fs.statSync(originalFilePath);
a.payload_size = stats["size"];
@@ -301,46 +259,41 @@ var resizeAndUploadImage = function(a, mime, size, fileName, fileNameOrg, imageF
a.payload_uri = results.original;
var factor = 320/size.width;
var newBoardSpecs = a.board;
newBoardSpecs.w = Math.round(size.width*factor);
newBoardSpecs.h = Math.round(size.height*factor);
a.board = newBoardSpecs;
a.w = Math.round(size.width*factor);
a.h = Math.round(size.height*factor);
a.updated_at = new Date();
a.save(function(err) {
if(err) payloadCallback(err, null);
else {
fs.unlink(originalFilePath, function (err) {
if (err){
console.error(err);
payloadCallback(err, null);
} else {
console.log('successfully deleted ' + originalFilePath);
payloadCallback(null, a);
}
});
}
a.save().then(function() {
fs.unlink(originalFilePath, function (err) {
if (err){
console.error(err);
payloadCallback(err, null);
} else {
console.log('successfully deleted ' + originalFilePath);
payloadCallback(null, a);
}
});
});
});
};
module.exports = {
convert: function(a, fileName, localFilePath, payloadCallback, progress_callback) {
getMime(fileName, localFilePath, function(err, mime){
console.log("[convert] fn: "+fileName+" local: "+localFilePath+" mime:", mime);
getMime(fileName, localFilePath, function(err, mimeType){
console.log("[convert] fn: "+fileName+" local: "+localFilePath+" mimeType:", mimeType);
if (!err) {
if (convertableImageTypes.indexOf(mime) > -1) {
if (convertableImageTypes.indexOf(mimeType) > -1) {
gm(localFilePath).size(function (err, size) {
console.log("[convert] gm:", err, size);
if (!err) {
if(mime == "application/pdf") {
if(mimeType == "application/pdf") {
var firstImagePath = localFilePath + ".jpeg";
exec.execFile("gs", ["-sDEVICE=jpeg","-dNOPAUSE", "-dJPEGQ=80", "-dBATCH", "-dFirstPage=1", "-dLastPage=1", "-sOutputFile=" + firstImagePath, "-r90", "-f", localFilePath], {}, function(error, stdout, stderr) {
if(error === null) {
resizeAndUploadImage(a, mime, size, fileName + ".jpeg", fileName, firstImagePath, localFilePath, function(err, a) {
resizeAndUploadImage(a, mimeType, size, fileName + ".jpeg", fileName, firstImagePath, localFilePath, function(err, a) {
fs.unlink(firstImagePath, function (err) {
payloadCallback(err, a);
});
@@ -350,7 +303,7 @@ module.exports = {
}
});
} else if(mime == "image/gif") {
} else if(mimeType == "image/gif") {
//gifs are buggy after convertion, so we should not convert them
var s3Key = "s"+ a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
@@ -362,7 +315,7 @@ module.exports = {
var stats = fs.statSync(localFilePath);
a.state = "idle";
a.mime = mime;
a.mime = mimeType;
a.payload_size = stats["size"];
a.payload_thumbnail_web_uri = url;
@@ -371,36 +324,31 @@ module.exports = {
a.payload_uri = url;
var factor = 320/size.width;
var newBoardSpecs = a.board;
newBoardSpecs.w = Math.round(size.width*factor);
newBoardSpecs.h = Math.round(size.height*factor);
a.board = newBoardSpecs;
a.w = Math.round(size.width*factor);
a.h = Math.round(size.height*factor);
a.updated_at = new Date();
a.save(function(err){
if(err) payloadCallback(err, null);
else {
fs.unlink(localFilePath, function (err) {
if (err){
console.error(err);
payloadCallback(err, null);
} else {
console.log('successfully deleted ' + localFilePath);
payloadCallback(null, a);
}
});
}
a.save().then(function() {
fs.unlink(localFilePath, function (err) {
if (err){
console.error(err);
payloadCallback(err, null);
} else {
console.log('successfully deleted ' + localFilePath);
payloadCallback(null, a);
}
});
});
}
});
} else {
resizeAndUploadImage(a, mime, size, fileName, fileName, localFilePath, localFilePath, payloadCallback);
resizeAndUploadImage(a, mimeType, size, fileName, fileName, localFilePath, localFilePath, payloadCallback);
}
} else payloadCallback(err);
});
} else if (convertableVideoTypes.indexOf(mime) > -1) {
} else if (convertableVideoTypes.indexOf(mimeType) > -1) {
async.parallel({
thumbnail: function(callback) {
createThumbnailForVideo(fileName, localFilePath, function(err, created){
@@ -416,7 +364,7 @@ module.exports = {
});
},
ogg: function(callback) {
if (mime == "video/ogg") {
if (mimeType == "video/ogg") {
callback(null, "org");
} else {
convertVideo(fileName, localFilePath, "ogg", function(err, file) {
@@ -432,7 +380,7 @@ module.exports = {
}
},
mp4: function(callback) {
if (mime == "video/mp4") {
if (mimeType == "video/mp4") {
callback(null, "org");
} else {
convertVideo(fileName, localFilePath, "mp4", function(err, file) {
@@ -448,7 +396,7 @@ module.exports = {
}
},
original: function(callback){
uploader.uploadFile(fileName, mime, localFilePath, function(err, url){
uploader.uploadFile(fileName, mimeType, localFilePath, function(err, url){
callback(null, url);
});
}
@@ -458,7 +406,7 @@ module.exports = {
if (err) payloadCallback(err, a);
else {
a.state = "idle";
a.mime = mime;
a.mime = mimeType;
var stats = fs.statSync(localFilePath);
a.payload_size = stats["size"];
@@ -467,7 +415,7 @@ module.exports = {
a.payload_thumbnail_big_uri = results.thumbnail;
a.payload_uri = results.original;
if (mime == "video/mp4") {
if (mimeType == "video/mp4") {
a.payload_alternatives = [
{
mime: "video/ogg",
@@ -483,6 +431,8 @@ module.exports = {
];
}
db.packArtifact(a);
a.updated_at = new Date();
a.save(function(err) {
if (err) payloadCallback(err, null);
@@ -501,7 +451,7 @@ module.exports = {
}
});
} else if (convertableAudioTypes.indexOf(mime) > -1) {
} else if (convertableAudioTypes.indexOf(mimeType) > -1) {
async.parallel({
ogg: function(callback) {
@@ -539,7 +489,7 @@ module.exports = {
},
original: function(callback) {
var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
uploader.uploadFile(keyName, mime, localFilePath, function(err, url){
uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url){
callback(null, url);
});
}
@@ -550,7 +500,7 @@ module.exports = {
else {
a.state = "idle";
a.mime = mime;
a.mime = mimeType;
var stats = fs.statSync(localFilePath);
a.payload_size = stats["size"];
@@ -564,43 +514,40 @@ module.exports = {
];
a.updated_at = new Date();
a.save(function(err){
if(err) payloadCallback(err, null);
else {
fs.unlink(localFilePath, function (err) {
if (err){
console.error(err);
payloadCallback(err, null);
} else {
console.log('successfully deleted ' + localFilePath);
payloadCallback(null, a);
}
});
}
db.packArtifact(a);
a.save().then(function(){
fs.unlink(localFilePath, function (err) {
if (err){
console.error(err);
payloadCallback(err, null);
} else {
console.log('successfully deleted ' + localFilePath);
payloadCallback(null, a);
}
});
});
}
});
} else {
console.log("mime not matched for conversion, storing file");
console.log("mimeType not matched for conversion, storing file");
var keyName = "s" + a.space_id.toString() + "/a" + a._id.toString() + "/" + fileName;
uploader.uploadFile(keyName, mime, localFilePath, function(err, url) {
uploader.uploadFile(keyName, mimeType, localFilePath, function(err, url) {
a.state = "idle";
a.mime = mime;
a.mime = mimeType;
var stats = fs.statSync(localFilePath);
a.payload_size = stats["size"];
a.payload_uri = url;
a.updated_at = new Date();
a.save(function(err) {
if(err) payloadCallback(err, null);
else {
fs.unlink(localFilePath, function (err) {
payloadCallback(null, a);
});
}
a.save().then(function() {
fs.unlink(localFilePath, function (err) {
payloadCallback(null, a);
});
});
});
}

43
helpers/importer.js
View File

@@ -5,7 +5,12 @@ const config = require('config')
const fs = require('fs')
const path = require('path')
require('../models/schema')
const db = require('../models/db')
const Sequelize = require('sequelize')
const Op = Sequelize.Op
const uuidv4 = require('uuid/v4')
require('../models/db')
module.exports = {
importZIP: function(user, zipPath) {
@@ -54,8 +59,8 @@ module.exports = {
let artifacts = JSON.parse(fs.readFileSync(importDir+'/'+space._id+'_artifacts.json'))
console.log('[import] space',space._id,'artifacts:',artifacts.length)
let q = {_id: space._id}
space.creator = user._id
//let q = {where: {_id: space._id}}
space.creator_id = user._id
delete space.__v
// transplant homefolder
@@ -64,17 +69,35 @@ module.exports = {
space.parent_space_id = user.home_folder_id
}
Space.findOneAndUpdate(q, space, {upsert: true}, function(err,res) {
if (err) console.log("[import] space upsert err:",err)
})
// move nested attrs
console.log(space)
for (k in space.advanced) {
space[k] = space.advanced[k]
}
db.Space.create(space)
.error((err) => {
console.error("[import] space upsert err:",err)
})
for (var j=0; j<artifacts.length; j++) {
let a = artifacts[j]
let q = {_id: a._id}
a.creator = user._id
a.user_id = user._id
delete a.__v
delete a.payload_thumbnail_big_uri
// move nested attrs
for (k in a.style) {
a[k] = a.style[k]
}
for (k in a.meta) {
a[k] = a.meta[k]
}
for (k in a.board) {
a[k] = a.board[k]
}
let prefix = "/storage/"+relativeImportDir+"/"+space._id+"_files/"
if (a.thumbnail_uri && a.thumbnail_uri[0]!='/') a.thumbnail_uri = prefix + a.thumbnail_uri
@@ -92,8 +115,10 @@ module.exports = {
}
}
Artifact.findOneAndUpdate(q, a, {upsert: true}, function(err,res) {
if (err) console.log("[import] artifact upsert err:",err)
db.packArtifact(a)
db.Artifact.create(a).error(function(err) {
console.error("[import] artifact upsert err:",err)
})
}
}

8
helpers/mailer.js
View File

@@ -1,7 +1,7 @@
'use strict';
var swig = require('swig');
var AWS = require('aws-sdk');
//var AWS = require('aws-sdk');
module.exports = {
sendMail: (to_email, subject, body, options) => {
@@ -29,9 +29,9 @@ module.exports = {
options: options
});
if (process.env.NODE_ENV === 'development') {
//if (process.env.NODE_ENV === 'development') {
console.log("Email: to " + to_email + " in production.\nreply_to: " + reply_to + "\nsubject: " + subject + "\nbody: \n" + htmlText + "\n\n plaintext:\n" + plaintext);
} else {
/*} else {
AWS.config.update({region: 'eu-west-1'});
var ses = new AWS.SES();
@@ -56,6 +56,6 @@ module.exports = {
if (err) console.error("Error sending email:", err);
else console.log("Email sent.");
});
}
}*/
}
};

21
helpers/phantom.js
View File

@@ -1,8 +1,9 @@
'use strict';
require('../models/schema');
var config = require('config');
var phantom = require('node-phantom-simple');
const db = require('../models/db');
const config = require('config');
const phantom = require('node-phantom-simple');
const os = require('os');
module.exports = {
// type = "pdf" or "png"
@@ -10,7 +11,7 @@ module.exports = {
var spaceId = space._id;
var space_url = config.get("endpoint")+"/api/spaces/"+spaceId+"/html";
var export_path = "/tmp/"+spaceId+"."+type;
var export_path = os.tmpdir()+"/"+spaceId+"."+type;
var timeout = 5000;
if (type=="pdf") timeout = 30000;
@@ -24,7 +25,7 @@ module.exports = {
var on_exit = function(exit_code) {
if (exit_code>0) {
console.log("phantom abnormal exit for url "+space_url);
console.error("phantom abnormal exit for url "+space_url);
if (!on_success_called && on_error) {
on_error();
}
@@ -32,16 +33,16 @@ module.exports = {
};
phantom.create({ path: require('phantomjs-prebuilt').path }, function (err, browser) {
if(err){
console.log(err);
}else{
if (err) {
console.error(err);
} else {
return browser.createPage(function (err, page) {
console.log("page created, opening ",space_url);
if (type=="pdf") {
var psz = {
width: space.advanced.width+"px",
height: space.advanced.height+"px"
width: space.width+"px",
height: space.height+"px"
};
page.set('paperSize', psz);
}

28
helpers/websockets.js
View File

@@ -1,14 +1,16 @@
'use strict';
require('../models/schema');
const db = require('../models/db');
const Sequelize = require('sequelize');
const Op = Sequelize.Op;
const config = require('config');
const WebSocketServer = require('ws').Server;
const RedisConnection = require('ioredis');
//const RedisConnection = require('ioredis');
const async = require('async');
const _ = require("underscore");
const mongoose = require("mongoose");
const crypto = require('crypto');
const redisMock = require("./redis.js");
@@ -45,11 +47,11 @@ module.exports = {
const editorAuth = msg.editor_auth;
const spaceId = msg.space_id;
Space.findOne({"_id": spaceId}).populate('creator').exec((err, space) => {
db.Space.findOne({where: {"_id": spaceId}}).then(space => {
if (space) {
const upgradeSocket = function() {
if (token) {
User.findBySessionToken(token, function(err, user) {
db.findUserBySessionToken(token, function(err, user) {
if (err) {
console.error(err, user);
} else {
@@ -268,10 +270,10 @@ module.exports = {
},
distributeUsers: function(spaceId) {
if(!spaceId)
if (!spaceId)
return;
this.state.smembers("space_" + spaceId, function(err, list) {
/*this.state.smembers("space_" + spaceId, function(err, list) {
async.map(list, function(item, callback) {
this.state.get(item, function(err, userId) {
console.log(item, "->", userId);
@@ -292,16 +294,14 @@ module.exports = {
return {nickname: realNickname, email: null, avatar_thumbnail_uri: null };
});
User.find({"_id" : { "$in" : validUserIds }}, { "nickname" : 1 , "email" : 1, "avatar_thumbnail_uri": 1 }, function(err, users) {
if (err)
console.error(err);
else {
db.User.findAll({where: {
"_id" : { "$in" : validUserIds }}, attributes: ["nickname","email","avatar_thumbnail_uri"]})
.then(users) {
const allUsers = users.concat(anonymousUsers);
const strUsers = JSON.stringify({users: allUsers, space_id: spaceId});
this.state.publish("users", strUsers);
}
}.bind(this));
}.bind(this));
}.bind(this));
}.bind(this));
}.bind(this));*/
}
};