first importer WIP

This commit is contained in:
mntmn 2018-01-08 15:57:59 +01:00
parent ffb7f30133
commit 462e9edaab
4 changed files with 117 additions and 1 deletions

103
helpers/importer.js Normal file
View File

@ -0,0 +1,103 @@
'use strict';
const extract = require('extract-zip')
const config = require('config')
const fs = require('fs')
const path = require('path')
require('../models/schema')
module.exports = {
importZIP: function(user, zipPath) {
// 1. extract zip to local storage folder
// 2. read spaces.json from this folder
// 3. iterate through spaces and read all their artifact jsons
// 4. fixup storage paths
// 5. replace creator id by user._id
let relativeImportDir = 'import_'+user._id
let importDir = path.resolve(config.get('storage_local_path')+'/'+config.get('storage_bucket')+'/'+relativeImportDir)
if (!fs.existsSync(importDir)) {
fs.mkdirSync(importDir)
}
extract(zipPath, {dir: importDir}, function(err) {
if (err) {
console.log(err)
return
}
console.log('[import] extracted to',importDir)
let spacesJson = fs.readFileSync(importDir+'/spaces.json')
let spaces = JSON.parse(spacesJson)
var homeFolderId = null
console.log('[import] spaces:',spaces.length)
// pass 1: find homefolder
for (var i=0; i<spaces.length; i++) {
let space = spaces[i]
if (!space.parent_space_id) {
homeFolderId = space._id
break
}
}
console.log("[import] homeFolderId:",homeFolderId)
for (var i=0; i<spaces.length; i++) {
let space = spaces[i]
if (space.parent_space_id) {
let artifacts = JSON.parse(fs.readFileSync(importDir+'/'+space._id+'_artifacts.json'))
console.log('[import] space',space._id,'artifacts:',artifacts.length)
let q = {_id: space._id}
space.creator = user._id
delete space.__v
// transplant homefolder
console.log("parent:",space.parent_space_id)
if (space.parent_space_id+"" == homeFolderId+"") {
space.parent_space_id = user.home_folder_id
}
Space.findOneAndUpdate(q, space, {upsert: true}, function(err,res) {
if (err) console.log("[import] space upsert err:",err)
})
for (var j=0; j<artifacts.length; j++) {
let a = artifacts[j]
let q = {_id: a._id}
a.creator = user._id
delete a.__v
delete a.payload_thumbnail_big_uri
let prefix = "/storage/"+relativeImportDir+"/"+space._id+"_files/"
if (a.thumbnail_uri && a.thumbnail_uri[0]!='/') a.thumbnail_uri = prefix + a.thumbnail_uri
if (a.payload_uri && a.payload_uri[0]!='/') a.payload_uri = prefix + a.payload_uri
if (a.payload_thumbnail_web_uri && a.payload_thumbnail_web_uri[0]!='/') a.payload_thumbnail_web_uri = prefix + a.payload_thumbnail_web_uri
if (a.payload_thumbnail_medium_uri && a.payload_thumbnail_medium_uri[0]!='/') a.payload_thumbnail_medium_uri = prefix + a.payload_thumbnail_medium_uri
if (a.payload_alternatives) {
for (var k=0; k<a.payload_alternatives.length; k++) {
let alt = a.payload_alternatives[k]
if (alt.payload_uri && alt.payload_uri[0]!='/') alt.payload_uri = prefix + alt.payload_uri
if (alt.payload_thumbnail_web_uri && alt.payload_thumbnail_web_uri[0]!='/') alt.payload_thumbnail_web_uri = prefix + alt.payload_thumbnail_web_uri
if (alt.payload_thumbnail_medium_uri && alt.payload_thumbnail_medium_uri[0]!='/') alt.payload_thumbnail_medium_uri = prefix + alt.payload_thumbnail_medium_uri
}
}
Artifact.findOneAndUpdate(q, a, {upsert: true}, function(err,res) {
if (err) console.log("[import] artifact upsert err:",err)
})
}
}
}
})
}
}

View File

@ -23,6 +23,7 @@
"debug": "~2.6.3",
"execSync": "latest",
"express": "~4.13.0",
"extract-zip": "^1.6.6",
"glob": "7.1.1",
"gm": "1.23.0",
"googleapis": "18.0.0",

View File

@ -59,7 +59,9 @@ router.get('/', (req, res) => {
"nickname": 1,
"email": 1
}).exec((err, user) => {
a['user'] = user.toObject();
if (user) {
a['user'] = user.toObject();
}
cb(err, a);
});
} else {

View File

@ -5,6 +5,7 @@ require('../../models/schema');
var mailer = require('../../helpers/mailer');
var uploader = require('../../helpers/uploader');
var importer = require('../../helpers/importer');
var bcrypt = require('bcryptjs');
var crypo = require('crypto');
@ -467,4 +468,13 @@ router.post('/:user_id/confirm', function(req, res, next) {
res.sendStatus(201);
});
router.get('/:user_id/import', function(req, res, next) {
if (req.query.zip) {
res.send("importing");
importer.importZIP(req.user, req.query.zip);
} else {
res.sendStatus(400);
}
});
module.exports = router;