Archives and restore now works to S3 and file system

If a file is stored on the file system and s3 settings later become 
available it is still correctly fetched from the file system.

app/imports/api/creature/archive/methods/archiveCreatureToFile.js

@@ -18,6 +18,7 @@ export function getArchiveObj(creatureId){
   const logs = CreatureLogs.find({creatureId}).fetch();
   let archiveCreature = {
     meta: {
+      type: 'DiceCloud V2 Creature Archive',
       schemaVersion: SCHEMA_VERSION,
       archiveDate: new Date(),
     },
@@ -33,7 +34,7 @@ export function getArchiveObj(creatureId){
 export function archiveCreature(creatureId){
   const archive = getArchiveObj(creatureId);
   const buffer = Buffer.from(JSON.stringify(archive, null, 2));
-  const result = ArchiveCreatureFiles.write(buffer, {
+  ArchiveCreatureFiles.write(buffer, {
     fileName: `${archive.creature.name || archive.creature._id}.json`,
     type: 'application/json',
     userId: archive.creature.owner,
@@ -42,9 +43,13 @@ export function archiveCreature(creatureId){
       creatureId: archive.creature._id,
       creatureName: archive.creature.name,
     },
-  });
-  removeCreatureWork(creatureId);
-  return result;
+  }, (error) => {
+    if (error){
+      throw error;
+    } else {
+      removeCreatureWork(creatureId);
+    }
+  }, true);
 }
 
 const archiveCreatureToFile = new ValidatedMethod({

app/imports/api/creature/archive/methods/archiveCreatures.js

@@ -1,66 +0,0 @@
-import SimpleSchema from 'simpl-schema';
-import { ValidatedMethod } from 'meteor/mdg:validated-method';
-import { RateLimiterMixin } from 'ddp-rate-limiter-mixin';
-import { assertOwnership } from '/imports/api/creature/creatures/creaturePermissions.js';
-import Creatures from '/imports/api/creature/creatures/Creatures.js';
-import CreatureProperties from '/imports/api/creature/creatureProperties/CreatureProperties.js';
-import CreatureLogs from '/imports/api/creature/log/CreatureLogs.js';
-import Experiences from '/imports/api/creature/experience/Experiences.js';
-import { removeCreatureWork } from '/imports/api/creature/creatures/methods/removeCreature.js';
-import ArchivedCreatures from '/imports/api/creature/archive/ArchivedCreatures.js';
-
-function archiveCreature(creatureId){
-  // Build the archive document
-  const creature = Creatures.findOne(creatureId);
-  const properties = CreatureProperties.find({'ancestors.id': creatureId}).fetch();
-  const experiences = Experiences.find({creatureId}).fetch();
-  const logs = CreatureLogs.find({creatureId}).fetch();
-  let archiveCreature = {
-    owner: creature.owner,
-    archiveDate: new Date(),
-    creature,
-    properties,
-    experiences,
-    logs,
-  };
-
-  // Insert it
-  let id = ArchivedCreatures.insert(archiveCreature);
-
-  // Remove the original creature
-  removeCreatureWork(creatureId);
-
-  return id;
-}
-
-const archiveCreatures = new ValidatedMethod({
-  name: 'Creatures.methods.archiveCreatures',
-  validate: new SimpleSchema({
-    creatureIds: {
-      type: Array,
-      max: 10,
-    },
-    'creatureIds.$': {
-      type: String,
-      regEx: SimpleSchema.RegEx.Id,
-    },
-  }).validator(),
-  mixins: [RateLimiterMixin],
-  rateLimit: {
-    numRequests: 1,
-    timeInterval: 5000,
-  },
-  run({creatureIds}) {
-    for (let id of creatureIds){
-      assertOwnership(id, this.userId)
-    }
-    let archivedIds = [];
-    for (let id of creatureIds){
-      let archivedId = archiveCreature(id);
-      archivedIds.push(archivedId);
-    }
-    return archivedIds;
-  },
-});
-
-export default archiveCreatures;

app/imports/api/creature/archive/methods/readFile.js

@@ -1,6 +0,0 @@
-import { promises as fs } from 'fs';
-
-// Read a file and return the result
-export default function read(file){
-  return fs.readFile(file.path, 'utf8');
-}

app/imports/api/creature/archive/methods/restoreCreatureFromFile.js

@@ -8,13 +8,12 @@ import CreatureLogs from '/imports/api/creature/log/CreatureLogs.js';
 import Experiences from '/imports/api/creature/experience/Experiences.js';
 import { removeCreatureWork } from '/imports/api/creature/creatures/methods/removeCreature.js';
 import ArchiveCreatureFiles from '/imports/api/creature/archive/ArchiveCreatureFiles.js';
-import readFile from '/imports/api/creature/archive/methods/readFile.js';
 let migrateArchive;
 if (Meteor.isServer){
   migrateArchive = require('/imports/migrations/server/migrateArchive.js').default;
 }
 
-function restoreCreature(file, archive){
+function restoreCreature(archive){
   if (SCHEMA_VERSION < archive.meta.schemaVersion){
     throw new Meteor.Error('Incompatible',
       'The archive file is from a newer version. Update required to read.')
@@ -72,12 +71,11 @@ const restoreCreaturefromFile = new ValidatedMethod({
     }
     if (Meteor.isServer){
       // Read the file data
-      const string = await readFile(file);
-      const archive = JSON.parse(string);
-      restoreCreature(file, archive);
-      //Remove the archive once the restore succeeded
-      ArchiveCreatureFiles.remove({_id: fileId})
+      const archive = await ArchiveCreatureFiles.readJSONFile(file);
+      restoreCreature(archive);
     }
+    //Remove the archive once the restore succeeded
+    ArchiveCreatureFiles.remove({_id: fileId});
   },
 });
 

app/imports/api/files/s3FileStorage.js

@@ -5,11 +5,12 @@ import { each, clone } from 'lodash';
 import { Random } from 'meteor/random';
 import { FilesCollection } from 'meteor/ostrio:files';
 import stream from 'stream';
+import S3 from 'aws-sdk/clients/s3';
 
-import S3 from 'aws-sdk/clients/s3'; /* http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html */
 /* See fs-extra and graceful-fs NPM packages */
 /* For better i/o performance */
 import fs from 'fs';
+import { promises as fsp } from 'fs';
 
 /* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "endpoint": "xxx""}}' meteor */
 if (process.env.S3) {
@@ -17,6 +18,10 @@ if (process.env.S3) {
 }
 
 const s3Conf = Meteor.settings.s3 || {};
+Meteor.settings.useS3 = !!(
+  s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.endpoint
+);
+
 const bound  = Meteor.bindEnvironment((callback) => {
   return callback();
 });
@@ -25,7 +30,7 @@ let createS3FilesCollection;
 
 /* Check settings existence in `Meteor.settings` */
 /* This is the best practice for app security */
-if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.endpoint) {
+if (Meteor.isServer && Meteor.settings.useS3) {
   // Create a new S3 object
   const s3 = new S3({
     accessKeyId: s3Conf.key,
@@ -98,7 +103,6 @@ if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket &&
       interceptDownload(http, fileRef, version) {
         // Intercept access to the file
         // And redirect request to AWS:S3
-
         let path;
 
         if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) {
@@ -189,6 +193,23 @@ if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket &&
       //remove original file from database
       _origRemove.call(this, search);
     };
+
+    collection.readJSONFile = async function(file){
+      // If there is the pipepath, use s3 to get the file
+      if (file?.versions?.original?.meta?.pipePath){
+        const path = file.versions.original.meta.pipePath;
+        const data = await s3.getObject({
+          Bucket: s3Conf.bucket,
+          Key: path
+        }).promise();
+        return JSON.parse(data.Body.toString('utf-8'));
+      } else {
+        // Otherwise use the normal filesystem
+        const fileString = await fsp.readFile(file.path, 'utf8');
+        return JSON.parse(fileString);
+      }
+    };
+
     return collection;
   }
 } else {
@@ -202,13 +223,23 @@ if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket &&
     debug = Meteor.isProduction,
     allowClientCode = false,
   }){
-    return new FilesCollection({
+    const collection = new FilesCollection({
       collectionName,
       storagePath,
       onBeforeUpload,
       debug,
       allowClientCode,
     });
+
+    if (Meteor.isServer){
+      // Use the normal file system to read files
+      collection.readJSONFile = async function(file){
+        const fileString = await fsp.readFile(file.path, 'utf8');
+        return JSON.parse(fileString);
+      };
+    }
+
+    return collection;
   }
 }