From 78c313e3d12abdab9d2e33ad0e3d73a5d9218ab2 Mon Sep 17 00:00:00 2001 From: Stefan Zermatten Date: Thu, 3 Feb 2022 11:48:03 +0200 Subject: [PATCH] Archives and restore now works to S3 and file system If a file is stored on the file system and s3 settings later become available it is still correctly fetched from the file system. --- .../archive/methods/archiveCreatureToFile.js | 13 ++-- .../archive/methods/archiveCreatures.js | 66 ------------------- .../api/creature/archive/methods/readFile.js | 6 -- .../methods/restoreCreatureFromFile.js | 12 ++-- app/imports/api/files/s3FileStorage.js | 39 +++++++++-- 5 files changed, 49 insertions(+), 87 deletions(-) delete mode 100644 app/imports/api/creature/archive/methods/archiveCreatures.js delete mode 100644 app/imports/api/creature/archive/methods/readFile.js diff --git a/app/imports/api/creature/archive/methods/archiveCreatureToFile.js b/app/imports/api/creature/archive/methods/archiveCreatureToFile.js index 1d15e85f..7599299c 100644 --- a/app/imports/api/creature/archive/methods/archiveCreatureToFile.js +++ b/app/imports/api/creature/archive/methods/archiveCreatureToFile.js @@ -18,6 +18,7 @@ export function getArchiveObj(creatureId){ const logs = CreatureLogs.find({creatureId}).fetch(); let archiveCreature = { meta: { + type: 'DiceCloud V2 Creature Archive', schemaVersion: SCHEMA_VERSION, archiveDate: new Date(), }, @@ -33,7 +34,7 @@ export function getArchiveObj(creatureId){ export function archiveCreature(creatureId){ const archive = getArchiveObj(creatureId); const buffer = Buffer.from(JSON.stringify(archive, null, 2)); - const result = ArchiveCreatureFiles.write(buffer, { + ArchiveCreatureFiles.write(buffer, { fileName: `${archive.creature.name || archive.creature._id}.json`, type: 'application/json', userId: archive.creature.owner, @@ -42,9 +43,13 @@ export function archiveCreature(creatureId){ creatureId: archive.creature._id, creatureName: archive.creature.name, }, - }); - removeCreatureWork(creatureId); - return result; + }, (error) => { + if (error){ + throw error; + } else { + removeCreatureWork(creatureId); + } + }, true); } const archiveCreatureToFile = new ValidatedMethod({ diff --git a/app/imports/api/creature/archive/methods/archiveCreatures.js b/app/imports/api/creature/archive/methods/archiveCreatures.js deleted file mode 100644 index 304cce11..00000000 --- a/app/imports/api/creature/archive/methods/archiveCreatures.js +++ /dev/null @@ -1,66 +0,0 @@ -import SimpleSchema from 'simpl-schema'; -import { ValidatedMethod } from 'meteor/mdg:validated-method'; -import { RateLimiterMixin } from 'ddp-rate-limiter-mixin'; -import { assertOwnership } from '/imports/api/creature/creatures/creaturePermissions.js'; -import Creatures from '/imports/api/creature/creatures/Creatures.js'; -import CreatureProperties from '/imports/api/creature/creatureProperties/CreatureProperties.js'; -import CreatureLogs from '/imports/api/creature/log/CreatureLogs.js'; -import Experiences from '/imports/api/creature/experience/Experiences.js'; -import { removeCreatureWork } from '/imports/api/creature/creatures/methods/removeCreature.js'; -import ArchivedCreatures from '/imports/api/creature/archive/ArchivedCreatures.js'; - -function archiveCreature(creatureId){ - // Build the archive document - const creature = Creatures.findOne(creatureId); - const properties = CreatureProperties.find({'ancestors.id': creatureId}).fetch(); - const experiences = Experiences.find({creatureId}).fetch(); - const logs = CreatureLogs.find({creatureId}).fetch(); - let archiveCreature = { - owner: creature.owner, - archiveDate: new Date(), - creature, - properties, - experiences, - logs, - }; - - // Insert it - let id = ArchivedCreatures.insert(archiveCreature); - - // Remove the original creature - removeCreatureWork(creatureId); - - return id; -} - -const archiveCreatures = new ValidatedMethod({ - name: 'Creatures.methods.archiveCreatures', - validate: new SimpleSchema({ - creatureIds: { - type: Array, - max: 10, - }, - 'creatureIds.$': { - type: String, - regEx: SimpleSchema.RegEx.Id, - }, - }).validator(), - mixins: [RateLimiterMixin], - rateLimit: { - numRequests: 1, - timeInterval: 5000, - }, - run({creatureIds}) { - for (let id of creatureIds){ - assertOwnership(id, this.userId) - } - let archivedIds = []; - for (let id of creatureIds){ - let archivedId = archiveCreature(id); - archivedIds.push(archivedId); - } - return archivedIds; - }, -}); - -export default archiveCreatures; diff --git a/app/imports/api/creature/archive/methods/readFile.js b/app/imports/api/creature/archive/methods/readFile.js deleted file mode 100644 index 412bd643..00000000 --- a/app/imports/api/creature/archive/methods/readFile.js +++ /dev/null @@ -1,6 +0,0 @@ -import { promises as fs } from 'fs'; - -// Read a file and return the result -export default function read(file){ - return fs.readFile(file.path, 'utf8'); -} diff --git a/app/imports/api/creature/archive/methods/restoreCreatureFromFile.js b/app/imports/api/creature/archive/methods/restoreCreatureFromFile.js index 89347a8f..986bb9ca 100644 --- a/app/imports/api/creature/archive/methods/restoreCreatureFromFile.js +++ b/app/imports/api/creature/archive/methods/restoreCreatureFromFile.js @@ -8,13 +8,12 @@ import CreatureLogs from '/imports/api/creature/log/CreatureLogs.js'; import Experiences from '/imports/api/creature/experience/Experiences.js'; import { removeCreatureWork } from '/imports/api/creature/creatures/methods/removeCreature.js'; import ArchiveCreatureFiles from '/imports/api/creature/archive/ArchiveCreatureFiles.js'; -import readFile from '/imports/api/creature/archive/methods/readFile.js'; let migrateArchive; if (Meteor.isServer){ migrateArchive = require('/imports/migrations/server/migrateArchive.js').default; } -function restoreCreature(file, archive){ +function restoreCreature(archive){ if (SCHEMA_VERSION < archive.meta.schemaVersion){ throw new Meteor.Error('Incompatible', 'The archive file is from a newer version. Update required to read.') @@ -72,12 +71,11 @@ const restoreCreaturefromFile = new ValidatedMethod({ } if (Meteor.isServer){ // Read the file data - const string = await readFile(file); - const archive = JSON.parse(string); - restoreCreature(file, archive); - //Remove the archive once the restore succeeded - ArchiveCreatureFiles.remove({_id: fileId}) + const archive = await ArchiveCreatureFiles.readJSONFile(file); + restoreCreature(archive); } + //Remove the archive once the restore succeeded + ArchiveCreatureFiles.remove({_id: fileId}); }, }); diff --git a/app/imports/api/files/s3FileStorage.js b/app/imports/api/files/s3FileStorage.js index f3e0c0eb..43dafb33 100644 --- a/app/imports/api/files/s3FileStorage.js +++ b/app/imports/api/files/s3FileStorage.js @@ -5,11 +5,12 @@ import { each, clone } from 'lodash'; import { Random } from 'meteor/random'; import { FilesCollection } from 'meteor/ostrio:files'; import stream from 'stream'; +import S3 from 'aws-sdk/clients/s3'; -import S3 from 'aws-sdk/clients/s3'; /* http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html */ /* See fs-extra and graceful-fs NPM packages */ /* For better i/o performance */ import fs from 'fs'; +import { promises as fsp } from 'fs'; /* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "endpoint": "xxx""}}' meteor */ if (process.env.S3) { @@ -17,6 +18,10 @@ if (process.env.S3) { } const s3Conf = Meteor.settings.s3 || {}; +Meteor.settings.useS3 = !!( + s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.endpoint +); + const bound = Meteor.bindEnvironment((callback) => { return callback(); }); @@ -25,7 +30,7 @@ let createS3FilesCollection; /* Check settings existence in `Meteor.settings` */ /* This is the best practice for app security */ -if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.endpoint) { +if (Meteor.isServer && Meteor.settings.useS3) { // Create a new S3 object const s3 = new S3({ accessKeyId: s3Conf.key, @@ -98,7 +103,6 @@ if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && interceptDownload(http, fileRef, version) { // Intercept access to the file // And redirect request to AWS:S3 - let path; if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) { @@ -189,6 +193,23 @@ if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && //remove original file from database _origRemove.call(this, search); }; + + collection.readJSONFile = async function(file){ + // If there is the pipepath, use s3 to get the file + if (file?.versions?.original?.meta?.pipePath){ + const path = file.versions.original.meta.pipePath; + const data = await s3.getObject({ + Bucket: s3Conf.bucket, + Key: path + }).promise(); + return JSON.parse(data.Body.toString('utf-8')); + } else { + // Otherwise use the normal filesystem + const fileString = await fsp.readFile(file.path, 'utf8'); + return JSON.parse(fileString); + } + }; + return collection; } } else { @@ -202,13 +223,23 @@ if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && debug = Meteor.isProduction, allowClientCode = false, }){ - return new FilesCollection({ + const collection = new FilesCollection({ collectionName, storagePath, onBeforeUpload, debug, allowClientCode, }); + + if (Meteor.isServer){ + // Use the normal file system to read files + collection.readJSONFile = async function(file){ + const fileString = await fsp.readFile(file.path, 'utf8'); + return JSON.parse(fileString); + }; + } + + return collection; } }