diff --git a/app/imports/api/creature/archive/ArchiveCreatureFiles.js b/app/imports/api/creature/archive/ArchiveCreatureFiles.js index 282bf265..f4fe867c 100644 --- a/app/imports/api/creature/archive/ArchiveCreatureFiles.js +++ b/app/imports/api/creature/archive/ArchiveCreatureFiles.js @@ -1,8 +1,7 @@ -import { FilesCollection } from 'meteor/ostrio:files'; +import { createS3FilesCollection } from '/imports/api/files/s3FileStorage.js'; -const ArchiveCreatureFiles = new FilesCollection({ +const ArchiveCreatureFiles = createS3FilesCollection({ collectionName: 'archiveCreatureFiles', - allowClientCode: false, // Disallow remove files from Client storagePath: '/DiceCloud/archiveCreatures/', onBeforeUpload(file) { // Allow upload files under 10MB, and only in json format diff --git a/app/imports/api/files/s3FileStorage.js b/app/imports/api/files/s3FileStorage.js new file mode 100644 index 00000000..f3e0c0eb --- /dev/null +++ b/app/imports/api/files/s3FileStorage.js @@ -0,0 +1,215 @@ +// https://github.com/VeliovGroup/Meteor-Files/blob/master/docs/aws-s3-integration.md + +import { Meteor } from 'meteor/meteor'; +import { each, clone } from 'lodash'; +import { Random } from 'meteor/random'; +import { FilesCollection } from 'meteor/ostrio:files'; +import stream from 'stream'; + +import S3 from 'aws-sdk/clients/s3'; /* http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html */ +/* See fs-extra and graceful-fs NPM packages */ +/* For better i/o performance */ +import fs from 'fs'; + +/* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "endpoint": "xxx""}}' meteor */ +if (process.env.S3) { + Meteor.settings.s3 = JSON.parse(process.env.S3).s3; +} + +const s3Conf = Meteor.settings.s3 || {}; +const bound = Meteor.bindEnvironment((callback) => { + return callback(); +}); + +let createS3FilesCollection; + +/* Check settings existence in `Meteor.settings` */ +/* This is the best practice for app security */ +if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.endpoint) { + // Create a new S3 object + const s3 = new S3({ + accessKeyId: s3Conf.key, + secretAccessKey: s3Conf.secret, + endpoint: s3Conf.endpoint, + sslEnabled: true, // optional + httpOptions: { + timeout: 6000, + agent: false + } + }); + + createS3FilesCollection = function({ + collectionName, + storagePath, + onBeforeUpload, + debug = Meteor.isProduction, + allowClientCode = false, + }){ + const collection = new FilesCollection({ + collectionName, + storagePath, + onBeforeUpload, + onAfterUpload(fileRef){ + // Start moving files to AWS:S3 + // after fully received by the Meteor server + + // Run through each of the uploaded file + each(fileRef.versions, (vRef, version) => { + // We use Random.id() instead of real file's _id + // to secure files from reverse engineering on the AWS client + const filePath = 'files/' + (Random.id()) + '-' + version + '.' + fileRef.extension; + + // Create the AWS:S3 object. + // Feel free to change the storage class from, see the documentation, + // `STANDARD_IA` is the best deal for low access files. + // Key is the file name we are creating on AWS:S3, so it will be like files/XXXXXXXXXXXXXXXXX-original.XXXX + // Body is the file stream we are sending to AWS + s3.putObject({ + // ServerSideEncryption: 'AES256', // Optional + StorageClass: 'STANDARD', + Bucket: s3Conf.bucket, + Key: filePath, + Body: fs.createReadStream(vRef.path), + ContentType: vRef.type, + }, (error) => { + bound(() => { + if (error) { + console.error(error); + } else { + // Update FilesCollection with link to the file at AWS + const upd = { $set: {} }; + upd['$set']['versions.' + version + '.meta.pipePath'] = filePath; + + this.collection.update({ + _id: fileRef._id + }, upd, (updError) => { + if (updError) { + console.error(updError); + } else { + // Unlink original files from FS after successful upload to AWS:S3 + this.unlink(this.collection.findOne(fileRef._id), version); + } + }); + } + }); + }); + }); + }, + interceptDownload(http, fileRef, version) { + // Intercept access to the file + // And redirect request to AWS:S3 + + let path; + + if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) { + path = fileRef.versions[version].meta.pipePath; + } + + if (path) { + // If file is successfully moved to AWS:S3 + // We will pipe request to AWS:S3 + // So, original link will stay always secure + + // To force ?play and ?download parameters + // and to keep original file name, content-type, + // content-disposition, chunked "streaming" and cache-control + // we're using low-level .serve() method + const opts = { + Bucket: s3Conf.bucket, + Key: path + }; + + if (http.request.headers.range) { + const vRef = fileRef.versions[version]; + let range = clone(http.request.headers.range); + const array = range.split(/bytes=([0-9]*)-([0-9]*)/); + const start = parseInt(array[1]); + let end = parseInt(array[2]); + if (isNaN(end)) { + // Request data from AWS:S3 by small chunks + end = (start + this.chunkSize) - 1; + if (end >= vRef.size) { + end = vRef.size - 1; + } + } + opts.Range = `bytes=${start}-${end}`; + http.request.headers.range = `bytes=${start}-${end}`; + } + + const fileColl = this; + s3.getObject(opts, function (error) { + if (error) { + console.error(error); + if (!http.response.finished) { + http.response.end(); + } + } else { + if (http.request.headers.range && this.httpResponse.headers['content-range']) { + // Set proper range header in according to what is returned from AWS:S3 + http.request.headers.range = this.httpResponse.headers['content-range'].split('/')[0].replace('bytes ', 'bytes='); + } + + const dataStream = new stream.PassThrough(); + fileColl.serve(http, fileRef, fileRef.versions[version], version, dataStream); + dataStream.end(this.data.Body); + } + }); + + return true; + } + // While file is not yet uploaded to AWS:S3 + // It will be served file from FS + return false; + }, + debug, + allowClientCode, + }); + // Intercept FilesCollection's remove method to remove file from AWS:S3 + const _origRemove = collection.remove; + collection.remove = function (search) { + const cursor = this.collection.find(search); + cursor.forEach((fileRef) => { + each(fileRef.versions, (vRef) => { + if (vRef && vRef.meta && vRef.meta.pipePath) { + // Remove the object from AWS:S3 first, then we will call the original FilesCollection remove + s3.deleteObject({ + Bucket: s3Conf.bucket, + Key: vRef.meta.pipePath, + }, (error) => { + bound(() => { + if (error) { + console.error(error); + } + }); + }); + } + }); + }); + + //remove original file from database + _origRemove.call(this, search); + }; + return collection; + } +} else { + if (Meteor.isServer){ + console.log('No S3 details specified, files will be stored in the local filesystem'); + } + createS3FilesCollection = function({ + collectionName, + storagePath, + onBeforeUpload, + debug = Meteor.isProduction, + allowClientCode = false, + }){ + return new FilesCollection({ + collectionName, + storagePath, + onBeforeUpload, + debug, + allowClientCode, + }); + } +} + +export { createS3FilesCollection }; diff --git a/app/imports/ui/pages/Admin.vue b/app/imports/ui/pages/Admin.vue index 5143829f..bb17f090 100644 --- a/app/imports/ui/pages/Admin.vue +++ b/app/imports/ui/pages/Admin.vue @@ -6,7 +6,7 @@

Current database version: {{ versions && versions.dbVersion }}

- Database is up to date with latest version + Database is up to date with latest version. Restart to enable navigation.

Expected database version: {{ schemaVersion }} diff --git a/app/package-lock.json b/app/package-lock.json index 2c31e8a5..26584fab 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -278,6 +278,29 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, + "aws-sdk": { + "version": "2.1059.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1059.0.tgz", + "integrity": "sha512-Q+6T9kpO6aobUNboTOk9MVAmWbs/KK0pxgCNFK0M8YO+7EWUFkNOLHM9tdYOP5vsJK5pLz6D2t2w3lHQjKzGlg==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + } + } + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -309,6 +332,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "bcrypt": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-5.0.1.tgz", @@ -357,6 +385,16 @@ "fill-range": "^7.0.1" } }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -937,6 +975,11 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -1170,6 +1213,11 @@ "sshpk": "^1.7.0" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", @@ -1275,6 +1323,11 @@ "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2540,6 +2593,11 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "railroad-diagrams": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz", @@ -2677,6 +2735,11 @@ "source-map-js": ">=0.6.2 <2.0.0" } }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, "semver": { "version": "7.3.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", @@ -2712,7 +2775,7 @@ }, "signal-exit": { "version": "3.0.2", - "resolved": "", + "resolved": false, "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" }, "simpl-schema": { @@ -3025,6 +3088,22 @@ "punycode": "^2.1.0" } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -3236,6 +3315,20 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz", "integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==" }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", diff --git a/app/package.json b/app/package.json index 94b647cb..c71d991d 100644 --- a/app/package.json +++ b/app/package.json @@ -21,6 +21,7 @@ "@chenfengyuan/vue-countdown": "^1.1.5", "@tozd/vue-observer-utils": "^0.5.0", "animejs": "^2.2.0", + "aws-sdk": "^2.1059.0", "bcrypt": "^5.0.0", "core-js": "^2.6.11", "css-box-shadow": "^1.0.0-3",