jschaufuss/DiceCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Began work on moving file storage to s3, not working yet

Browse Source
This commit is contained in:
Stefan Zermatten
2022-01-19 16:01:07 +02:00
parent 90820452af
commit 2abaa86795
5 changed files with 313 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
app/imports/api/creature/archive/ArchiveCreatureFiles.js
View File

@@ -1,8 +1,7 @@
import { FilesCollection } from 'meteor/ostrio:files';
import { createS3FilesCollection } from '/imports/api/files/s3FileStorage.js';
const ArchiveCreatureFiles = new FilesCollection({
const ArchiveCreatureFiles = createS3FilesCollection({
collectionName: 'archiveCreatureFiles',
allowClientCode: false, // Disallow remove files from Client
storagePath: '/DiceCloud/archiveCreatures/',
onBeforeUpload(file) {
// Allow upload files under 10MB, and only in json format

215
app/imports/api/files/s3FileStorage.js Normal file
View File

@@ -0,0 +1,215 @@
// https://github.com/VeliovGroup/Meteor-Files/blob/master/docs/aws-s3-integration.md
import { Meteor } from 'meteor/meteor';
import { each, clone } from 'lodash';
import { Random } from 'meteor/random';
import { FilesCollection } from 'meteor/ostrio:files';
import stream from 'stream';
import S3 from 'aws-sdk/clients/s3'; /* http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html */
/* See fs-extra and graceful-fs NPM packages */
/* For better i/o performance */
import fs from 'fs';
/* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "endpoint": "xxx""}}' meteor */
if (process.env.S3) {
Meteor.settings.s3 = JSON.parse(process.env.S3).s3;
}
const s3Conf = Meteor.settings.s3 || {};
const bound = Meteor.bindEnvironment((callback) => {
return callback();
});
let createS3FilesCollection;
/* Check settings existence in `Meteor.settings` */
/* This is the best practice for app security */
if (Meteor.isServer && s3Conf && s3Conf.key && s3Conf.secret && s3Conf.bucket && s3Conf.endpoint) {
// Create a new S3 object
const s3 = new S3({
accessKeyId: s3Conf.key,
secretAccessKey: s3Conf.secret,
endpoint: s3Conf.endpoint,
sslEnabled: true, // optional
httpOptions: {
timeout: 6000,
agent: false
}
});
createS3FilesCollection = function({
collectionName,
storagePath,
onBeforeUpload,
debug = Meteor.isProduction,
allowClientCode = false,
}){
const collection = new FilesCollection({
collectionName,
storagePath,
onBeforeUpload,
onAfterUpload(fileRef){
// Start moving files to AWS:S3
// after fully received by the Meteor server
// Run through each of the uploaded file
each(fileRef.versions, (vRef, version) => {
// We use Random.id() instead of real file's _id
// to secure files from reverse engineering on the AWS client
const filePath = 'files/' + (Random.id()) + '-' + version + '.' + fileRef.extension;
// Create the AWS:S3 object.
// Feel free to change the storage class from, see the documentation,
// `STANDARD_IA` is the best deal for low access files.
// Key is the file name we are creating on AWS:S3, so it will be like files/XXXXXXXXXXXXXXXXX-original.XXXX
// Body is the file stream we are sending to AWS
s3.putObject({
// ServerSideEncryption: 'AES256', // Optional
StorageClass: 'STANDARD',
Bucket: s3Conf.bucket,
Key: filePath,
Body: fs.createReadStream(vRef.path),
ContentType: vRef.type,
}, (error) => {
bound(() => {
if (error) {
console.error(error);
} else {
// Update FilesCollection with link to the file at AWS
const upd = { $set: {} };
upd['$set']['versions.' + version + '.meta.pipePath'] = filePath;
this.collection.update({
_id: fileRef._id
}, upd, (updError) => {
if (updError) {
console.error(updError);
} else {
// Unlink original files from FS after successful upload to AWS:S3
this.unlink(this.collection.findOne(fileRef._id), version);
}
});
}
});
});
});
},
interceptDownload(http, fileRef, version) {
// Intercept access to the file
// And redirect request to AWS:S3
let path;
if (fileRef && fileRef.versions && fileRef.versions[version] && fileRef.versions[version].meta && fileRef.versions[version].meta.pipePath) {
path = fileRef.versions[version].meta.pipePath;
}
if (path) {
// If file is successfully moved to AWS:S3
// We will pipe request to AWS:S3
// So, original link will stay always secure
// To force ?play and ?download parameters
// and to keep original file name, content-type,
// content-disposition, chunked "streaming" and cache-control
// we're using low-level .serve() method
const opts = {
Bucket: s3Conf.bucket,
Key: path
};
if (http.request.headers.range) {
const vRef = fileRef.versions[version];
let range = clone(http.request.headers.range);
const array = range.split(/bytes=([0-9]*)-([0-9]*)/);
const start = parseInt(array[1]);
let end = parseInt(array[2]);
if (isNaN(end)) {
// Request data from AWS:S3 by small chunks
end = (start + this.chunkSize) - 1;
if (end >= vRef.size) {
end = vRef.size - 1;
}
}
opts.Range = `bytes=${start}-${end}`;
http.request.headers.range = `bytes=${start}-${end}`;
}
const fileColl = this;
s3.getObject(opts, function (error) {
if (error) {
console.error(error);
if (!http.response.finished) {
http.response.end();
}
} else {
if (http.request.headers.range && this.httpResponse.headers['content-range']) {
// Set proper range header in according to what is returned from AWS:S3
http.request.headers.range = this.httpResponse.headers['content-range'].split('/')[0].replace('bytes ', 'bytes=');
}
const dataStream = new stream.PassThrough();
fileColl.serve(http, fileRef, fileRef.versions[version], version, dataStream);
dataStream.end(this.data.Body);
}
});
return true;
}
// While file is not yet uploaded to AWS:S3
// It will be served file from FS
return false;
},
debug,
allowClientCode,
});
// Intercept FilesCollection's remove method to remove file from AWS:S3
const _origRemove = collection.remove;
collection.remove = function (search) {
const cursor = this.collection.find(search);
cursor.forEach((fileRef) => {
each(fileRef.versions, (vRef) => {
if (vRef && vRef.meta && vRef.meta.pipePath) {
// Remove the object from AWS:S3 first, then we will call the original FilesCollection remove
s3.deleteObject({
Bucket: s3Conf.bucket,
Key: vRef.meta.pipePath,
}, (error) => {
bound(() => {
if (error) {
console.error(error);
}
});
});
}
});
});
//remove original file from database
_origRemove.call(this, search);
};
return collection;
}
} else {
if (Meteor.isServer){
console.log('No S3 details specified, files will be stored in the local filesystem');
}
createS3FilesCollection = function({
collectionName,
storagePath,
onBeforeUpload,
debug = Meteor.isProduction,
allowClientCode = false,
}){
return new FilesCollection({
collectionName,
storagePath,
onBeforeUpload,
debug,
allowClientCode,
});
}
}
export { createS3FilesCollection };

2
app/imports/ui/pages/Admin.vue
View File

@@ -6,7 +6,7 @@
<v-card-text>
<h4>Current database version: {{ versions && versions.dbVersion }}</h4>
<h4 v-if="schemaVersion == versions.dbVersion ">
Database is up to date with latest version
Database is up to date with latest version. Restart to enable navigation.
</h4>
<h4 v-else>
Expected database version: {{ schemaVersion }}

95
app/package-lock.json generated
View File

@@ -278,6 +278,29 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sdk": {
"version": "2.1059.0",
"resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1059.0.tgz",
"integrity": "sha512-Q+6T9kpO6aobUNboTOk9MVAmWbs/KK0pxgCNFK0M8YO+7EWUFkNOLHM9tdYOP5vsJK5pLz6D2t2w3lHQjKzGlg==",
"requires": {
"buffer": "4.9.2",
"events": "1.1.1",
"ieee754": "1.1.13",
"jmespath": "0.15.0",
"querystring": "0.2.0",
"sax": "1.2.1",
"url": "0.10.3",
"uuid": "3.3.2",
"xml2js": "0.4.19"
},
"dependencies": {
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
}
}
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@@ -309,6 +332,11 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"bcrypt": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/bcrypt/-/bcrypt-5.0.1.tgz",
@@ -357,6 +385,16 @@
"fill-range": "^7.0.1"
}
},
"buffer": {
"version": "4.9.2",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz",
"integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4",
"isarray": "^1.0.0"
}
},
"buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@@ -937,6 +975,11 @@
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="
},
"events": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
"integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
},
"extend": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -1170,6 +1213,11 @@
"sshpk": "^1.7.0"
}
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
},
"ignore": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz",
@@ -1275,6 +1323,11 @@
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jmespath": {
"version": "0.15.0",
"resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz",
"integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc="
},
"js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -2540,6 +2593,11 @@
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
},
"querystring": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
"integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
},
"railroad-diagrams": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/railroad-diagrams/-/railroad-diagrams-1.0.0.tgz",
@@ -2677,6 +2735,11 @@
"source-map-js": ">=0.6.2 <2.0.0"
}
},
"sax": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz",
"integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o="
},
"semver": {
"version": "7.3.5",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz",
@@ -2712,7 +2775,7 @@
},
"signal-exit": {
"version": "3.0.2",
"resolved": "",
"resolved": false,
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
},
"simpl-schema": {
@@ -3025,6 +3088,22 @@
"punycode": "^2.1.0"
}
},
"url": {
"version": "0.10.3",
"resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz",
"integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=",
"requires": {
"punycode": "1.3.2",
"querystring": "0.2.0"
},
"dependencies": {
"punycode": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
"integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
}
}
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -3236,6 +3315,20 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-7.5.3.tgz",
"integrity": "sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg=="
},
"xml2js": {
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
"requires": {
"sax": ">=0.6.0",
"xmlbuilder": "~9.0.1"
}
},
"xmlbuilder": {
"version": "9.0.7",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
"integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0="
},
"y18n": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz",

1
app/package.json
View File

@@ -21,6 +21,7 @@
"@chenfengyuan/vue-countdown": "^1.1.5",
"@tozd/vue-observer-utils": "^0.5.0",
"animejs": "^2.2.0",
"aws-sdk": "^2.1059.0",
"bcrypt": "^5.0.0",
"core-js": "^2.6.11",
"css-box-shadow": "^1.0.0-3",