From 665a372f5ef7d7076acd4b1329a34b31e7de1cce Mon Sep 17 00:00:00 2001 From: oat Date: Wed, 2 Sep 2020 19:03:49 +0300 Subject: [PATCH 1/2] we uploadin files now --- package-lock.json | 29 +++++++++++++++++++++++++++++ package.json | 1 + src/html/upload.html | 23 +++++++++++++++-------- src/index.ts | 22 +++++----------------- src/lib/util.ts | 10 ++++++++++ src/upload.ts | 11 +++++++++++ 6 files changed, 71 insertions(+), 25 deletions(-) create mode 100644 src/lib/util.ts create mode 100644 src/upload.ts diff --git a/package-lock.json b/package-lock.json index bbed7f8..15b608d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -483,6 +483,14 @@ "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz", "integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q==" }, + "busboy": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz", + "integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==", + "requires": { + "dicer": "0.3.0" + } + }, "bytes": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", @@ -665,6 +673,14 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "dicer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", + "integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==", + "requires": { + "streamsearch": "0.1.2" + } + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -922,6 +938,14 @@ "vary": "~1.1.2" } }, + "express-fileupload": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.2.0.tgz", + "integrity": "sha512-oe4WpKcSppXnl5peornawWUa6tKmIc1/kJxMNRGJR1A0v4zyLL6VsFR6wZ8P2a4Iq3aGx8xae3Vlr+MOMQhFPw==", + "requires": { + "busboy": "^0.3.1" + } + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1812,6 +1836,11 @@ "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, "string-width": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", diff --git a/package.json b/package.json index 804b78b..74fb601 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@types/express": "github:types/express", "@types/mongoose": "^5.7.36", "express": "^4.17.1", + "express-fileupload": "^1.2.0", "mongoose": "^5.10.2", "mongoose-int32": "^0.4.1", "typescript": "^4.0.2", diff --git a/src/html/upload.html b/src/html/upload.html index 8670e9e..e118003 100644 --- a/src/html/upload.html +++ b/src/html/upload.html @@ -1,16 +1,23 @@ file upload + -
-
-
-
-
-
-
- + +
+
\ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 0596a04..16d87d1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,10 +2,13 @@ import * as express from 'express'; import * as mongoose from 'mongoose'; import * as fs from 'fs'; import * as winston from 'winston'; +import * as fileUpload from 'express-fileupload'; import * as format from './lib/format'; import { File } from './schema'; +import * as upload from './upload'; + const config = JSON.parse(fs.readFileSync('./config/config.json', {encoding: 'utf8'})); const db = mongoose.connect(`${config.dbconnectionURL}/${config.dbname}`, { @@ -37,15 +40,6 @@ const logger = winston.createLogger({ ] }); -function returnStatic(page) { - return (req, res) => { - fs.readFile(`src/html/${page}`, 'utf8', (err, data) => { - if (err) throw err; - res.send(data); - }); - }; -} - logger.info('connecting to mongodb database'); db.then(() => { logger.info('connected to database!'); @@ -54,19 +48,13 @@ db.then(() => { // @ts-ignore app.use(express.urlencoded({extended: true})); + app.use(fileUpload({limits: { fileSize: 50 * 1024 * 1024 }})); app.set('db', db); app.set('config', config); app.set('logger', logger); - app.get('/upload', returnStatic('upload.html')); - - app.post('/upload', async (req, res) => { // only for testing, very abusable - const file = new File(req.body); - await file.save(); - - res.send('uploaded file'); - }); + upload.run(app); app.get('/list', async (req, res) => { // only for testing const docs = await File.find({}); diff --git a/src/lib/util.ts b/src/lib/util.ts new file mode 100644 index 0000000..77a0c60 --- /dev/null +++ b/src/lib/util.ts @@ -0,0 +1,10 @@ +import * as fs from 'fs'; + +export function returnStatic(page) { + return (req, res) => { + fs.readFile(`src/html/${page}`, 'utf8', (err, data) => { + if (err) throw err; + res.send(data); + }); + }; +} \ No newline at end of file diff --git a/src/upload.ts b/src/upload.ts new file mode 100644 index 0000000..b6a686a --- /dev/null +++ b/src/upload.ts @@ -0,0 +1,11 @@ +import { returnStatic } from './lib/util'; + +export function run(app) { + app.get('/upload', returnStatic('upload.html')); + + app.post('/upload', async (req, res) => { // only for testing, very abusable + console.log(req.files); + + res.send('ok, i got something'); + }); +} \ No newline at end of file From 108eec139d961e7549ab0572be5a56d2cd774eb8 Mon Sep 17 00:00:00 2001 From: oat Date: Wed, 2 Sep 2020 21:30:23 +0300 Subject: [PATCH 2/2] file parsing!! you can now just drop in a zip in /upload and it'll parse, handle and upload everything for you right now only the metadata is saved, so the actual file downloads aren't available, but that shouldn't be too hard to implement this is really hacky however, and dropping in a file with a malformed .sm or 2 .sms will likely make it have a panic attack definetly needs lots and lots of testing, it barely works --- package-lock.json | 5 ++++ package.json | 1 + src/index.ts | 16 ++++++------- src/lib/smparse.ts | 57 ++++++++++++++++++++++++++++++++++++++++++++++ src/schema.ts | 13 ++++++----- src/upload.ts | 48 +++++++++++++++++++++++++++++++++++--- 6 files changed, 122 insertions(+), 18 deletions(-) create mode 100644 src/lib/smparse.ts diff --git a/package-lock.json b/package-lock.json index 15b608d..9ec1de8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1499,6 +1499,11 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "node-stream-zip": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.11.3.tgz", + "integrity": "sha512-GY+9LxkQuIT3O7K8BTdHVGKFcBYBy2vAVcTBtkKpu+OlBef/NSb6VuIWSyLiVDfmLMkggHeRJZN0F3W0GWU/uw==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", diff --git a/package.json b/package.json index 74fb601..b3188f9 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "express-fileupload": "^1.2.0", "mongoose": "^5.10.2", "mongoose-int32": "^0.4.1", + "node-stream-zip": "^1.11.3", "typescript": "^4.0.2", "winston": "^3.3.3" }, diff --git a/src/index.ts b/src/index.ts index 16d87d1..cc1f8e9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -58,15 +58,13 @@ db.then(() => { app.get('/list', async (req, res) => { // only for testing const docs = await File.find({}); - const map = docs.map(d => { - const keys = Object.keys(d.toObject()).filter(k => !k.startsWith('_')); - const obj = {}; - for (const key of keys) obj[key] = d[key]; - - return obj; - }); - - res.send(JSON.stringify(map)); + + res.send(docs.map((doc: any) => + `${doc.artist} - ${doc.title} by ${doc.credit}
` + + doc.charts.map(ch => + `- ${ch.difficulty} ${ch.rating}: ${ch.name}` + ).join('
') + ).join('

')); }); app.get('/', (req, res) => { diff --git a/src/lib/smparse.ts b/src/lib/smparse.ts new file mode 100644 index 0000000..6736c6e --- /dev/null +++ b/src/lib/smparse.ts @@ -0,0 +1,57 @@ +export function parseSM(data: string) { + data = data.replace(/[\n\r]/g,''); + + // steps + const difficulties = []; + const steps = data.split('#NOTES:'); + steps.slice(1); + + for (const step of steps) { + if (step.includes(';')) { + const diff: any = {}; + const stepsSplit = step.split(';')[0].split(':').map(seg => seg.trim()); + + if (stepsSplit.length === 6) { + diff.type = stepsSplit[0]; + diff.name = stepsSplit[1]; + diff.difficulty = stepsSplit[2]; + diff.rating = Number(stepsSplit[3]); + diff.radarvalues = stepsSplit[4].split(',').map(v => Number(v)); + + difficulties.push(diff); + } + } + } + + // metadata + const lines = data.split(';').filter(l => l.startsWith('#')); + const obj: any = {}; + + for (const l of lines) { + const key = l.split(':')[0].slice(1); + let value: any = l.split(':')[1]; + + if (value !== '' && !(key === 'FGCHANGES' || key === 'BGCHANGES' || key === 'BETTERBGCHANGES' || key === 'SPELLCARDS')) { // no + if (!isNaN(value)) value = Number(value); + if (value === 'YES') value = true; + if (value === 'NO') value = false; + if (typeof value === 'string' && value.includes('=')) { // likely a map + const keys = value.split(',').map(v => v.split('=')[0]); + const values = value.split(',').map(v => v.split('=')[1]); + const map = {}; + + for (const i in keys) { + map[Number(keys[i])] = Number(values[i]); // afaik maps are only numbers? + } + + value = map; + } + + obj[key.toLowerCase()] = value; + } + } + + obj.charts = difficulties; + + return obj; +} \ No newline at end of file diff --git a/src/schema.ts b/src/schema.ts index df9bec1..2bf463c 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -1,17 +1,18 @@ import * as mongoose from 'mongoose'; -import * as Int32 from 'mongoose-int32'; const Schema = mongoose.Schema; const Sample = new Schema({ - start: {type: Int32, default: new Int32(0)}, - length: {type: Int32, default: new Int32(0)} + start: {type: Number, default: 0}, + length: {type: Number, default: 0} }); const Chart = new Schema({ + type: {type: String, default: 'dance-single'}, name: {type: String, default: ''}, - rating: {type: Int32, default: new Int32(0)}, - type: {type: String, default: 'Challenge'} + difficulty: {type: String, default: 'Challenge'}, + rating: {type: Number, default: 0}, + radarvalues: [Number] }); const FileSchema = new Schema({ @@ -24,7 +25,7 @@ const FileSchema = new Schema({ credit: String, uploader: {type: String, default: '00000000-0000-4000-a000-000000000000'}, sample: Sample, - bpms: {type: [Number], default: 0}, + bpms: {type: Object, default: {'0': 0}}, charts: [Chart] }); diff --git a/src/upload.ts b/src/upload.ts index b6a686a..d8b94c6 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -1,11 +1,53 @@ +import { tmpdir } from 'os'; +import * as fs from 'fs'; +const StreamZip = require('node-stream-zip'); + import { returnStatic } from './lib/util'; +import { parseSM } from './lib/smparse'; +import { File } from './schema'; export function run(app) { + const logger = app.get('logger'); + app.get('/upload', returnStatic('upload.html')); app.post('/upload', async (req, res) => { // only for testing, very abusable - console.log(req.files); - - res.send('ok, i got something'); + const file = req.files.file; + if (file.mimetype !== 'application/zip') return res.status(400).send('Invalid filetype'); + + const dir = tmpdir() + '/' + file.md5; + fs.writeFile(dir, file.data, (err) => { + if (err) throw err; + + const zip = new StreamZip({ + file: dir, + storeEntries: true + }); + + zip.on('ready', () => { + const smFile = Object.values(zip.entries()).find((f: any) => + !f.isDirectory && (f.name.endsWith('.sm')) + ); + + if (!smFile) { + res.status(400).send('No .sm found'); + } else { + const data = zip.entryDataSync((smFile as any).name); + const chart = parseSM(data.toString()); + + logger.info(`${chart.artist} - ${chart.title} was just uploaded`); + + const file = new File(chart); + file.save(); + + res.send(`your file "${chart.artist} - ${chart.title}" has been uploaded! check the listing`); // todo: change to actual url + } + + zip.close(); + fs.unlink(dir, (err) => { + if (err) throw err; + }); + }); + }); }); } \ No newline at end of file