file parsing!!

you can now just drop in a zip in /upload and it'll parse, handle and upload everything for you right now only the metadata is saved, so the actual file downloads aren't available, but that shouldn't be too hard to implement this is really hacky however, and dropping in a file with a malformed .sm or 2 .sms will likely make it have a panic attack definetly needs lots and lots of testing, it barely works
we uploadin files now
2020-09-02 21:30:23 +03:00 · 2020-09-02 19:03:49 +03:00
8 changed files with 190 additions and 40 deletions
--- a/package-lock.json
+++ b/package-lock.json
@ -483,6 +483,14 @@
      "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz",
      "integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q=="
    },
+    "busboy": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz",
+      "integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==",
+      "requires": {
+        "dicer": "0.3.0"
+      }
+    },
    "bytes": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@ -665,6 +673,14 @@
      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
      "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
    },
+    "dicer": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz",
+      "integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==",
+      "requires": {
+        "streamsearch": "0.1.2"
+      }
+    },
    "dir-glob": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@ -922,6 +938,14 @@
        "vary": "~1.1.2"
      }
    },
+    "express-fileupload": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.2.0.tgz",
+      "integrity": "sha512-oe4WpKcSppXnl5peornawWUa6tKmIc1/kJxMNRGJR1A0v4zyLL6VsFR6wZ8P2a4Iq3aGx8xae3Vlr+MOMQhFPw==",
+      "requires": {
+        "busboy": "^0.3.1"
+      }
+    },
    "fast-deep-equal": {
      "version": "3.1.3",
      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@ -1475,6 +1499,11 @@
      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
      "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
    },
+    "node-stream-zip": {
+      "version": "1.11.3",
+      "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.11.3.tgz",
+      "integrity": "sha512-GY+9LxkQuIT3O7K8BTdHVGKFcBYBy2vAVcTBtkKpu+OlBef/NSb6VuIWSyLiVDfmLMkggHeRJZN0F3W0GWU/uw=="
+    },
    "on-finished": {
      "version": "2.3.0",
      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -1812,6 +1841,11 @@
      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
      "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
    },
+    "streamsearch": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
+      "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
+    },
    "string-width": {
      "version": "3.1.0",
      "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
--- a/package.json
+++ b/package.json
@ -14,8 +14,10 @@
    "@types/express": "github:types/express",
    "@types/mongoose": "^5.7.36",
    "express": "^4.17.1",
+    "express-fileupload": "^1.2.0",
    "mongoose": "^5.10.2",
    "mongoose-int32": "^0.4.1",
+    "node-stream-zip": "^1.11.3",
    "typescript": "^4.0.2",
    "winston": "^3.3.3"
  },
--- a/src/html/upload.html
+++ b/src/html/upload.html
@ -1,16 +1,23 @@
 <html>
 	<head>
 		<title>file upload</title>
+		<script>
+			async function upload() {
+				let file = document.getElementById('file');
+
+				console.log(file.files[0]);
+
+				await fetch('/upload', {
+					method: 'POST',
+					body: file.files[0]
+				});
+			}
+		</script>
 	</head>
 	<body>
-		<form action="/upload" method="post">
-			<label for="title">Title: </label><br>
-			<input type="text" id="title" name="title"><br>
-			<label for="artist">Artist: </label><br>
-			<input type="text" id="artist" name="artist"><br>
-			<label for="credit">Credit: </label><br>
-			<input type="text" id="credit" name="credit"><br>
-			<input type="submit" value="upload">
+		<form method="post" encType="multipart/form-data">
+			<input type="file" name="file" id="file"><br>
+			<input type="submit" value="upload" onclick="upload()">
 		</form>
 	</body>
 </html>
--- a/src/index.ts
+++ b/src/index.ts
@ -2,10 +2,13 @@ import * as express from 'express';
 import * as mongoose from 'mongoose';
 import * as fs from 'fs';
 import * as winston from 'winston';
+import * as fileUpload from 'express-fileupload';

 import * as format from './lib/format';
 import { File } from './schema';

+import * as upload from './upload';
+
 const config = JSON.parse(fs.readFileSync('./config/config.json', {encoding: 'utf8'}));

 const db = mongoose.connect(`${config.dbconnectionURL}/${config.dbname}`, {
@ -37,15 +40,6 @@ const logger = winston.createLogger({
 	]
 });

-function returnStatic(page) {
-	return (req, res) => {
-		fs.readFile(`src/html/${page}`, 'utf8', (err, data) => {
-			if (err) throw err;
-			res.send(data);
-		});
-	};
-}
-
 logger.info('connecting to mongodb database');
 db.then(() => {
 	logger.info('connected to database!');
@ -54,31 +48,23 @@ db.then(() => {

 	// @ts-ignore
 	app.use(express.urlencoded({extended: true}));
+	app.use(fileUpload({limits: { fileSize: 50 * 1024 * 1024 }}));

 	app.set('db', db);
 	app.set('config', config);
 	app.set('logger', logger);

-	app.get('/upload', returnStatic('upload.html'));
-	
-	app.post('/upload', async (req, res) => { // only for testing, very abusable
-		const file = new File(req.body);
-		await file.save();
-
-		res.send('uploaded file');
-	});
+	upload.run(app);

 	app.get('/list', async (req, res) => { // only for testing
 		const docs = await File.find({});
-		const map = docs.map(d => {
-			const keys = Object.keys(d.toObject()).filter(k => !k.startsWith('_'));
-			const obj = {};
-			for (const key of keys) obj[key] = d[key];
 		
-			return obj;
-		});
-
-		res.send(JSON.stringify(map));
+		res.send(docs.map((doc: any) => 
+			`${doc.artist} - ${doc.title} by ${doc.credit}<br>` +
+			doc.charts.map(ch =>
+				`- ${ch.difficulty} ${ch.rating}: ${ch.name}`
+			).join('<br>')
+		).join('<br><br>'));
 	});

 	app.get('/', (req, res) => {
--- a/src/lib/smparse.ts
+++ b/src/lib/smparse.ts
@ -0,0 +1,57 @@
+export function parseSM(data: string) {
+	data = data.replace(/[\n\r]/g,'');
+
+	// steps
+	const difficulties = [];
+	const steps = data.split('#NOTES:');
+	steps.slice(1);
+
+	for (const step of steps) {
+		if (step.includes(';')) {
+			const diff: any = {};
+			const stepsSplit = step.split(';')[0].split(':').map(seg => seg.trim());
+
+			if (stepsSplit.length === 6) {
+				diff.type = stepsSplit[0];
+				diff.name = stepsSplit[1];
+				diff.difficulty = stepsSplit[2];
+				diff.rating = Number(stepsSplit[3]);
+				diff.radarvalues = stepsSplit[4].split(',').map(v => Number(v));
+
+				difficulties.push(diff);
+			}
+		}
+	}
+
+	// metadata
+	const lines = data.split(';').filter(l => l.startsWith('#'));
+	const obj: any = {};
+
+	for (const l of lines) {
+		const key = l.split(':')[0].slice(1);
+		let value: any = l.split(':')[1];
+
+		if (value !== '' && !(key === 'FGCHANGES' || key === 'BGCHANGES' || key === 'BETTERBGCHANGES' || key === 'SPELLCARDS')) { // no
+			if (!isNaN(value)) value = Number(value);
+			if (value === 'YES') value = true;
+			if (value === 'NO') value = false;
+			if (typeof value === 'string' && value.includes('=')) { // likely a map
+				const keys = value.split(',').map(v => v.split('=')[0]);
+				const values = value.split(',').map(v => v.split('=')[1]);
+				const map = {};
+
+				for (const i in keys) {
+					map[Number(keys[i])] = Number(values[i]); // afaik maps are only numbers?
+				}
+
+				value = map;
+			}
+	
+			obj[key.toLowerCase()] = value;
+		}
+	}
+
+	obj.charts = difficulties;
+
+	return obj;
+}
--- a/src/lib/util.ts
+++ b/src/lib/util.ts
@ -0,0 +1,10 @@
+import * as fs from 'fs';
+
+export function returnStatic(page) {
+	return (req, res) => {
+		fs.readFile(`src/html/${page}`, 'utf8', (err, data) => {
+			if (err) throw err;
+			res.send(data);
+		});
+	};
+}
--- a/src/schema.ts
+++ b/src/schema.ts
@ -1,17 +1,18 @@
 import * as mongoose from 'mongoose';
-import * as Int32 from 'mongoose-int32';

 const Schema = mongoose.Schema;
 
 const Sample = new Schema({
-	start: {type: Int32, default: new Int32(0)},
-	length: {type: Int32, default: new Int32(0)}
+	start: {type: Number, default: 0},
+	length: {type: Number, default: 0}
 });

 const Chart = new Schema({
+	type: {type: String, default: 'dance-single'},
 	name: {type: String, default: ''},
-	rating: {type: Int32, default: new Int32(0)},
-	type: {type: String, default: 'Challenge'}
+	difficulty: {type: String, default: 'Challenge'},
+	rating: {type: Number, default: 0},
+	radarvalues: [Number]
 });

 const FileSchema = new Schema({
@ -24,7 +25,7 @@ const FileSchema = new Schema({
 	credit: String,
 	uploader: {type: String, default: '00000000-0000-4000-a000-000000000000'},
 	sample: Sample,
-	bpms: {type: [Number], default: 0},
+	bpms: {type: Object, default: {'0': 0}},
 	charts: [Chart]
 });

--- a/src/upload.ts
+++ b/src/upload.ts
@ -0,0 +1,53 @@
+import { tmpdir } from 'os';
+import * as fs from 'fs';
+const StreamZip = require('node-stream-zip');
+
+import { returnStatic } from './lib/util';
+import { parseSM } from './lib/smparse';
+import { File } from './schema';
+
+export function run(app) {
+	const logger = app.get('logger');
+
+	app.get('/upload', returnStatic('upload.html'));
+	
+	app.post('/upload', async (req, res) => { // only for testing, very abusable
+		const file = req.files.file;
+		if (file.mimetype !== 'application/zip') return res.status(400).send('Invalid filetype');
+
+		const dir = tmpdir() + '/' + file.md5;
+		fs.writeFile(dir, file.data, (err) => {
+			if (err) throw err;
+
+			const zip = new StreamZip({
+				file: dir,
+				storeEntries: true
+			});
+
+			zip.on('ready', () => {
+				const smFile = Object.values(zip.entries()).find((f: any) =>
+					!f.isDirectory && (f.name.endsWith('.sm'))
+				);
+
+				if (!smFile) {
+					res.status(400).send('No .sm found');
+				} else {
+					const data = zip.entryDataSync((smFile as any).name);
+					const chart = parseSM(data.toString());
+
+					logger.info(`${chart.artist} - ${chart.title} was just uploaded`);
+
+					const file = new File(chart);
+					file.save();
+
+					res.send(`your file "${chart.artist} - ${chart.title}" has been uploaded! check <a href="http://0.0.0.0:8080/list">the listing</a>`); // todo: change to actual url
+				}
+
+				zip.close();
+				fs.unlink(dir, (err) => {
+					if (err) throw err;
+				});
+			});
+		});
+	});
+}