oat/in-the-database-2
oat/in-the-database-2
1
1
Fork 1
Code Issues 6 Pull requests Releases Activity

Compare commits

base: oat:69d93c1f9f549127f3cfca0a6e98e405c1958df7
Branches Tags
oat:master
...
compare: oat:108eec139d961e7549ab0572be5a56d2cd774eb8
Branches Tags
oat:master

2 commits
69d93c1f9f ... 108eec139d

Author SHA1 Message Date
oat
108eec139d
file parsing!! 
you can now just drop in a zip in /upload and it'll parse, handle and upload everything for you

right now only the metadata is saved, so the actual file downloads aren't available, but that shouldn't be too hard to implement

this is really hacky however, and dropping in a file with a malformed .sm or 2 .sms will likely make it have a panic attack

definetly needs lots and lots of testing, it barely works
 2020-09-02 21:30:23 +03:00
oat
665a372f5e
we uploadin files now 2020-09-02 19:03:49 +03:00
8 changed files with 190 additions and 40 deletions
Show stats Expand all files Collapse all files

34
package-lock.json generated
View file

@ -483,6 +483,14 @@
"resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz", "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.4.tgz",
"integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q==" "integrity": "sha512-S/yKGU1syOMzO86+dGpg2qGoDL0zvzcb262G+gqEy6TgP6rt6z6qxSFX/8X6vLC91P7G7C3nLs0+bvDzmvBA3Q=="
}, },
"busboy": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.3.1.tgz",
"integrity": "sha512-y7tTxhGKXcyBxRKAni+awqx8uqaJKrSFSNFSeRG5CsWNdmy2BIK+6VGWEW7TZnIO/533mtMEA4rOevQV815YJw==",
"requires": {
"dicer": "0.3.0"
}
},
"bytes": { "bytes": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
@ -665,6 +673,14 @@
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
}, },
"dicer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz",
"integrity": "sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==",
"requires": {
"streamsearch": "0.1.2"
}
},
"dir-glob": { "dir-glob": {
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
@ -922,6 +938,14 @@
"vary": "~1.1.2" "vary": "~1.1.2"
} }
}, },
"express-fileupload": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/express-fileupload/-/express-fileupload-1.2.0.tgz",
"integrity": "sha512-oe4WpKcSppXnl5peornawWUa6tKmIc1/kJxMNRGJR1A0v4zyLL6VsFR6wZ8P2a4Iq3aGx8xae3Vlr+MOMQhFPw==",
"requires": {
"busboy": "^0.3.1"
}
},
"fast-deep-equal": { "fast-deep-equal": {
"version": "3.1.3", "version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@ -1475,6 +1499,11 @@
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
"integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
}, },
"node-stream-zip": {
"version": "1.11.3",
"resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.11.3.tgz",
"integrity": "sha512-GY+9LxkQuIT3O7K8BTdHVGKFcBYBy2vAVcTBtkKpu+OlBef/NSb6VuIWSyLiVDfmLMkggHeRJZN0F3W0GWU/uw=="
},
"on-finished": { "on-finished": {
"version": "2.3.0", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -1812,6 +1841,11 @@
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
}, },
"streamsearch": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
},
"string-width": { "string-width": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",

2
package.json
View file

@ -14,8 +14,10 @@
"@types/express": "github:types/express", "@types/express": "github:types/express",
"@types/mongoose": "^5.7.36", "@types/mongoose": "^5.7.36",
"express": "^4.17.1", "express": "^4.17.1",
"express-fileupload": "^1.2.0",
"mongoose": "^5.10.2", "mongoose": "^5.10.2",
"mongoose-int32": "^0.4.1", "mongoose-int32": "^0.4.1",
"node-stream-zip": "^1.11.3",
"typescript": "^4.0.2", "typescript": "^4.0.2",
"winston": "^3.3.3" "winston": "^3.3.3"
}, },

23
src/html/upload.html
View file

@ -1,16 +1,23 @@
<html> <html>
<head> <head>
<title>file upload</title> <title>file upload</title>
<script>
async function upload() {
let file = document.getElementById('file');
console.log(file.files[0]);
await fetch('/upload', {
method: 'POST',
body: file.files[0]
});
}
</script>
</head> </head>
<body> <body>
<form action="/upload" method="post"> <form method="post" encType="multipart/form-data">
<label for="title">Title: </label><br> <input type="file" name="file" id="file"><br>
<input type="text" id="title" name="title"><br> <input type="submit" value="upload" onclick="upload()">
<label for="artist">Artist: </label><br>
<input type="text" id="artist" name="artist"><br>
<label for="credit">Credit: </label><br>
<input type="text" id="credit" name="credit"><br>
<input type="submit" value="upload">
</form> </form>
</body> </body>
</html> </html>

36
src/index.ts
View file

@ -2,10 +2,13 @@ import * as express from 'express';
import * as mongoose from 'mongoose'; import * as mongoose from 'mongoose';
import * as fs from 'fs'; import * as fs from 'fs';
import * as winston from 'winston'; import * as winston from 'winston';
import * as fileUpload from 'express-fileupload';
import * as format from './lib/format'; import * as format from './lib/format';
import { File } from './schema'; import { File } from './schema';
import * as upload from './upload';
const config = JSON.parse(fs.readFileSync('./config/config.json', {encoding: 'utf8'})); const config = JSON.parse(fs.readFileSync('./config/config.json', {encoding: 'utf8'}));
const db = mongoose.connect(`${config.dbconnectionURL}/${config.dbname}`, { const db = mongoose.connect(`${config.dbconnectionURL}/${config.dbname}`, {
@ -37,15 +40,6 @@ const logger = winston.createLogger({
] ]
}); });
function returnStatic(page) {
return (req, res) => {
fs.readFile(`src/html/${page}`, 'utf8', (err, data) => {
if (err) throw err;
res.send(data);
});
};
}
logger.info('connecting to mongodb database'); logger.info('connecting to mongodb database');
db.then(() => { db.then(() => {
logger.info('connected to database!'); logger.info('connected to database!');
@ -54,31 +48,23 @@ db.then(() => {
// @ts-ignore // @ts-ignore
app.use(express.urlencoded({extended: true})); app.use(express.urlencoded({extended: true}));
app.use(fileUpload({limits: { fileSize: 50 * 1024 * 1024 }}));
app.set('db', db); app.set('db', db);
app.set('config', config); app.set('config', config);
app.set('logger', logger); app.set('logger', logger);
app.get('/upload', returnStatic('upload.html')); upload.run(app);
app.post('/upload', async (req, res) => { // only for testing, very abusable
const file = new File(req.body);
await file.save();
res.send('uploaded file');
});
app.get('/list', async (req, res) => { // only for testing app.get('/list', async (req, res) => { // only for testing
const docs = await File.find({}); const docs = await File.find({});
const map = docs.map(d => {
const keys = Object.keys(d.toObject()).filter(k => !k.startsWith('_'));
const obj = {};
for (const key of keys) obj[key] = d[key];
return obj; res.send(docs.map((doc: any) =>
}); `${doc.artist} - ${doc.title} by ${doc.credit}<br>` +
doc.charts.map(ch =>
res.send(JSON.stringify(map)); `- ${ch.difficulty} ${ch.rating}: ${ch.name}`
).join('<br>')
).join('<br><br>'));
}); });
app.get('/', (req, res) => { app.get('/', (req, res) => {

57
src/lib/smparse.ts Normal file
View file

@ -0,0 +1,57 @@
export function parseSM(data: string) {
data = data.replace(/[\n\r]/g,'');
// steps
const difficulties = [];
const steps = data.split('#NOTES:');
steps.slice(1);
for (const step of steps) {
if (step.includes(';')) {
const diff: any = {};
const stepsSplit = step.split(';')[0].split(':').map(seg => seg.trim());
if (stepsSplit.length === 6) {
diff.type = stepsSplit[0];
diff.name = stepsSplit[1];
diff.difficulty = stepsSplit[2];
diff.rating = Number(stepsSplit[3]);
diff.radarvalues = stepsSplit[4].split(',').map(v => Number(v));
difficulties.push(diff);
}
}
}
// metadata
const lines = data.split(';').filter(l => l.startsWith('#'));
const obj: any = {};
for (const l of lines) {
const key = l.split(':')[0].slice(1);
let value: any = l.split(':')[1];
if (value !== '' && !(key === 'FGCHANGES' || key === 'BGCHANGES' || key === 'BETTERBGCHANGES' || key === 'SPELLCARDS')) { // no
if (!isNaN(value)) value = Number(value);
if (value === 'YES') value = true;
if (value === 'NO') value = false;
if (typeof value === 'string' && value.includes('=')) { // likely a map
const keys = value.split(',').map(v => v.split('=')[0]);
const values = value.split(',').map(v => v.split('=')[1]);
const map = {};
for (const i in keys) {
map[Number(keys[i])] = Number(values[i]); // afaik maps are only numbers?
}
value = map;
}
obj[key.toLowerCase()] = value;
}
}
obj.charts = difficulties;
return obj;
}

10
src/lib/util.ts Normal file
View file

@ -0,0 +1,10 @@
import * as fs from 'fs';
export function returnStatic(page) {
return (req, res) => {
fs.readFile(`src/html/${page}`, 'utf8', (err, data) => {
if (err) throw err;
res.send(data);
});
};
}

13
src/schema.ts
View file

@ -1,17 +1,18 @@
import * as mongoose from 'mongoose'; import * as mongoose from 'mongoose';
import * as Int32 from 'mongoose-int32';
const Schema = mongoose.Schema; const Schema = mongoose.Schema;
const Sample = new Schema({ const Sample = new Schema({
start: {type: Int32, default: new Int32(0)}, start: {type: Number, default: 0},
length: {type: Int32, default: new Int32(0)} length: {type: Number, default: 0}
}); });
const Chart = new Schema({ const Chart = new Schema({
type: {type: String, default: 'dance-single'},
name: {type: String, default: ''}, name: {type: String, default: ''},
rating: {type: Int32, default: new Int32(0)}, difficulty: {type: String, default: 'Challenge'},
type: {type: String, default: 'Challenge'} rating: {type: Number, default: 0},
radarvalues: [Number]
}); });
const FileSchema = new Schema({ const FileSchema = new Schema({
@ -24,7 +25,7 @@ const FileSchema = new Schema({
credit: String, credit: String,
uploader: {type: String, default: '00000000-0000-4000-a000-000000000000'}, uploader: {type: String, default: '00000000-0000-4000-a000-000000000000'},
sample: Sample, sample: Sample,
bpms: {type: [Number], default: 0}, bpms: {type: Object, default: {'0': 0}},
charts: [Chart] charts: [Chart]
}); });

53
src/upload.ts Normal file
View file

@ -0,0 +1,53 @@
import { tmpdir } from 'os';
import * as fs from 'fs';
const StreamZip = require('node-stream-zip');
import { returnStatic } from './lib/util';
import { parseSM } from './lib/smparse';
import { File } from './schema';
export function run(app) {
const logger = app.get('logger');
app.get('/upload', returnStatic('upload.html'));
app.post('/upload', async (req, res) => { // only for testing, very abusable
const file = req.files.file;
if (file.mimetype !== 'application/zip') return res.status(400).send('Invalid filetype');
const dir = tmpdir() + '/' + file.md5;
fs.writeFile(dir, file.data, (err) => {
if (err) throw err;
const zip = new StreamZip({
file: dir,
storeEntries: true
});
zip.on('ready', () => {
const smFile = Object.values(zip.entries()).find((f: any) =>
!f.isDirectory && (f.name.endsWith('.sm'))
);
if (!smFile) {
res.status(400).send('No .sm found');
} else {
const data = zip.entryDataSync((smFile as any).name);
const chart = parseSM(data.toString());
logger.info(`${chart.artist} - ${chart.title} was just uploaded`);
const file = new File(chart);
file.save();
res.send(`your file "${chart.artist} - ${chart.title}" has been uploaded! check <a href="http://0.0.0.0:8080/list">the listing</a>`); // todo: change to actual url
}
zip.close();
fs.unlink(dir, (err) => {
if (err) throw err;
});
});
});
});
}