commit 6365c58bd45d51c1c44bac9321290aa7917e70de Author: Daniel Seiller Date: Thu Jun 6 01:15:49 2019 +0200 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..832207a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +.vscode/** \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..416bf99 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,183 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "csv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "csv-core" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ed_ldr" +version = "0.1.0" +dependencies = [ + "csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rstar 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "either" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fnv" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "humantime" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itertools" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pdqselect" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rstar" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ryu" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.15.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" +"checksum csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9044e25afb0924b5a5fc5511689b0918629e85d68ea591e5e87fbf1e85ea1b3b" +"checksum csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa5cdef62f37e6ffe7d1f07a381bc0db32b7a3ff1cac0de56cb0d81e71f53d65" +"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" +"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" +"checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" +"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" +"checksum libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "42914d39aad277d9e176efbdad68acb1d5443ab65afe0e0e4f0d49352a950880" +"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" +"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" +"checksum pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ec91767ecc0a0bbe558ce8c9da33c068066c57ecc8bb8477ef8c1ad3ef77c27" +"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" +"checksum rstar 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dd08ae4f9661517777346592956ea6cdbba2895a28037af7daa600382f4b4001" +"checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f" +"checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be" +"checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e" +"checksum syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)" = "a1393e4a97a19c01e900df2aec855a29f71cf02c402e2f443b8d2747c25c5dbe" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..a874f2a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "ed_ldr" +version = "0.1.0" +authors = ["Daniel Seiller "] +edition = "2018" + +# [profile.release] +# debug = true + +[dependencies] +csv = "1.0.7" +serde = "1.0.92" +serde_derive = "1.0.92" +rstar = "0.4.0" +humantime = "1.2.0" +fnv = "1.0.6" diff --git a/README.md b/README.md new file mode 100644 index 0000000..a65ba25 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +# Elite: Dangerous Long Range Router (Rust Version) + +## Usage: + +#. run `download.sh` and `process.py` (you can hit Ctrl+C when it says "Building KD-Tree..") +#. edit source, destination and range in `src/main.rs` +#. (optional) `set RUSTFLAGS=-C target-cpu=native` (Windows) or `export RUSTFLAGS=-C target-cpu=native` (Linux) +#. `cargo run --release` + + + +## Dependencies + +- Python 3.7 + - Pandas + - uJSON +- Working nightly Rust Compiler (tested with `rustc 1.37.0-nightly (5d8f59f4b 2019-06-04)`) +- ~8GB of free RAM \ No newline at end of file diff --git a/download.sh b/download.sh new file mode 100644 index 0000000..ea5226e --- /dev/null +++ b/download.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash +rm systemsWithCoordinates.json bodies.json *.aria2 +wget https://www.edsm.net/dump/systemsWithCoordinates.json https://www.edsm.net/dump/bodies.json \ No newline at end of file diff --git a/process.py b/process.py new file mode 100644 index 0000000..6416cbc --- /dev/null +++ b/process.py @@ -0,0 +1,172 @@ +import ujson as json +from tqdm import tqdm +from pprint import pprint +import itertools as ITT +import os +import sys +import csv +import sqlite3 +import pandas as pd + + +def is_scoopable(entry): + first = entry.type.split()[0] + return first == "Neutron" or first == "White" or first in "KGBFOAM" + + +def get_mult(name): + try: + first = name.split()[0] + except: + return 1 + if first == "Neutron": + return 4 + if first == "White": + return 1.5 + return 1 + + +def dict_factory(cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + + +def blocks(files, size=65536): + while True: + b = files.read(size) + if not b: + break + yield b + + +def getlines(f, fn, show_progbar=False): + f.seek(0, 2) + size = f.tell() + f.seek(0) + progbar = tqdm( + desc="Processing " + fn, + total=size, + unit="b", + unit_scale=True, + unit_divisor=1024, + ascii=True, + leave=True, + disable=(not show_progbar), + ) + buffer = [] + for block in blocks(f): + progbar.n = f.tell() + progbar.update(0) + if buffer: + buffer += (buffer.pop(0) + block).splitlines(keepends=True) + else: + buffer += block.splitlines(keepends=True) + while buffer and buffer[0].endswith("\n"): + try: + yield json.loads(buffer.pop(0).strip().rstrip(",")) + except ValueError: + pass + while buffer: + try: + yield json.loads(buffer.pop(0).strip().rstrip(",")) + except ValueError: + pass + + +def process_file(fn, show_progbar=False): + with open(fn, "r") as f: + for line in tqdm( + getlines(f, fn, show_progbar), + desc=fn, + unit=" lines", + unit_scale=True, + ascii=True, + leave=True, + disable=(not show_progbar), + ): + yield line + + +if not os.path.isfile("stars.jl"): + print("Filtering for Neutron Stars") + with open("stars.jl", "w") as neut: + for body in process_file("bodies.json", True): + T = body.get("type") or "" + if "Star" in T: + neut.write(json.dumps(body) + "\n") + + +def load_systems(load=False): + load = not os.path.isfile("systems.db") + cache = sqlite3.connect("systems.db") + cache.row_factory = dict_factory + c = cache.cursor() + if load: + print("Caching Systems") + c.execute("DROP TABLE IF EXISTS systems") + c.execute( + "CREATE TABLE systems (id64 int primary key, name text, x real, y real, z real)" + ) + cache.commit() + recs = [] + for system in process_file("systemsWithCoordinates.json", True): + rec = [ + system["id64"], + system["name"], + system["coords"]["x"], + system["coords"]["y"], + system["coords"]["z"], + ] + recs.append(rec) + if len(recs) % 1024 * 1024 == 0: + c.executemany("INSERT INTO systems VALUES (?,?,?,?,?)", recs) + recs.clear() + c.executemany("INSERT INTO systems VALUES (?,?,?,?,?)", recs) + cache.commit() + return cache, c + + +if not os.path.isfile("stars.csv"): + cache, cur = load_systems() + rows = [] + with open("stars.csv", "w", newline="") as sys_csv: + csv_writer = csv.writer(sys_csv, dialect="excel") + for neut in process_file("stars.jl", True): + cur.execute( + "SELECT * FROM systems WHERE id64==?", (neut.get("systemId64"),) + ) + system = cur.fetchone() + if not system: + continue + row = [ + neut["systemId64"], + neut["subType"], + neut["name"], + get_mult(neut["subType"]), + system["x"], + system["y"], + system["z"], + ] + rows.append(row) + if len(rows) > 1024: + csv_writer.writerows(rows) + rows.clear() + csv_writer.writerows(rows) + print() + cache.close() + +if not os.path.isfile("stars.kdt"): + tqdm.pandas(ascii=True, leave=True) + print("Loading data...") + data = pd.read_csv( + "stars.csv", + encoding="utf-8", + names=["id", "type", "name", "mult", "x", "y", "z"], + ) + print("Cleaning data...") + data.type.fillna("Unknown", inplace=True) + data.drop_duplicates("id", inplace=True) + print("Writing CSV...") + data.to_csv("stars.csv", header=False, index=False) diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..c9ed87a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,272 @@ +extern crate csv; +extern crate serde; +#[macro_use] +extern crate serde_derive; + +extern crate fnv; +extern crate humantime; +use fnv::{FnvHashMap, FnvHashSet}; +use humantime::format_duration; +use rstar::{PointDistance, RTree, RTreeObject, AABB}; +use std::collections::VecDeque; +use std::hash::{Hash, Hasher}; +use std::io::Write; +use std::time::Instant; + +#[derive(Debug, Deserialize)] +struct Record { + id: i64, + star_type: String, + name: String, + mult: f32, + x: f32, + y: f32, + z: f32, +} +#[derive(Debug)] +struct System { + id: i64, + star_type: String, + name: String, + mult: f32, +} + +#[derive(Debug)] +struct Point { + id: i64, + x: f32, + y: f32, + z: f32, +} + +impl Point { + pub fn dist2(&self, p: &[f32; 3]) -> f32 { + let dx = self.x - p[0]; + let dy = self.y - p[1]; + let dz = self.z - p[2]; + + return dx * dx + dy * dy + dz * dz; + } + pub fn distp(&self, p: &Point) -> f32 { + return self.distp2(p).sqrt(); + } + pub fn distp2(&self, p: &Point) -> f32 { + return self.dist2(&[p.x, p.y, p.z]); + } +} +impl PartialEq for Point { + fn eq(&self, other: &Self) -> bool { + self.id == other.id + } +} + +impl Eq for Point {} + +impl Hash for Point { + fn hash(&self, state: &mut H) { + self.id.hash(state); + } +} + +impl RTreeObject for Point { + type Envelope = AABB<[f32; 3]>; + + fn envelope(&self) -> Self::Envelope { + return AABB::from_point([self.x, self.y, self.z]); + } +} + +impl PointDistance for Point { + fn distance_2(&self, point: &[f32; 3]) -> f32 { + return self.dist2(&point); + } +} + +struct Router { + tree: RTree, + systems: FnvHashMap, + range: f32, + scoopable: FnvHashSet, +} + +impl Router { + pub fn new(path: &str, range: f32) -> Self { + let mut scoopable = FnvHashSet::default(); + let mut systems: FnvHashMap = FnvHashMap::default(); + let mut reader = csv::ReaderBuilder::new() + .has_headers(false) + .from_path(path) + .unwrap(); + println!("Loading {}...", path); + let points = reader + .deserialize() + .map(|res: Result| { + let sys = res.unwrap(); + systems.insert( + sys.id, + System { + id: sys.id, + star_type: sys.star_type.clone(), + name: sys.name, + mult: sys.mult, + }, + ); + if sys.mult > 1.0f32 { + scoopable.insert(sys.id); + } else { + for c in "KGBFOAM".chars() { + if sys.star_type.starts_with(c) { + scoopable.insert(sys.id); + break; + } + } + } + return Point { + x: sys.x, + y: sys.y, + z: sys.z, + id: sys.id, + }; + }) + .collect(); + return Self { + tree: RTree::::bulk_load(points), + systems, + range, + scoopable, + }; + } + + fn preload_points(&self) -> FnvHashMap { + let mut ret = FnvHashMap::default(); + for point in &self.tree { + ret.insert(point.id, point); + } + return ret; + } + + fn closest(&self, point: &[f32; 3]) -> &Point { + return self.tree.nearest_neighbor(point).unwrap(); + } + fn points_in_sphere(&self, center: &[f32; 3], radius: f32) -> impl Iterator { + let center: [f32; 3] = *center; + return self + .tree + .locate_in_envelope(&AABB::from_corners( + [ + center[0] - radius * 1f32, + center[1] - radius * 1f32, + center[2] - radius * 1f32, + ], + [ + center[0] + radius * 1f32, + center[1] + radius * 1f32, + center[2] + radius * 1f32, + ], + )) + .filter(move |p| (p.dist2(¢er) < (radius * radius))); + } + + fn mult(&self, id: i64) -> f32 { + if let Some(sys) = self.systems.get(&id) { + return sys.mult; + }; + return 1.0; + } + fn neighbours(&self, sys: &Point) -> impl Iterator { + return self.points_in_sphere(&[sys.x, sys.y, sys.z], self.mult(sys.id) * self.range); + } + + fn valid(&self, sys: &Point) -> bool { + return self.scoopable.contains(&sys.id); + } + + pub fn route(&mut self, src: &[f32; 3], dst: &[f32; 3]) -> Vec<(&System, &Point)> { + let start_sys = self.closest(src); + let goal_sys = self.closest(dst); + let total = self.tree.size() as f32; + let mut prev = FnvHashMap::default(); + let mut seen = FnvHashSet::default(); + let t_start = Instant::now(); + let mut depth = 0; + let mut queue: VecDeque<(usize, &Point)> = VecDeque::new(); + let mut r_queue: VecDeque<(usize, &Point)> = VecDeque::new(); + queue.push_front((0, &start_sys)); + r_queue.push_front((0, &goal_sys)); + seen.insert(start_sys.id); + while let Some((d, sys)) = queue.pop_front() { + if d != depth { + depth = d; + print!( + "\r[{}] Depth: {}, Queue: {}, Seen: {} ({:.2}%) ", + format_duration(t_start.elapsed()), + d, + queue.len(), + prev.len(), + ((prev.len() as f32) / total) * 100.0 + ); + std::io::stdout().flush().unwrap(); + } + if sys.id == goal_sys.id { + println!(); + let points = self.preload_points(); + let mut v: Vec<(&System, &Point)> = Vec::new(); + let mut prev_sys_id = sys.id; + loop { + if let Some(sys) = self.systems.get(&prev_sys_id) { + v.push((sys, points[&sys.id])); + } else { + break; + }; + match prev.get(&prev_sys_id) { + Some(sys_id) => prev_sys_id = *sys_id, + None => { + break; + } + } + } + v.reverse(); + return v; + } + let nbs = self + .neighbours(&sys) + .filter(|&nb| (self.valid(nb) || (nb.id == goal_sys.id))); + for nb in nbs { + if seen.insert(nb.id) { + prev.insert(nb.id, sys.id); + queue.push_back((d + 1, nb)); + } + } + } + println!("No route found!"); + return Vec::new(); + } +} + +fn main() { + let path = r#"D:\devel\files\python\EDSM\stars.csv"#; + let t_load = Instant::now(); + let mut router: Router = Router::new(path, 48.0); + println!("Done in {}!", format_duration(t_load.elapsed())); + let t_route = Instant::now(); + let route = router.route( + &[-65.21875, 7.75, -111.03125], // Ix + &[-1111.5625, -134.21875, 65269.75], // Beagle Point + // &[-7095.375, 401.25, 2396.8125], // V1357 Cygni, + ); // Ix -> BP 537 + println!( + "Done in {} ({} Jumps)!\n", + format_duration(t_route.elapsed()), + route.len() + ); + + let mut total: f32 = 0.0; + for ((sys1, p1), (_, p2)) in route.iter().zip(route.iter().skip(1)) { + let dist = p1.distp(p2); + total += dist; + println!("{} [{}]: {:.2} Ly", sys1.name, sys1.star_type, dist); + } + let sys = route.iter().last().unwrap().0; + println!("{} [{}]: {:.2} Ly\n", sys.name, sys.star_type, 0.0); + println!("Total: {:.2} Ly", total); +}