Initial commit

This commit is contained in:
Daniel S. 2019-06-06 01:15:49 +02:00
commit 6365c58bd4
7 changed files with 667 additions and 0 deletions

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
/target
**/*.rs.bk
.vscode/**

183
Cargo.lock generated Normal file
View File

@ -0,0 +1,183 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "autocfg"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "csv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "csv-core"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ed_ldr"
version = "0.1.0"
dependencies = [
"csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)",
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rstar 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "either"
version = "1.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "fnv"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "humantime"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "itertools"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "itoa"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-traits"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "pdqselect"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "proc-macro2"
version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "quick-error"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "quote"
version = "0.6.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rstar"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ryu"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde_derive"
version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "syn"
version = "0.15.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-xid"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[metadata]
"checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf"
"checksum csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9044e25afb0924b5a5fc5511689b0918629e85d68ea591e5e87fbf1e85ea1b3b"
"checksum csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa5cdef62f37e6ffe7d1f07a381bc0db32b7a3ff1cac0de56cb0d81e71f53d65"
"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114"
"checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358"
"checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f"
"checksum libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "42914d39aad277d9e176efbdad68acb1d5443ab65afe0e0e4f0d49352a950880"
"checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39"
"checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32"
"checksum pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ec91767ecc0a0bbe558ce8c9da33c068066c57ecc8bb8477ef8c1ad3ef77c27"
"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
"checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db"
"checksum rstar 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dd08ae4f9661517777346592956ea6cdbba2895a28037af7daa600382f4b4001"
"checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f"
"checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be"
"checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e"
"checksum syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)" = "a1393e4a97a19c01e900df2aec855a29f71cf02c402e2f443b8d2747c25c5dbe"
"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"

16
Cargo.toml Normal file
View File

@ -0,0 +1,16 @@
[package]
name = "ed_ldr"
version = "0.1.0"
authors = ["Daniel Seiller <earthnuker@gmail.com>"]
edition = "2018"
# [profile.release]
# debug = true
[dependencies]
csv = "1.0.7"
serde = "1.0.92"
serde_derive = "1.0.92"
rstar = "0.4.0"
humantime = "1.2.0"
fnv = "1.0.6"

18
README.md Normal file
View File

@ -0,0 +1,18 @@
# Elite: Dangerous Long Range Router (Rust Version)
## Usage:
#. run `download.sh` and `process.py` (you can hit Ctrl+C when it says "Building KD-Tree..")
#. edit source, destination and range in `src/main.rs`
#. (optional) `set RUSTFLAGS=-C target-cpu=native` (Windows) or `export RUSTFLAGS=-C target-cpu=native` (Linux)
#. `cargo run --release`
## Dependencies
- Python 3.7
- Pandas
- uJSON
- Working nightly Rust Compiler (tested with `rustc 1.37.0-nightly (5d8f59f4b 2019-06-04)`)
- ~8GB of free RAM

3
download.sh Normal file
View File

@ -0,0 +1,3 @@
#!/usr/bin/bash
rm systemsWithCoordinates.json bodies.json *.aria2
wget https://www.edsm.net/dump/systemsWithCoordinates.json https://www.edsm.net/dump/bodies.json

172
process.py Normal file
View File

@ -0,0 +1,172 @@
import ujson as json
from tqdm import tqdm
from pprint import pprint
import itertools as ITT
import os
import sys
import csv
import sqlite3
import pandas as pd
def is_scoopable(entry):
first = entry.type.split()[0]
return first == "Neutron" or first == "White" or first in "KGBFOAM"
def get_mult(name):
try:
first = name.split()[0]
except:
return 1
if first == "Neutron":
return 4
if first == "White":
return 1.5
return 1
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
def blocks(files, size=65536):
while True:
b = files.read(size)
if not b:
break
yield b
def getlines(f, fn, show_progbar=False):
f.seek(0, 2)
size = f.tell()
f.seek(0)
progbar = tqdm(
desc="Processing " + fn,
total=size,
unit="b",
unit_scale=True,
unit_divisor=1024,
ascii=True,
leave=True,
disable=(not show_progbar),
)
buffer = []
for block in blocks(f):
progbar.n = f.tell()
progbar.update(0)
if buffer:
buffer += (buffer.pop(0) + block).splitlines(keepends=True)
else:
buffer += block.splitlines(keepends=True)
while buffer and buffer[0].endswith("\n"):
try:
yield json.loads(buffer.pop(0).strip().rstrip(","))
except ValueError:
pass
while buffer:
try:
yield json.loads(buffer.pop(0).strip().rstrip(","))
except ValueError:
pass
def process_file(fn, show_progbar=False):
with open(fn, "r") as f:
for line in tqdm(
getlines(f, fn, show_progbar),
desc=fn,
unit=" lines",
unit_scale=True,
ascii=True,
leave=True,
disable=(not show_progbar),
):
yield line
if not os.path.isfile("stars.jl"):
print("Filtering for Neutron Stars")
with open("stars.jl", "w") as neut:
for body in process_file("bodies.json", True):
T = body.get("type") or ""
if "Star" in T:
neut.write(json.dumps(body) + "\n")
def load_systems(load=False):
load = not os.path.isfile("systems.db")
cache = sqlite3.connect("systems.db")
cache.row_factory = dict_factory
c = cache.cursor()
if load:
print("Caching Systems")
c.execute("DROP TABLE IF EXISTS systems")
c.execute(
"CREATE TABLE systems (id64 int primary key, name text, x real, y real, z real)"
)
cache.commit()
recs = []
for system in process_file("systemsWithCoordinates.json", True):
rec = [
system["id64"],
system["name"],
system["coords"]["x"],
system["coords"]["y"],
system["coords"]["z"],
]
recs.append(rec)
if len(recs) % 1024 * 1024 == 0:
c.executemany("INSERT INTO systems VALUES (?,?,?,?,?)", recs)
recs.clear()
c.executemany("INSERT INTO systems VALUES (?,?,?,?,?)", recs)
cache.commit()
return cache, c
if not os.path.isfile("stars.csv"):
cache, cur = load_systems()
rows = []
with open("stars.csv", "w", newline="") as sys_csv:
csv_writer = csv.writer(sys_csv, dialect="excel")
for neut in process_file("stars.jl", True):
cur.execute(
"SELECT * FROM systems WHERE id64==?", (neut.get("systemId64"),)
)
system = cur.fetchone()
if not system:
continue
row = [
neut["systemId64"],
neut["subType"],
neut["name"],
get_mult(neut["subType"]),
system["x"],
system["y"],
system["z"],
]
rows.append(row)
if len(rows) > 1024:
csv_writer.writerows(rows)
rows.clear()
csv_writer.writerows(rows)
print()
cache.close()
if not os.path.isfile("stars.kdt"):
tqdm.pandas(ascii=True, leave=True)
print("Loading data...")
data = pd.read_csv(
"stars.csv",
encoding="utf-8",
names=["id", "type", "name", "mult", "x", "y", "z"],
)
print("Cleaning data...")
data.type.fillna("Unknown", inplace=True)
data.drop_duplicates("id", inplace=True)
print("Writing CSV...")
data.to_csv("stars.csv", header=False, index=False)

272
src/main.rs Normal file
View File

@ -0,0 +1,272 @@
extern crate csv;
extern crate serde;
#[macro_use]
extern crate serde_derive;
extern crate fnv;
extern crate humantime;
use fnv::{FnvHashMap, FnvHashSet};
use humantime::format_duration;
use rstar::{PointDistance, RTree, RTreeObject, AABB};
use std::collections::VecDeque;
use std::hash::{Hash, Hasher};
use std::io::Write;
use std::time::Instant;
#[derive(Debug, Deserialize)]
struct Record {
id: i64,
star_type: String,
name: String,
mult: f32,
x: f32,
y: f32,
z: f32,
}
#[derive(Debug)]
struct System {
id: i64,
star_type: String,
name: String,
mult: f32,
}
#[derive(Debug)]
struct Point {
id: i64,
x: f32,
y: f32,
z: f32,
}
impl Point {
pub fn dist2(&self, p: &[f32; 3]) -> f32 {
let dx = self.x - p[0];
let dy = self.y - p[1];
let dz = self.z - p[2];
return dx * dx + dy * dy + dz * dz;
}
pub fn distp(&self, p: &Point) -> f32 {
return self.distp2(p).sqrt();
}
pub fn distp2(&self, p: &Point) -> f32 {
return self.dist2(&[p.x, p.y, p.z]);
}
}
impl PartialEq for Point {
fn eq(&self, other: &Self) -> bool {
self.id == other.id
}
}
impl Eq for Point {}
impl Hash for Point {
fn hash<H: Hasher>(&self, state: &mut H) {
self.id.hash(state);
}
}
impl RTreeObject for Point {
type Envelope = AABB<[f32; 3]>;
fn envelope(&self) -> Self::Envelope {
return AABB::from_point([self.x, self.y, self.z]);
}
}
impl PointDistance for Point {
fn distance_2(&self, point: &[f32; 3]) -> f32 {
return self.dist2(&point);
}
}
struct Router {
tree: RTree<Point>,
systems: FnvHashMap<i64, System>,
range: f32,
scoopable: FnvHashSet<i64>,
}
impl Router {
pub fn new(path: &str, range: f32) -> Self {
let mut scoopable = FnvHashSet::default();
let mut systems: FnvHashMap<i64, System> = FnvHashMap::default();
let mut reader = csv::ReaderBuilder::new()
.has_headers(false)
.from_path(path)
.unwrap();
println!("Loading {}...", path);
let points = reader
.deserialize()
.map(|res: Result<Record, _>| {
let sys = res.unwrap();
systems.insert(
sys.id,
System {
id: sys.id,
star_type: sys.star_type.clone(),
name: sys.name,
mult: sys.mult,
},
);
if sys.mult > 1.0f32 {
scoopable.insert(sys.id);
} else {
for c in "KGBFOAM".chars() {
if sys.star_type.starts_with(c) {
scoopable.insert(sys.id);
break;
}
}
}
return Point {
x: sys.x,
y: sys.y,
z: sys.z,
id: sys.id,
};
})
.collect();
return Self {
tree: RTree::<Point>::bulk_load(points),
systems,
range,
scoopable,
};
}
fn preload_points(&self) -> FnvHashMap<i64, &Point> {
let mut ret = FnvHashMap::default();
for point in &self.tree {
ret.insert(point.id, point);
}
return ret;
}
fn closest(&self, point: &[f32; 3]) -> &Point {
return self.tree.nearest_neighbor(point).unwrap();
}
fn points_in_sphere(&self, center: &[f32; 3], radius: f32) -> impl Iterator<Item = &Point> {
let center: [f32; 3] = *center;
return self
.tree
.locate_in_envelope(&AABB::from_corners(
[
center[0] - radius * 1f32,
center[1] - radius * 1f32,
center[2] - radius * 1f32,
],
[
center[0] + radius * 1f32,
center[1] + radius * 1f32,
center[2] + radius * 1f32,
],
))
.filter(move |p| (p.dist2(&center) < (radius * radius)));
}
fn mult(&self, id: i64) -> f32 {
if let Some(sys) = self.systems.get(&id) {
return sys.mult;
};
return 1.0;
}
fn neighbours(&self, sys: &Point) -> impl Iterator<Item = &Point> {
return self.points_in_sphere(&[sys.x, sys.y, sys.z], self.mult(sys.id) * self.range);
}
fn valid(&self, sys: &Point) -> bool {
return self.scoopable.contains(&sys.id);
}
pub fn route(&mut self, src: &[f32; 3], dst: &[f32; 3]) -> Vec<(&System, &Point)> {
let start_sys = self.closest(src);
let goal_sys = self.closest(dst);
let total = self.tree.size() as f32;
let mut prev = FnvHashMap::default();
let mut seen = FnvHashSet::default();
let t_start = Instant::now();
let mut depth = 0;
let mut queue: VecDeque<(usize, &Point)> = VecDeque::new();
let mut r_queue: VecDeque<(usize, &Point)> = VecDeque::new();
queue.push_front((0, &start_sys));
r_queue.push_front((0, &goal_sys));
seen.insert(start_sys.id);
while let Some((d, sys)) = queue.pop_front() {
if d != depth {
depth = d;
print!(
"\r[{}] Depth: {}, Queue: {}, Seen: {} ({:.2}%) ",
format_duration(t_start.elapsed()),
d,
queue.len(),
prev.len(),
((prev.len() as f32) / total) * 100.0
);
std::io::stdout().flush().unwrap();
}
if sys.id == goal_sys.id {
println!();
let points = self.preload_points();
let mut v: Vec<(&System, &Point)> = Vec::new();
let mut prev_sys_id = sys.id;
loop {
if let Some(sys) = self.systems.get(&prev_sys_id) {
v.push((sys, points[&sys.id]));
} else {
break;
};
match prev.get(&prev_sys_id) {
Some(sys_id) => prev_sys_id = *sys_id,
None => {
break;
}
}
}
v.reverse();
return v;
}
let nbs = self
.neighbours(&sys)
.filter(|&nb| (self.valid(nb) || (nb.id == goal_sys.id)));
for nb in nbs {
if seen.insert(nb.id) {
prev.insert(nb.id, sys.id);
queue.push_back((d + 1, nb));
}
}
}
println!("No route found!");
return Vec::new();
}
}
fn main() {
let path = r#"D:\devel\files\python\EDSM\stars.csv"#;
let t_load = Instant::now();
let mut router: Router = Router::new(path, 48.0);
println!("Done in {}!", format_duration(t_load.elapsed()));
let t_route = Instant::now();
let route = router.route(
&[-65.21875, 7.75, -111.03125], // Ix
&[-1111.5625, -134.21875, 65269.75], // Beagle Point
// &[-7095.375, 401.25, 2396.8125], // V1357 Cygni,
); // Ix -> BP 537
println!(
"Done in {} ({} Jumps)!\n",
format_duration(t_route.elapsed()),
route.len()
);
let mut total: f32 = 0.0;
for ((sys1, p1), (_, p2)) in route.iter().zip(route.iter().skip(1)) {
let dist = p1.distp(p2);
total += dist;
println!("{} [{}]: {:.2} Ly", sys1.name, sys1.star_type, dist);
}
let sys = route.iter().last().unwrap().0;
println!("{} [{}]: {:.2} Ly\n", sys.name, sys.star_type, 0.0);
println!("Total: {:.2} Ly", total);
}