From 84bc08a2e1cd13ebbeee2488a6792d73f840ccdd Mon Sep 17 00:00:00 2001 From: Daniel Seiller Date: Sat, 29 Jun 2019 10:32:47 +0200 Subject: [PATCH] Big update - Add ed_lrr_pp to preprocess JSON-Dumps into CSV files - Add route-graph-precomputation to ed_lrr - Add option to only route through primary stars --- .cargo/config | 2 + .gitignore | 6 +- Cargo.lock | 543 ++++++++++++++++++++++++++++++++++--- Cargo.toml | 21 +- README.md | 14 +- dumps/process.py | 2 +- dumps/urls.txt | 2 + src/bin/ed_lrr_pp.rs | 188 +++++++++++++ src/main.rs | 632 ++++++++++++++++++++++++++----------------- 9 files changed, 1109 insertions(+), 301 deletions(-) create mode 100644 .cargo/config create mode 100644 dumps/urls.txt create mode 100644 src/bin/ed_lrr_pp.rs diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000..f0b7483 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,2 @@ +[build] +rustflags = ["-C", "target-cpu=native"] \ No newline at end of file diff --git a/.gitignore b/.gitignore index 832207a..ac00f51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ /target **/*.rs.bk -.vscode/** \ No newline at end of file +.vscode/** +*.csv +*.router +dumps/*.json +plot.py diff --git a/Cargo.lock b/Cargo.lock index a5bbd6e..67bca1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,13 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -13,8 +21,8 @@ name = "atty" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -23,11 +31,66 @@ name = "autocfg" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bincode" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bitflags" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "block-padding" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bstr" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-automata 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "clap" version = "2.33.0" @@ -43,36 +106,85 @@ dependencies = [ ] [[package]] -name = "csv" -version = "1.0.7" +name = "clicolors-control" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "console" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "clicolors-control 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "csv" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bstr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "csv-core 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "csv-core" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "digest" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ed_lrr" version = "0.1.0" dependencies = [ - "csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "csv 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "indicatif 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "permutohedron 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "rstar 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -80,11 +192,29 @@ name = "either" version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "encode_unicode" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "fnv" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "heck" version = "0.3.1" @@ -101,6 +231,18 @@ dependencies = [ "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "indicatif" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "console 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itertools" version = "0.8.0" @@ -115,16 +257,34 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "libc" -version = "0.2.55" +name = "keccak" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "lazy_static" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.58" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lock_api" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "memchr" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -135,11 +295,49 @@ dependencies = [ "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "number_prefix" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "numtoa" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "opaque-debug" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "parking_lot" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot_core" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pdqselect" version = "0.1.0" @@ -171,6 +369,110 @@ dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "redox_syscall" version = "0.1.54" @@ -184,6 +486,34 @@ dependencies = [ "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "regex" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-automata" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rstar" version = "0.4.0" @@ -192,6 +522,15 @@ dependencies = [ "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -200,20 +539,73 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "serde" -version = "1.0.92" +name = "ryu" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_derive" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.39 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_json" +version = "1.0.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha3" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "smallvec" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "strsim" version = "0.8.0" @@ -221,27 +613,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "structopt" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt-derive 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt-derive 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "structopt-derive" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.39 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "syn" -version = "0.15.34" +version = "0.15.39" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", @@ -251,15 +643,23 @@ dependencies = [ [[package]] name = "termion" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", "numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "termios" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -268,6 +668,24 @@ dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "typenum" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ucd-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-segmentation" version = "1.3.0" @@ -283,6 +701,11 @@ name = "unicode-xid" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "utf8-ranges" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "vec_map" version = "0.8.1" @@ -308,43 +731,93 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] +"checksum aho-corasick 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e6f484ae0c99fec2e858eb6134949117399f222608d84cadb3f58c1f97c2364c" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" "checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" +"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" "checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd" +"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +"checksum block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc3af3ee2e12f3e5d224e5e1e3d73668abbeb69e566d361f7d5563a4fdf09" +"checksum bstr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6cc0572e02f76cb335f309b19e0a0d585b4f62788f7d26de2a13a836a637385f" +"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b486ce3ccf7ffd79fdeb678eac06a9e6c09fc88d33836340becb8fffe87c5e33" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" -"checksum csv 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9044e25afb0924b5a5fc5511689b0918629e85d68ea591e5e87fbf1e85ea1b3b" -"checksum csv-core 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa5cdef62f37e6ffe7d1f07a381bc0db32b7a3ff1cac0de56cb0d81e71f53d65" +"checksum clicolors-control 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "73abfd4c73d003a674ce5d2933fca6ce6c42480ea84a5ffe0a2dc39ed56300f9" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum console 0.7.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8ca57c2c14b8a2bf3105bc9d15574aad80babf6a9c44b1058034cdf8bd169628" +"checksum csv 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a54cd62557f353f140b42305fb4efcff2ae08e32fbabaf5b0929423000febb63" +"checksum csv-core 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9b5cadb6b25c77aeff80ba701712494213f4a8418fcda2ee11b6560c3ad0bf4c" +"checksum digest 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05f47366984d3ad862010e22c7ce81a7dbcaebbdfb37241a620f8b6596ee135c" "checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" +"checksum encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "90b2c9496c001e8cb61827acdefad780795c42264c137744cae6f7d9e3450abd" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" +"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" "checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" +"checksum indicatif 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c60da1c9abea75996b70a931bba6c750730399005b61ccd853cee50ef3d0d0c" "checksum itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5b8467d9c1cebe26feb08c640139247fac215782d35371ade9a2136ed6085358" "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" -"checksum libc 0.2.55 (registry+https://github.com/rust-lang/crates.io-index)" = "42914d39aad277d9e176efbdad68acb1d5443ab65afe0e0e4f0d49352a950880" +"checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" +"checksum lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bc5729f27f159ddd61f4df6228e827e86643d4d3e7c32183cb30a1c08f604a14" +"checksum libc 0.2.58 (registry+https://github.com/rust-lang/crates.io-index)" = "6281b86796ba5e4366000be6e9e18bf35580adf9e63fbe2294aadb587613a319" +"checksum lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed946d4529956a20f2d63ebe1b69996d5a2137c91913fe3ebbeff957f5bca7ff" "checksum memchr 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2efc7bc57c883d4a4d6e3246905283d8dae951bb3bd32f49d6ef297f546e1c39" "checksum num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "6ba9a427cfca2be13aa6f6403b0b7e7368fe982bfa16fccc450ce74c46cd9b32" +"checksum number_prefix 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dbf9993e59c894e3c08aa1c2712914e9e6bf1fcbfc6bef283e2183df345a4fee" "checksum numtoa 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b8f8bdf33df195859076e54ab11ee78a1b208382d3a26ec40d142ffc1ecc49ef" +"checksum opaque-debug 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "93f5bb2e8e8dec81642920ccff6b61f1eb94fa3020c5a325c9851ff604152409" +"checksum parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fa7767817701cce701d5585b9c4db3cdd02086398322c1d7e8bf5094a96a2ce7" +"checksum parking_lot_core 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cb88cb1cb3790baa6776844f968fea3be44956cf184fa1be5a03341f5491278c" "checksum pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ec91767ecc0a0bbe558ce8c9da33c068066c57ecc8bb8477ef8c1ad3ef77c27" "checksum permutohedron 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b687ff7b5da449d39e418ad391e5e08da53ec334903ddbb921db208908fc372c" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" +"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +"checksum rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d0e7a549d590831370895ab7ba4ea0c1b6b011d106b5ff2da6eee112615e6dc0" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.54 (registry+https://github.com/rust-lang/crates.io-index)" = "12229c14a0f65c4f1cb046a3b52047cdd9da1f4b30f8a39c5063c8bae515e252" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum regex 1.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "0b2f0808e7d7e4fb1cb07feb6ff2f4bc827938f24f8c2e6a3beb7370af544bdd" +"checksum regex-automata 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "3ed09217220c272b29ef237a974ad58515bde75f194e3ffa7e6d0bf0f3b01f86" +"checksum regex-syntax 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "9d76410686f9e3a17f06128962e0ecc5755870bb890c34820c7af7f1db2e1d48" "checksum rstar 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dd08ae4f9661517777346592956ea6cdbba2895a28037af7daa600382f4b4001" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "b96a9549dc8d48f2c283938303c4b5a77aa29bfbc5b54b084fb1630408899a8f" -"checksum serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "32746bf0f26eab52f06af0d0aa1984f641341d06d8d673c693871da2d188c9be" -"checksum serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)" = "46a3223d0c9ba936b61c0d2e3e559e3217dbfb8d65d06d26e8b3c25de38bae3e" +"checksum ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c92464b447c0ee8c4fb3824ecc8383b81717b9f1e74ba2e72540aef7b9f82997" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)" = "076a696fdea89c19d3baed462576b8f6d663064414b5c793642da8dfeb99475b" +"checksum serde_derive 1.0.94 (registry+https://github.com/rust-lang/crates.io-index)" = "ef45eb79d6463b22f5f9e16d283798b7c0175ba6050bc25c1a946c122727fe7b" +"checksum serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)" = "5a23aa71d4a4d43fdbfaac00eff68ba8a06a51759a89ac3304323e800c4dd40d" +"checksum sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd26bc0e7a2e3a7c959bc494caf58b72ee0c71d67704e9520f736ca7e4853ecf" +"checksum smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ab606a9c5e214920bb66c458cd7be8ef094f813f20fe77a54cc7dbfff220d4b7" "checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" -"checksum structopt 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "fa19a5a708e22bb5be31c1b6108a2a902f909c4b9ba85cba44c06632386bc0ff" -"checksum structopt-derive 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "c6d59d0ae8ef8de16e49e3ca7afa16024a3e0dfd974a75ef93fdc5464e34523f" -"checksum syn 0.15.34 (registry+https://github.com/rust-lang/crates.io-index)" = "a1393e4a97a19c01e900df2aec855a29f71cf02c402e2f443b8d2747c25c5dbe" -"checksum termion 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dde0593aeb8d47accea5392b39350015b5eccb12c0d98044d856983d89548dea" +"checksum structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "16c2cdbf9cc375f15d1b4141bc48aeef444806655cd0e904207edc8d68d86ed7" +"checksum structopt-derive 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "53010261a84b37689f9ed7d395165029f9cc7abb9f56bbfe86bee2597ed25107" +"checksum syn 0.15.39 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d960b829a55e56db167e861ddb43602c003c7be0bee1d345021703fac2fb7c" +"checksum termion 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6a8fb22f7cde82c8220e5aeacb3258ed7ce996142c77cba193f203515e26c330" +"checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" +"checksum ucd-util 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535c204ee4d8434478593480b8f86ab45ec9aae0e83c568ca81abf0fd0e88f86" "checksum unicode-segmentation 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1967f4cdfc355b37fd76d2a954fb2ed3871034eb4f26d60537d88795cfc332a9" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum utf8-ranges 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9d50aa7650df78abf942826607c62468ce18d9019673d4a2ebe1865dbb96ffde" "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "f10e386af2b13e47c89e7236a7a14a086791a2b88ebad6df9bf42040195cf770" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" diff --git a/Cargo.toml b/Cargo.toml index 05abe0f..0283f87 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,16 +6,19 @@ edition = "2018" repository = "https://gitlab.com/Earthnuker/ed_lrr.git" license = "WTFPL" -[[bin]] -name = "ed_lrr" -path = "src/main.rs" +[profile.release] +# debug=true [dependencies] -csv = "1.0.7" -serde = "1.0.92" -serde_derive = "1.0.92" -rstar = "0.4.0" +csv = "1.1.0" +serde = "1.0.94" +serde_derive = "1.0.94" +rstar = {version="0.4.0",features=["serde"]} humantime = "1.2.0" -fnv = "1.0.6" -structopt = "0.2.16" +structopt = "0.2.18" permutohedron = "0.2.4" +serde_json = "1.0.39" +indicatif = "0.11.0" +fnv = "1.0.6" +bincode = "1.1.4" +sha3 = "0.8.2" diff --git a/README.md b/README.md index 3172f13..46f73b0 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,18 @@ ## Usage: 1. download `bodies.json` and `systemsWithCoordinates.json` from https://www.edsm.net/en/nightly-dumps/ and place them in the `dumps` folder -2. run `process.py` in the `dumps` folder -3. run `cargo install --path .` or `cargo install --git https://gitlab.com/Earthnuker/ed_lrr.git` -5. run `ed_lrr --help` +2. run `cargo install --path .` or `cargo install --git https://gitlab.com/Earthnuker/ed_lrr.git` +3. run `ed_lrr_pp --bodies dumps/bodies.json --systems dumps/systemsWithCoordinates.json` + - Alternatively run `proces.py` in the `dumps` folder +4. run `ed_lrr --help` ## Dependencies -- Python 3.7 +- Working nightly Rust Compiler (tested with `rustc 1.37.0-nightly (5d8f59f4b 2019-06-04)`) +- ~8GB of free RAM +- Optional: + - Python 3.7 - Pandas - uJSON -- Working nightly Rust Compiler (tested with `rustc 1.37.0-nightly (5d8f59f4b 2019-06-04)`) -- ~8GB of free RAM \ No newline at end of file diff --git a/dumps/process.py b/dumps/process.py index 02b8102..039d9e4 100644 --- a/dumps/process.py +++ b/dumps/process.py @@ -165,7 +165,7 @@ if not os.path.isfile("stars.csv"): print() cache.close() -if not os.path.isfile("stars.kdt"): +if not os.path.isfile("stars.csv"): tqdm.pandas(ascii=True, leave=True) print("Loading data...") data = pd.read_csv( diff --git a/dumps/urls.txt b/dumps/urls.txt new file mode 100644 index 0000000..58a4f38 --- /dev/null +++ b/dumps/urls.txt @@ -0,0 +1,2 @@ +https://www.edsm.net/dump/systemsWithCoordinates.json +https://www.edsm.net/dump/bodies.json \ No newline at end of file diff --git a/src/bin/ed_lrr_pp.rs b/src/bin/ed_lrr_pp.rs new file mode 100644 index 0000000..df0ec67 --- /dev/null +++ b/src/bin/ed_lrr_pp.rs @@ -0,0 +1,188 @@ +#![feature(seek_convenience)] +use fnv::FnvHashMap; +use humantime::format_duration; +use indicatif::{ProgressBar, ProgressStyle}; +use serde::{Deserialize, Serialize}; +use serde_json::Result; +use std::fs::File; +use std::io::{BufRead, BufReader, BufWriter, Seek}; +use std::path::PathBuf; +use std::str; +use std::time::Instant; +use structopt::StructOpt; + +#[derive(Debug, Clone, Serialize)] +struct Record { + id: u64, + star_type: String, + name: String, + mult: f32, + distance: u32, + x: f64, + y: f64, + z: f64, +} +#[derive(Debug, Deserialize)] +#[allow(non_snake_case)] +struct Body { + name: String, + subType: String, + #[serde(rename = "type")] + body_type: String, + systemId: i32, + systemId64: i64, + #[serde(rename = "distanceToArrival")] + distance: u32, +} + +#[derive(Debug, Deserialize)] +struct Coords { + x: f64, + y: f64, + z: f64, +} + +#[derive(Debug, Deserialize)] +struct System { + id: i32, + id64: i64, + name: String, + coords: Coords, + date: String, +} + +#[derive(Debug, StructOpt)] +#[structopt( + name = "ed_lrr_pp", + about = "Preprocessor for Elite: Dangerous Long-Range Router", + rename_all = "snake_case" +)] +/// Preprocess data for ed_lrr +struct Opt { + #[structopt(short, long = "bodies")] + /// Path to bodies.json + bodies: PathBuf, + #[structopt(short, long = "systems")] + /// Path to systemsWithCoordinates.json + systems: PathBuf, + #[structopt(default_value = "stars")] + /// outfile prefix + prefix: String, +} + +fn get_mult(star_type: &str) -> f32 { + if star_type.contains("White Dwarf") { + return 1.5; + } + if star_type.contains("Neutron") { + return 4.0; + } + return 1.0; +} + +fn process(path: &PathBuf, func: &mut dyn for<'r> FnMut(&'r str) -> ()) -> std::io::Result<()> { + let mut cnt = 0; + let mut buffer = String::new(); + let t_start = Instant::now(); + let fh = File::open(path)?; + let prog_bar = ProgressBar::new(fh.metadata()?.len()); + prog_bar.set_style( + ProgressStyle::default_bar() + .template( + "[{elapsed_precise}/{eta_precise}]{spinner} [{wide_bar}] {binary_bytes}/{binary_total_bytes} ({percent}%)", + ) + .progress_chars("#9876543210 ") + .tick_chars("/-\\|"), + ); + prog_bar.set_draw_delta(1024 * 1024); + let mut reader = BufReader::new(fh); + println!("Loading {} ...", path.to_str().unwrap()); + while let Ok(n) = reader.read_line(&mut buffer) { + if n == 0 { + break; + } + buffer = buffer.trim_end().trim_end_matches(|c| c == ',').to_string(); + if !buffer.is_empty() { + func(&buffer); + } + prog_bar.set_position(reader.stream_position().unwrap()); + cnt += 1; + buffer.clear(); + } + prog_bar.finish_and_clear(); + println!( + "Processed {} lines in {} ...", + cnt, + format_duration(t_start.elapsed()) + ); + return Ok(()); +} + +fn process_systems(path: &PathBuf) -> FnvHashMap { + let mut ret = FnvHashMap::default(); + process(path, &mut |line| { + let sys_res: Result = serde_json::from_str(&line); + if let Ok(sys) = sys_res { + ret.insert(sys.id64, sys); + } else { + eprintln!("\nError parsing: {}\n\t{:?}\n", line, sys_res.unwrap_err()); + } + }) + .unwrap(); + return ret; +} + +fn process_bodies( + path: &PathBuf, + out_prefix: &String, + systems: &FnvHashMap, +) -> std::io::Result<()> { + let out_path = PathBuf::from(format!("{}.csv", out_prefix)); + println!( + "Processing {} into {} ...", + path.to_str().unwrap(), + out_path.to_str().unwrap(), + ); + let mut n: u64 = 0; + let mut wtr = csv::Writer::from_writer(BufWriter::new(File::create(out_path).unwrap())); + process(path, &mut |line| { + if !line.contains("Star") { + return; + } + let body_res: Result = serde_json::from_str(&line); + if let Ok(body) = body_res { + if !body.body_type.contains("Star") { + return; + } + if let Some(sys) = systems.get(&body.systemId64) { + let sub_type = body.subType; + let mult = get_mult(&sub_type); + let body_name = body.name; + let rec = Record { + id: n, + star_type: sub_type, + name: body_name, + mult, + distance: body.distance, + x: sys.coords.x, + y: sys.coords.y, + z: sys.coords.z, + }; + wtr.serialize(rec).unwrap(); + n += 1; + }; + } else { + eprintln!("\nError parsing: {}\n\t{:?}\n", line, body_res.unwrap_err()); + } + }) + .unwrap(); + println!("Total Systems: {}", n); + return Ok(()); +} + +fn main() -> std::io::Result<()> { + let opts = Opt::from_args(); + let systems = process_systems(&opts.systems); + process_bodies(&opts.bodies, &opts.prefix, &systems)?; + return Ok(()); +} diff --git a/src/main.rs b/src/main.rs index efa6305..8f66e74 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,49 +1,48 @@ -extern crate csv; -extern crate serde; -extern crate structopt; -#[macro_use] -extern crate serde_derive; -extern crate fnv; -extern crate humantime; -extern crate permutohedron; use core::cmp::Ordering; use fnv::{FnvHashMap, FnvHashSet}; use humantime::format_duration; use permutohedron::LexicalPermutation; use rstar::{PointDistance, RTree, RTreeObject, AABB}; +use serde::{Deserialize, Serialize}; +use sha3::{Digest, Sha3_256}; use std::collections::VecDeque; +use std::fs::File; use std::hash::{Hash, Hasher}; -use std::io::Write; +use std::io::{BufReader, BufWriter, Write}; use std::path::PathBuf; use std::str::FromStr; use std::time::Instant; use structopt::StructOpt; -#[derive(Debug, Deserialize)] -struct Record { - id: i64, - star_type: String, - name: String, - mult: f32, - x: f32, - y: f32, - z: f32, -} -#[derive(Debug, Clone)] -struct System { - id: i64, - star_type: String, - name: String, - mult: f32, +#[derive(Debug)] +#[allow(dead_code)] +enum StarType { + Neutron, + WhiteDwarf, + Scoopable, + NonScoopable, } -#[derive(Debug, Clone, Copy)] -struct Point { - id: i64, +#[derive(Debug, Clone, Deserialize)] +struct SystemSerde { + id: u32, + star_type: String, + name: String, + mult: f32, + distance: u32, x: f32, y: f32, z: f32, } +#[derive(Debug, Clone, Deserialize, Serialize)] +struct System { + id: u32, + star_type: String, + name: String, + mult: f32, + distance: u32, + pos: [f32; 3], +} #[derive(Debug)] enum Mode { @@ -68,29 +67,53 @@ impl FromStr for Mode { #[structopt( name = "ed_lrr", about = "Elite: Dangerous Long-Range Router", - rename_all = "kebab-case" + rename_all = "snake_case" )] -/// Plots a route through multiple systems using breadth-first search (Currently needs a lot of RAM (about 6GB), sorry) -struct Opt { +/// Plots a route through multiple systems +struct Opts { #[structopt(short, long = "range")] /// Jump Range - range: f32, + range: Option, #[structopt( parse(from_os_str), - short = "f", + short = "i", long = "path", default_value = "./stars.csv" )] /// Path to stars.csv /// - /// Generate using process.py (https://gitlab.com/Earthnuker/ed_lrr/raw/master/dumps/process.py) + /// Generate using ed_lrr_pp file_path: PathBuf, + #[structopt( + parse(from_os_str), + long = "precomp_file", + conflicts_with = "precompute" + )] + /// Path to precomputed route graph + /// + /// Generate using --precompute option + precomp_file: Option, + + #[structopt( + short = "c", + long = "precompute", + conflicts_with = "full_permute", + conflicts_with = "permute", + conflicts_with = "precomp_file" + )] + /// Precompute all routes for the specified jump range starting at the specified system and write the result to {system}_{range}.bin + precompute: bool, + #[structopt(short = "p", long = "permute", conflicts_with = "full_permute")] /// Permute intermediate hops to find shortest route permute: bool, - #[structopt(short = "fp", long = "full_permute", conflicts_with = "permute")] + #[structopt(short = "o", long = "primary")] + /// Only route through the primary star of a system + primary: bool, + + #[structopt(short = "f", long = "full_permute", conflicts_with = "permute")] /// Permute all hops to find shortest route full_permute: bool, @@ -100,8 +123,9 @@ struct Opt { #[structopt( short, - long = "mode", - raw(possible_values = "&[\"bfs\", \"greedy\",\"astar\"]") + long, + raw(possible_values = "&[\"bfs\", \"greedy\",\"astar\"]"), + default_value = "bfs" )] /// Search mode /// @@ -114,15 +138,19 @@ struct Opt { /// Systems to route through systems: Vec, } -fn dist(p1: &[f32; 3], p2: &[f32; 3]) -> f32 { +fn dist2(p1: &[f32; 3], p2: &[f32; 3]) -> f32 { let dx = p1[0] - p2[0]; let dy = p1[1] - p2[1]; let dz = p1[2] - p2[2]; - return (dx * dx + dy * dy + dz * dz).sqrt(); + return dx * dx + dy * dy + dz * dz; } -fn fcmp(a: &f32, b: &f32) -> Ordering { +fn dist(p1: &[f32; 3], p2: &[f32; 3]) -> f32 { + return dist2(p1, p2).sqrt(); +} + +fn fcmp(a: f32, b: f32) -> Ordering { match (a, b) { (x, y) if x.is_nan() && y.is_nan() => Ordering::Equal, (x, _) if x.is_nan() => Ordering::Greater, @@ -131,80 +159,83 @@ fn fcmp(a: &f32, b: &f32) -> Ordering { } } -impl Point { +impl System { pub fn dist2(&self, p: &[f32; 3]) -> f32 { - let dx = self.x - p[0]; - let dy = self.y - p[1]; - let dz = self.z - p[2]; - - return dx * dx + dy * dy + dz * dz; + return dist2(&self.pos, p); } - pub fn distp(&self, p: &Point) -> f32 { - return self.distp2(p).sqrt(); + pub fn distp(&self, p: &System) -> f32 { + return dist(&self.pos, &p.pos); } - pub fn distp2(&self, p: &Point) -> f32 { - return self.dist2(&[p.x, p.y, p.z]); + pub fn distp2(&self, p: &System) -> f32 { + return self.dist2(&p.pos); } } -impl PartialEq for Point { +impl PartialEq for System { fn eq(&self, other: &Self) -> bool { self.id == other.id } } -impl Eq for Point {} +impl Eq for System {} -impl Hash for Point { +impl Hash for System { fn hash(&self, state: &mut H) { self.id.hash(state); } } -impl RTreeObject for Point { +impl RTreeObject for System { type Envelope = AABB<[f32; 3]>; fn envelope(&self) -> Self::Envelope { - return AABB::from_point([self.x, self.y, self.z]); + return AABB::from_point(self.pos); } } -impl PointDistance for Point { +impl PointDistance for System { fn distance_2(&self, point: &[f32; 3]) -> f32 { return self.dist2(&point); } } +fn hash_file(path: &PathBuf) -> Vec { + let mut hash_reader = BufReader::new(File::open(path).unwrap()); + let mut hasher = Sha3_256::new(); + std::io::copy(&mut hash_reader, &mut hasher).unwrap(); + hasher.result().iter().map(|v| *v).collect() +} + +#[derive(Deserialize, Serialize)] struct Router { - tree: RTree, - systems: FnvHashMap, - scoopable: FnvHashSet, + tree: RTree, + scoopable: FnvHashSet, + route_tree: Option>, + range: f32, + primary_only: bool, } impl Router { - pub fn new(path: &PathBuf) -> Self { + pub fn new(path: &PathBuf, range: f32, primary_only: bool) -> Self { let mut scoopable = FnvHashSet::default(); - let mut systems: FnvHashMap = FnvHashMap::default(); let mut reader = csv::ReaderBuilder::new() - .has_headers(false) .from_path(path) .unwrap_or_else(|e| { - println!("Error loading {}: {}", path.to_str().unwrap(), e); + println!("Error opening {}: {}", path.to_str().unwrap(), e); std::process::exit(1); }); + let t_load = Instant::now(); println!("Loading {}...", path.to_str().unwrap()); - let points = reader - .deserialize() - .map(|res: Result| { - let sys = res.unwrap(); - systems.insert( - sys.id, - System { - id: sys.id, - star_type: sys.star_type.clone(), - name: sys.name, - mult: sys.mult, - }, - ); + let systems: Vec = reader + .deserialize::() + .map(|res| res.unwrap()) + .filter(|sys| { + if primary_only { + sys.distance == 0 + } else { + true + } + }) + .map(|sys| { if sys.mult > 1.0f32 { scoopable.insert(sys.id); } else { @@ -215,61 +246,79 @@ impl Router { } } } - return Point { - x: sys.x, - y: sys.y, - z: sys.z, + return System { id: sys.id, + star_type: sys.star_type, + name: sys.name, + mult: sys.mult, + distance: sys.distance, + pos: [sys.x, sys.y, sys.z], }; }) .collect(); - return Self { - tree: RTree::::bulk_load(points), - systems, + println!("Building RTree..."); + let ret = Self { + tree: RTree::bulk_load(systems), scoopable, + route_tree: None, + range, + primary_only, }; - } - - fn preload_points(&self) -> FnvHashMap { - let mut ret = FnvHashMap::default(); - for point in &self.tree { - ret.insert(point.id, point); - } + println!( + "{} Systems loaded in {}", + ret.tree.size(), + format_duration(t_load.elapsed()) + ); return ret; } - fn closest(&self, point: &[f32; 3]) -> &Point { + pub fn from_file(filename: &PathBuf) -> Self { + let t_load = Instant::now(); + let mut reader = BufReader::new(File::open(&filename).unwrap()); + println!("Loading {}", filename.to_str().unwrap()); + let (primary, range, file_hash, path, route_tree): ( + bool, + f32, + Vec, + String, + FnvHashMap, + ) = bincode::deserialize_from(&mut reader).unwrap(); + let path = PathBuf::from(path); + println!("Done in {}!", format_duration(t_load.elapsed())); + if hash_file(&path) != file_hash { + panic!("File hash mismatch!") + } + let mut rtr = Self::new(&path, range, primary); + rtr.route_tree = Some(route_tree); + return rtr; + } + + fn closest(&self, point: &[f32; 3]) -> &System { return self.tree.nearest_neighbor(point).unwrap(); } - fn points_in_sphere(&self, center: &[f32; 3], radius: f32) -> impl Iterator { + fn points_in_sphere(&self, center: &[f32; 3], radius: f32) -> impl Iterator { return self.tree.locate_within_distance(*center, radius * radius); } - fn mult(&self, id: i64) -> f32 { - if let Some(sys) = self.systems.get(&id) { - return sys.mult; - }; - return 1.0; - } - fn neighbours(&self, sys: &Point, r: f32) -> impl Iterator { - return self.points_in_sphere(&[sys.x, sys.y, sys.z], self.mult(sys.id) * r); + fn neighbours(&self, sys: &System, r: f32) -> impl Iterator { + return self.points_in_sphere(&sys.pos, sys.mult * r); } - fn valid(&self, sys: &Point) -> bool { + fn valid(&self, sys: &System) -> bool { return self.scoopable.contains(&sys.id); } pub fn best_name_multiroute( &self, - waypoints: &Vec, + waypoints: &[String], range: f32, full: bool, mode: Mode, factor: f32, - ) -> Vec<(&System, &Point)> { + ) -> Vec { let mut best_score: f32 = std::f32::MAX; - let mut waypoints = waypoints.clone(); - let mut best_permutation_waypoints = waypoints.clone(); + let mut waypoints = waypoints.to_owned(); + let mut best_permutation_waypoints = waypoints.to_owned(); let first = waypoints.first().cloned(); let last = waypoints.last().cloned(); let t_start = Instant::now(); @@ -283,7 +332,11 @@ impl Router { for pair in waypoints.windows(2) { match pair { [src, dst] => { - let (src, dst) = (self.name_to_point(&src), self.name_to_point(&dst)); + let (mut src, dst) = + (self.name_to_systems(&src), self.name_to_systems(&dst)); + src.sort_by_key(|&p| (p.mult * 10.0) as u8); + let src = src.last().unwrap(); + let dst = dst.last().unwrap(); total_d += src.distp2(dst); } _ => panic!("Invalid routing parameters!"), @@ -291,7 +344,7 @@ impl Router { } if total_d < best_score { best_score = total_d; - best_permutation_waypoints = waypoints.clone(); + best_permutation_waypoints = waypoints.to_owned(); } } if !waypoints.next_permutation() { @@ -306,39 +359,44 @@ impl Router { pub fn name_multiroute( &self, - waypoints: &Vec, + waypoints: &[String], range: f32, mode: Mode, factor: f32, - ) -> Vec<(&System, &Point)> { + ) -> Vec { let mut coords = Vec::new(); - for p in waypoints { - let p = self.name_to_point(p); - let s = [p.x, p.y, p.z]; - coords.push(s); + for p_name in waypoints { + let mut p_l = self.name_to_systems(p_name); + p_l.sort_by_key(|&p| (p.mult * 10.0) as u8); + let p = p_l.last().unwrap(); + coords.push((p_name, p.pos)); } return self.multiroute(coords.as_slice(), range, mode, factor); } pub fn multiroute( &self, - waypoints: &[[f32; 3]], + waypoints: &[(&String, [f32; 3])], range: f32, mode: Mode, factor: f32, - ) -> Vec<(&System, &Point)> { - let mut route = Vec::new(); + ) -> Vec { + let mut route: Vec = Vec::new(); for pair in waypoints.windows(2) { - match pair { - &[src, dst] => { + match *pair { + [src, dst] => { let block = match mode { Mode::BFS => self.route_bfs(&src, &dst, range), Mode::Greedy => self.route_greedy(&src, &dst, range), Mode::AStar => self.route_astar(&src, &dst, range, factor), }; if route.is_empty() { - route.extend(block.iter()); + for sys in block.iter() { + route.push(sys.clone()); + } } else { - route.extend(block.iter().skip(1)); + for sys in block.iter().skip(1) { + route.push(sys.clone()); + } } } _ => panic!("Invalid routing parameters!"), @@ -347,20 +405,10 @@ impl Router { return route; } - fn sys_to_point(&self, id: i64) -> &Point { - for p in &self.tree { - if p.id == id { - return p; - } - } - eprintln!("Sytem-ID not found: \"{}\"", id); - std::process::exit(1); - } - - fn name_to_point(&self, name: &str) -> &Point { - for sys in self.systems.values() { + fn name_to_systems(&self, name: &str) -> Vec<&System> { + for sys in &self.tree { if sys.name == name { - return self.sys_to_point(sys.id); + return self.neighbours(&sys, 0.0).collect(); } } eprintln!("Sytem not found: \"{}\"", name); @@ -369,25 +417,22 @@ impl Router { pub fn route_astar( &self, - src: &[f32; 3], - dst: &[f32; 3], + src: &(&String, [f32; 3]), + dst: &(&String, [f32; 3]), range: f32, factor: f32, - ) -> Vec<(&System, &Point)> { + ) -> Vec { if factor == 0.0 { return self.route_bfs(src, dst, range); } println!("Running A-Star with greedy factor of {}", factor); + let (src_name, src) = src; + let (dst_name, dst) = dst; let start_sys = self.closest(src); let goal_sys = self.closest(dst); - let start_sys_name = self.systems.get(&start_sys.id).unwrap().name.clone(); - let goal_sys_name = self.systems.get(&goal_sys.id).unwrap().name.clone(); { let d = dist(src, dst); - println!( - "Plotting route from {} to {}...", - start_sys_name, goal_sys_name - ); + println!("Plotting route from {} to {}...", src_name, dst_name); println!( "Jump Range: {} Ly, Distance: {} Ly, Theoretical Jumps: {}", range, @@ -401,7 +446,7 @@ impl Router { let t_start = Instant::now(); let mut found = false; let mut maxd = 0; - let mut queue: Vec<(usize, usize, &Point)> = Vec::new(); + let mut queue: Vec<(usize, usize, &System)> = Vec::new(); queue.push(( 0, // depth (start_sys.distp(goal_sys) / range) as usize, // h @@ -432,7 +477,7 @@ impl Router { .filter(|&nb| (self.valid(nb) || (nb.id == goal_sys.id))) .filter(|&nb| seen.insert(nb.id)) .map(|nb| { - prev.insert(nb.id, sys.id); + prev.insert(nb.id, sys); let d_g = (nb.distp(goal_sys) / range) as usize; return (depth + 1, d_g, nb); }), @@ -442,7 +487,7 @@ impl Router { let (b_0, b_1) = (b.0 as f32, b.1 as f32); let v_a = a_0 + a_1 * factor; let v_b = b_0 + b_1 * factor; - return fcmp(&v_a, &v_b); + return fcmp(v_a, v_b); }); // queue.reverse(); } @@ -451,23 +496,15 @@ impl Router { println!(); if !found { - eprintln!( - "No route from {} to {} found!", - start_sys_name, goal_sys_name - ); + eprintln!("No route from {} to {} found!", src_name, dst_name); return Vec::new(); } - let points = self.preload_points(); - let mut v: Vec<(&System, &Point)> = Vec::new(); - let mut prev_sys_id = goal_sys.id; + let mut v: Vec = Vec::new(); + let mut curr_sys = goal_sys; loop { - if let Some(sys) = self.systems.get(&prev_sys_id) { - v.push((sys, points[&sys.id])); - } else { - break; - }; - match prev.get(&prev_sys_id) { - Some(sys_id) => prev_sys_id = *sys_id, + v.push(curr_sys.clone()); + match prev.get(&curr_sys.id) { + Some(sys) => curr_sys = *sys, None => { break; } @@ -479,21 +516,18 @@ impl Router { pub fn route_greedy( &self, - src: &[f32; 3], - dst: &[f32; 3], + src: &(&String, [f32; 3]), + dst: &(&String, [f32; 3]), range: f32, - ) -> Vec<(&System, &Point)> { + ) -> Vec { println!("Running Greedy-Search"); + let (src_name, src) = src; + let (dst_name, dst) = dst; let start_sys = self.closest(src); let goal_sys = self.closest(dst); - let start_sys_name = self.systems.get(&start_sys.id).unwrap().name.clone(); - let goal_sys_name = self.systems.get(&goal_sys.id).unwrap().name.clone(); { let d = dist(src, dst); - println!( - "Plotting route from {} to {}...", - start_sys_name, goal_sys_name - ); + println!("Plotting route from {} to {}...", src_name, dst_name); println!( "Jump Range: {} Ly, Distance: {} Ly, Theoretical Jumps: {}", range, @@ -507,13 +541,8 @@ impl Router { let t_start = Instant::now(); let mut found = false; let mut maxd = 0; - let mut queue: Vec<(f32, f32, usize, &Point)> = Vec::new(); - queue.push(( - -self.mult(goal_sys.id), - start_sys.distp2(goal_sys), - 0, - &start_sys, - )); + let mut queue: Vec<(f32, f32, usize, &System)> = Vec::new(); + queue.push((-goal_sys.mult, start_sys.distp2(goal_sys), 0, &start_sys)); seen.insert(start_sys.id); while !(queue.is_empty() || found) { while let Some((_, _, depth, sys)) = queue.pop() { @@ -538,33 +567,26 @@ impl Router { .filter(|&nb| (self.valid(nb) || (nb.id == goal_sys.id))) .filter(|&nb| seen.insert(nb.id)) .map(|nb| { - prev.insert(nb.id, sys.id); - return (-self.mult(nb.id), nb.distp2(goal_sys), depth + 1, nb); + prev.insert(nb.id, sys); + return (-nb.mult, nb.distp2(goal_sys), depth + 1, nb); }), ); - queue.sort_by(|a, b| fcmp(&a.0, &b.0).then(fcmp(&a.1, &b.1))); + queue.sort_by(|a, b| fcmp(a.0, b.0).then(fcmp(a.1, b.1))); queue.reverse(); } } println!(); + println!(); if !found { - eprintln!( - "No route from {} to {} found!", - start_sys_name, goal_sys_name - ); + eprintln!("No route from {} to {} found!", src_name, dst_name); return Vec::new(); } - let points = self.preload_points(); - let mut v: Vec<(&System, &Point)> = Vec::new(); - let mut prev_sys_id = goal_sys.id; + let mut v: Vec = Vec::new(); + let mut curr_sys = goal_sys; loop { - if let Some(sys) = self.systems.get(&prev_sys_id) { - v.push((sys, points[&sys.id])); - } else { - break; - }; - match prev.get(&prev_sys_id) { - Some(sys_id) => prev_sys_id = *sys_id, + v.push(curr_sys.clone()); + match prev.get(&curr_sys.id) { + Some(sys) => curr_sys = *sys, None => { break; } @@ -574,18 +596,97 @@ impl Router { return v; } - pub fn route_bfs(&self, src: &[f32; 3], dst: &[f32; 3], range: f32) -> Vec<(&System, &Point)> { + pub fn precompute(&mut self, src: &String) { + println!("Precomputing route starting at {} ...", src); + let mut sys_l = self.name_to_systems(src); + sys_l.sort_by_key(|&sys| (sys.mult * 10.0) as u8); + let sys = sys_l.last().unwrap(); + let total = self.tree.size() as f32; + let mut prev = FnvHashMap::default(); + let mut seen = FnvHashSet::default(); + let t_start = Instant::now(); + let mut depth = 0; + let mut queue: VecDeque<(usize, &System)> = VecDeque::new(); + let mut queue_next: VecDeque<(usize, &System)> = VecDeque::new(); + queue.push_front((0, &sys)); + seen.insert(sys.id); + while !queue.is_empty() { + print!( + "[{}] Depth: {}, Queue: {}, Seen: {} ({:.02}%) \r", + format_duration(t_start.elapsed()), + depth, + queue.len(), + seen.len(), + ((seen.len() * 100) as f32) / total + ); + std::io::stdout().flush().unwrap(); + while let Some((d, sys)) = queue.pop_front() { + queue_next.extend( + self.neighbours(&sys, self.range) + .filter(|&nb| seen.insert(nb.id)) + .map(|nb| { + prev.insert(nb.id, sys.id); + return (d + 1, nb); + }), + ); + } + std::mem::swap(&mut queue, &mut queue_next); + depth += 1; + } + self.route_tree = Some(prev); + } + + pub fn route_to(&mut self, dst: &str) -> Vec { + let mut id_map: FnvHashMap = FnvHashMap::default(); + let mut id_set: FnvHashSet = FnvHashSet::default(); + let prev = self.route_tree.as_ref().unwrap(); + let mut dst = self.name_to_systems(dst); + dst.sort_by_key(|&p| (p.mult * 10.0) as u8); + let dst: System = dst.last().cloned().unwrap().clone(); + if !prev.contains_key(&dst.id) { + eprintln!("System-ID {} not found", dst.id); + std::process::exit(1); + }; + let mut v_ids: Vec = Vec::new(); + let mut v: Vec = Vec::new(); + let mut curr_sys: u32 = dst.id; + loop { + v_ids.push(curr_sys); + id_set.insert(curr_sys); + match prev.get(&curr_sys) { + Some(sys_id) => curr_sys = *sys_id, + None => { + break; + } + } + } + v_ids.reverse(); + for sys in &self.tree { + if id_set.contains(&sys.id) { + id_map.insert(sys.id, &sys); + } + } + for sys_id in v_ids { + let sys = *(id_map.get(&sys_id).unwrap()); + v.push(sys.clone()) + } + return v; + } + + pub fn route_bfs( + &self, + src: &(&String, [f32; 3]), + dst: &(&String, [f32; 3]), + range: f32, + ) -> Vec { println!("Running BFS"); + let (src_name, src) = src; + let (dst_name, dst) = dst; let start_sys = self.closest(src); let goal_sys = self.closest(dst); - let start_sys_name = self.systems.get(&start_sys.id).unwrap().name.clone(); - let goal_sys_name = self.systems.get(&goal_sys.id).unwrap().name.clone(); { let d = dist(src, dst); - println!( - "Plotting route from {} to {}...", - start_sys_name, goal_sys_name - ); + println!("Plotting route from {} to {}...", src_name, dst_name); println!( "Jump Range: {} Ly, Distance: {} Ly, Theoretical Jumps: {}", range, @@ -599,8 +700,8 @@ impl Router { let t_start = Instant::now(); let mut depth = 0; let mut found = false; - let mut queue: VecDeque<(usize, &Point)> = VecDeque::new(); - let mut queue_next: VecDeque<(usize, &Point)> = VecDeque::new(); + let mut queue: VecDeque<(usize, &System)> = VecDeque::new(); + let mut queue_next: VecDeque<(usize, &System)> = VecDeque::new(); queue.push_front((0, &start_sys)); seen.insert(start_sys.id); while !(queue.is_empty() || found) { @@ -623,7 +724,7 @@ impl Router { .filter(|&nb| (self.valid(nb) || (nb.id == goal_sys.id))) .filter(|&nb| seen.insert(nb.id)) .map(|nb| { - prev.insert(nb.id, sys.id); + prev.insert(nb.id, sys); return (d + 1, nb); }), ); @@ -632,24 +733,17 @@ impl Router { depth += 1; } println!(); + println!(); if !found { - eprintln!( - "No route from {} to {} found!", - start_sys_name, goal_sys_name - ); + eprintln!("No route from {} to {} found!", src_name, dst_name); return Vec::new(); } - let points = self.preload_points(); - let mut v: Vec<(&System, &Point)> = Vec::new(); - let mut prev_sys_id = goal_sys.id; + let mut v: Vec = Vec::new(); + let mut curr_sys = goal_sys; loop { - if let Some(sys) = self.systems.get(&prev_sys_id) { - v.push((sys, points[&sys.id])); - } else { - break; - }; - match prev.get(&prev_sys_id) { - Some(sys_id) => prev_sys_id = *sys_id, + v.push(curr_sys.clone()); + match prev.get(&curr_sys.id) { + Some(sys) => curr_sys = *sys, None => { break; } @@ -661,50 +755,90 @@ impl Router { } fn main() { - let opts = Opt::from_args(); + let t_start = Instant::now(); + let opts = Opts::from_args(); + if opts.systems.len() == 0 { + if opts.precomp_file.is_some() { + eprintln!("Error: Please specify exatly one system"); + } else { + if opts.precompute { + eprintln!("Error: Please specify at least one system"); + } else { + eprintln!("Error: Please specify at least two systems"); + } + } + std::process::exit(1); + } let path = opts.file_path; - let t_load = Instant::now(); - let router: Router = Router::new(&path); - println!("Done in {}!", format_duration(t_load.elapsed())); - let t_route = Instant::now(); - let route = if opts.permute || opts.full_permute { - router.best_name_multiroute( - &opts.systems, - opts.range, - opts.full_permute, - opts.mode, - opts.factor.unwrap_or(1.0), - ) + let mut router: Router = if opts.precomp_file.is_some() { + Router::from_file(&opts.precomp_file.unwrap()) } else { - router.name_multiroute( - &opts.systems, - opts.range, - opts.mode, - opts.factor.unwrap_or(1.0), - ) + Router::new(&path, opts.range.unwrap(), opts.primary) }; - println!( - "Done in {} ({} Jumps)!\n", - format_duration(t_route.elapsed()), - route.len(), - ); - if route.len() == 0 { + if opts.precompute { + for sys in opts.systems { + router.route_tree = None; + router.precompute(&sys); + let ofn = format!( + "{}_{}{}.router", + sys.replace("*", "").replace(" ", "_"), + opts.range.unwrap(), + if router.primary_only { "_primary" } else { "" } + ); + println!("\nSaving to {}", ofn); + let mut out_fh = BufWriter::new(File::create(&ofn).unwrap()); + // (range, path, route_tree) + let data = ( + router.primary_only, + router.range, + hash_file(&path), + String::from(path.to_str().unwrap()), + router.route_tree.unwrap(), + ); + bincode::serialize_into(&mut out_fh, &data).unwrap(); + } + std::process::exit(0); + } + let t_route = Instant::now(); + let route = if router.route_tree.is_some() { + router.route_to(opts.systems.first().unwrap()) + } else { + if opts.permute || opts.full_permute { + router.best_name_multiroute( + &opts.systems, + opts.range.unwrap(), + opts.full_permute, + opts.mode, + opts.factor.unwrap_or(1.0), + ) + } else { + router.name_multiroute( + &opts.systems, + opts.range.unwrap(), + opts.mode, + opts.factor.unwrap_or(1.0), + ) + } + }; + println!("Route computed in {}\n", format_duration(t_route.elapsed())); + if route.is_empty() { eprintln!("No route found!"); return; } let mut total: f32 = 0.0; - for ((sys1, p1), (_, p2)) in route.iter().zip(route.iter().skip(1)) { - let dist = p1.distp(p2); + for (sys1, sys2) in route.iter().zip(route.iter().skip(1)) { + let dist = sys1.distp(sys2); total += dist; println!( - "{} [{}] ({},{},{}): {:.2} Ly", - sys1.name, sys1.star_type, p1.x, p1.y, p1.z, dist + "{} [{}] ({},{},{}) [{} Ls]: {:.2} Ly", + sys1.name, sys1.star_type, sys1.pos[0], sys1.pos[1], sys1.pos[2], sys1.distance, dist ); } - let (sys, p) = route.iter().last().unwrap(); + let sys = route.iter().last().unwrap(); println!( - "{} [{}] ({},{},{}): {:.2} Ly", - sys.name, sys.star_type, p.x, p.y, p.z, 0.0 + "{} [{}] ({},{},{}) [{} Ls]: {:.2} Ly", + sys.name, sys.star_type, sys.pos[0], sys.pos[1], sys.pos[2], sys.distance, 0.0 ); - println!("Total: {:.2} Ly", total); + println!("Total: {:.2} Ly ({} Jumps)", total, route.len()); + println!("Total time: {}", format_duration(t_start.elapsed())); }