diff --git a/.env b/.env new file mode 100644 index 0000000..40fe399 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +FLASK_APP="ed_lrr_gui.web:app" \ No newline at end of file diff --git a/.gitignore b/.gitignore index d5bb9d9..a79c276 100644 --- a/.gitignore +++ b/.gitignore @@ -17,9 +17,12 @@ build .history .tox pip-wheel-metadata -.eggs/ -exe/ -installer/Output/ -workspace.code-workspace -ed_lrr_gui/web/jobs.db -ed_lrr_gui/web/ed_lrr_web_ui.db +.eggs/ +exe/ +installer/Output/ +workspace.code-workspace +ed_lrr_gui/web/jobs.db +ed_lrr_gui/web/ed_lrr_web_ui.db +__version__.py +.nox/ +dist_vis.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5819736..21c64d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,12 @@ ## [Unreleased] -## [v0.24.0] - 2019-10-23 +## [v0.14.0] - 2019-10-23 ### Features - **tests:** Add pytest.ini for pytest-qt [eb43ca8](https://gitlab.com/Earthnuker/ed_lrr/commit/eb43ca8) -## [v0.23.0] - 2019-10-23 +## [v0.13.0] - 2019-10-23 ### Documentation - **readme:** Update README with more infos [634af75](https://gitlab.com/Earthnuker/ed_lrr/commit/634af75) @@ -14,22 +14,22 @@ - **setup.py:** Update add dependencies for HTTP API [45c11da](https://gitlab.com/Earthnuker/ed_lrr/commit/45c11da) -## [v0.22.0] - 2019-10-23 +## [v0.12.0] - 2019-10-23 ### Features - **tox:** Add more python versions to test against [3115f9d](https://gitlab.com/Earthnuker/ed_lrr/commit/3115f9d) -## [v0.21.0] - 2019-10-23 +## [v0.11.0] - 2019-10-23 ### Features - **html_export:** Add basic HTML-Export using Jinja2 template [bb2ee8c](https://gitlab.com/Earthnuker/ed_lrr/commit/bb2ee8c) -## [v0.20.0] - 2019-10-23 +## [v0.10.0] - 2019-10-23 ### Features - **route_progress:** minor change to progress dialog message [35fc135](https://gitlab.com/Earthnuker/ed_lrr/commit/35fc135) -## [v0.19.1] - 2019-10-23 +## [v0.9.4] - 2019-10-23 ### Bug Fixes - **config:** Add missing defaults [30dbd24](https://gitlab.com/Earthnuker/ed_lrr/commit/30dbd24) @@ -37,17 +37,17 @@ - Update CHANGELOG.md and README.md [38acfc7](https://gitlab.com/Earthnuker/ed_lrr/commit/38acfc7) -## [v0.19.0] - 2019-09-28 +## [v0.9.3] - 2019-09-28 ### Features - **config:** impelemt save and load to GUI [65fe131](https://gitlab.com/Earthnuker/ed_lrr/commit/65fe131) -## [v0.18.0] - 2019-09-28 +## [v0.9.2] - 2019-09-28 ### Features - **GUI:** implement preprocessing [f34d37a](https://gitlab.com/Earthnuker/ed_lrr/commit/f34d37a) -## [v0.17.0] - 2019-09-28 +## [v0.9.1] - 2019-09-28 ### Documentation - Update TODO list in readme [4663d4e](https://gitlab.com/Earthnuker/ed_lrr/commit/4663d4e) @@ -55,17 +55,17 @@ - **installer:** switch from LZMA to LZMA2 [3ee952e](https://gitlab.com/Earthnuker/ed_lrr/commit/3ee952e) -## [v0.16.0] - 2019-09-28 +## [v0.9.0] - 2019-09-28 ### Features - **build scripts:** skip .history folder when building UI files [92bd1b1](https://gitlab.com/Earthnuker/ed_lrr/commit/92bd1b1) -## [v0.15.0] - 2019-09-28 +## [v0.8.1] - 2019-09-28 ### Features - **config:** update config format [9ec4687](https://gitlab.com/Earthnuker/ed_lrr/commit/9ec4687) -## [v0.14.0] - 2019-09-28 +## [v0.8.0] - 2019-09-28 ### Features - **UI:** made dropdowns non-editable [37f5547](https://gitlab.com/Earthnuker/ed_lrr/commit/37f5547) @@ -73,12 +73,12 @@ - Alsways compile in release mode [e9bcd3e](https://gitlab.com/Earthnuker/ed_lrr/commit/e9bcd3e) -## [v0.13.0] - 2019-09-28 +## [v0.7.1] - 2019-09-28 ### Features - **config:** finish integrating new configuration system [a8d6a32](https://gitlab.com/Earthnuker/ed_lrr/commit/a8d6a32) -## [v0.12.0] - 2019-09-28 +## [v0.7.0] - 2019-09-28 ### Features - **Router:** finish implementing route computation progress display [93f655c](https://gitlab.com/Earthnuker/ed_lrr/commit/93f655c) @@ -88,12 +88,12 @@ - **CI:** rename main executable before building installer [dc0e3fc](https://gitlab.com/Earthnuker/ed_lrr/commit/dc0e3fc) -## [v0.11.0] - 2019-09-28 +## [v0.6.2] - 2019-09-28 ### Features - **GUI:** Integrate route computation Job [f4e6bd3](https://gitlab.com/Earthnuker/ed_lrr/commit/f4e6bd3) -## [v0.10.0] - 2019-09-28 +## [v0.6.1] - 2019-09-28 ### Features - **GUI:** Integrate new config system [22ca2d2](https://gitlab.com/Earthnuker/ed_lrr/commit/22ca2d2) @@ -101,12 +101,12 @@ - re-export config from main GUI module [0d89106](https://gitlab.com/Earthnuker/ed_lrr/commit/0d89106) -## [v0.9.0] - 2019-09-28 +## [v0.6.0] - 2019-09-28 ### Features - **cli:** Add config editor [8b0b56f](https://gitlab.com/Earthnuker/ed_lrr/commit/8b0b56f) -## [v0.8.0] - 2019-09-28 +## [v0.5.1] - 2019-09-28 ### Features - **router:** Implement route computation job with progress dualog [2c000da](https://gitlab.com/Earthnuker/ed_lrr/commit/2c000da) @@ -114,7 +114,7 @@ - **setup.py:** Pull version info from git, unify scripts (one script for GUI and CLI) [a630851](https://gitlab.com/Earthnuker/ed_lrr/commit/a630851) -## [v0.7.0] - 2019-09-28 +## [v0.5.0] - 2019-09-28 ### Documentation - Update README.md [66267e7](https://gitlab.com/Earthnuker/ed_lrr/commit/66267e7) @@ -125,67 +125,67 @@ - Update icon generator and regenerate icon [7838480](https://gitlab.com/Earthnuker/ed_lrr/commit/7838480) -## [v0.6.0] - 2019-09-28 +## [v0.4.1] - 2019-09-28 ### Features - **router:** Start implementing route pruning support [88a0378](https://gitlab.com/Earthnuker/ed_lrr/commit/88a0378) -## [v0.5.0] - 2019-09-28 +## [v0.4.0] - 2019-09-28 ### Features - **config:** Update config system to use profig [b1143c3](https://gitlab.com/Earthnuker/ed_lrr/commit/b1143c3) -## [v0.4.0] - 2019-09-28 +## [v0.3.0] - 2019-09-28 ### Features - **build system:** Remove build.py (switched to tox), add output to build_gui.py [fb3a130](https://gitlab.com/Earthnuker/ed_lrr/commit/fb3a130) -## [v0.3.0] - 2019-09-28 +## [v0.2.0] - 2019-09-28 ### Features - **router:** Add dialog to display computed route [d498746](https://gitlab.com/Earthnuker/ed_lrr/commit/d498746) -## [v0.2.11] - 2019-09-21 +## [v0.1.12] - 2019-09-21 ### Bug Fixes - switch inno setup version in appveyor [fe3534e](https://gitlab.com/Earthnuker/ed_lrr/commit/fe3534e) -## [v0.2.10] - 2019-09-21 +## [v0.1.11] - 2019-09-21 ### Bug Fixes - typo in appveyor.yml [09e6f0a](https://gitlab.com/Earthnuker/ed_lrr/commit/09e6f0a) -## [v0.2.9] - 2019-09-21 +## [v0.1.10] - 2019-09-21 ### Bug Fixes - disable confirmation for conda install [eaddb18](https://gitlab.com/Earthnuker/ed_lrr/commit/eaddb18) -## [v0.2.8] - 2019-09-21 +## [v0.1.9] - 2019-09-21 ### Bug Fixes - add missing conda channel [6d9f1ef](https://gitlab.com/Earthnuker/ed_lrr/commit/6d9f1ef) -## [v0.2.7] - 2019-09-21 +## [v0.1.8] - 2019-09-21 ### Bug Fixes - add missing dependencies to appveyor.yml [e8beb55](https://gitlab.com/Earthnuker/ed_lrr/commit/e8beb55) -## [v0.2.6] - 2019-09-20 +## [v0.1.7] - 2019-09-20 ### Bug Fixes - fix nuikta call to clean build directory [fa485be](https://gitlab.com/Earthnuker/ed_lrr/commit/fa485be) -## [v0.2.5] - 2019-09-20 +## [v0.1.6] - 2019-09-20 ### Bug Fixes - fix nuikta call to download without confirmation [11efe30](https://gitlab.com/Earthnuker/ed_lrr/commit/11efe30) -## [v0.2.4] - 2019-09-20 +## [v0.1.5] - 2019-09-20 ### Bug Fixes - fixed appveyor.yml [59390fe](https://gitlab.com/Earthnuker/ed_lrr/commit/59390fe) -## [v0.2.3] - 2019-09-20 +## [v0.1.4] - 2019-09-20 ### Bug Fixes - fixed tox.ini [6bb7e1e](https://gitlab.com/Earthnuker/ed_lrr/commit/6bb7e1e) @@ -193,7 +193,7 @@ - small wording change [3779911](https://gitlab.com/Earthnuker/ed_lrr/commit/3779911) -## [v0.2.2] - 2019-08-31 +## [v0.1.3] - 2019-08-31 ### Bug Fixes - **setup:** fix setup script to include subdirectories [4a392d9](https://gitlab.com/Earthnuker/ed_lrr/commit/4a392d9) @@ -208,12 +208,12 @@ - **formatting:** ran `cargo fmt` and `cargo clippy`, fixed all warnings [fb3f79b](https://gitlab.com/Earthnuker/ed_lrr/commit/fb3f79b) -## [v0.2.1] - 2019-08-05 +## [v0.1.2] - 2019-08-05 ### Bug Fixes - **router:** Fixed some syntax errors created by botched merge [4b14643](https://gitlab.com/Earthnuker/ed_lrr/commit/4b14643) -## [v0.2.0] - 2019-08-05 +## [v0.1.1] - 2019-08-05 ### Features - **GUI:** Implement route plotting and fuzzy search [c290d5e](https://gitlab.com/Earthnuker/ed_lrr/commit/c290d5e) @@ -225,40 +225,40 @@ ## v0.0.0 - 2019-07-15 -[Unreleased]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.24.0...HEAD -[v0.24.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.23.0...v0.24.0 -[v0.23.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.22.0...v0.23.0 -[v0.22.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.21.0...v0.22.0 -[v0.21.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.20.0...v0.21.0 -[v0.20.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.19.1...v0.20.0 -[v0.19.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.19.0...v0.19.1 -[v0.19.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.18.0...v0.19.0 -[v0.18.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.17.0...v0.18.0 -[v0.17.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.16.0...v0.17.0 -[v0.16.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.15.0...v0.16.0 -[v0.15.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.14.0...v0.15.0 +[Unreleased]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.14.0...HEAD [v0.14.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.13.0...v0.14.0 [v0.13.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.12.0...v0.13.0 [v0.12.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.11.0...v0.12.0 [v0.11.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.10.0...v0.11.0 -[v0.10.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.9.0...v0.10.0 -[v0.9.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.8.0...v0.9.0 -[v0.8.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.7.0...v0.8.0 -[v0.7.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.6.0...v0.7.0 -[v0.6.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.5.0...v0.6.0 -[v0.5.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.4.0...v0.5.0 +[v0.10.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.9.4...v0.10.0 +[v0.9.4]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.9.3...v0.9.4 +[v0.9.3]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.9.2...v0.9.3 +[v0.9.2]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.9.1...v0.9.2 +[v0.9.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.9.0...v0.9.1 +[v0.9.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.8.1...v0.9.0 +[v0.8.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.8.0...v0.8.1 +[v0.8.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.7.1...v0.8.0 +[v0.7.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.7.0...v0.7.1 +[v0.7.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.6.2...v0.7.0 +[v0.6.2]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.6.1...v0.6.2 +[v0.6.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.6.0...v0.6.1 +[v0.6.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.5.1...v0.6.0 +[v0.5.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.5.0...v0.5.1 +[v0.5.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.4.1...v0.5.0 +[v0.4.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.4.0...v0.4.1 [v0.4.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.3.0...v0.4.0 -[v0.3.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.11...v0.3.0 -[v0.2.11]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.10...v0.2.11 -[v0.2.10]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.9...v0.2.10 -[v0.2.9]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.8...v0.2.9 -[v0.2.8]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.7...v0.2.8 -[v0.2.7]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.6...v0.2.7 -[v0.2.6]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.5...v0.2.6 -[v0.2.5]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.4...v0.2.5 -[v0.2.4]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.3...v0.2.4 -[v0.2.3]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.2...v0.2.3 -[v0.2.2]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.1...v0.2.2 -[v0.2.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.0...v0.2.1 -[v0.2.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.0...v0.2.0 +[v0.3.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.2.0...v0.3.0 +[v0.2.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.12...v0.2.0 +[v0.1.12]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.11...v0.1.12 +[v0.1.11]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.10...v0.1.11 +[v0.1.10]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.9...v0.1.10 +[v0.1.9]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.8...v0.1.9 +[v0.1.8]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.7...v0.1.8 +[v0.1.7]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.6...v0.1.7 +[v0.1.6]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.5...v0.1.6 +[v0.1.5]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.4...v0.1.5 +[v0.1.4]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.3...v0.1.4 +[v0.1.3]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.2...v0.1.3 +[v0.1.2]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.1...v0.1.2 +[v0.1.1]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.1.0...v0.1.1 [v0.1.0]: https://gitlab.com/Earthnuker/ed_lrr/compare/v0.0.0...v0.1.0 diff --git a/MANIFEST.in b/MANIFEST.in index a159d6a..c2cd5ac 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include rust/Cargo.toml include rust/.cargo/config recursive-include rust/src * -recursive-include ed_lrr_gui * \ No newline at end of file +recursive-include ed_lrr_gui * diff --git a/README.md b/README.md index db04633..bfd9d1e 100644 --- a/README.md +++ b/README.md @@ -80,4 +80,3 @@ then you can run `ed_lrr -h` from your command prompt to get help - Luigi based Task queue for distributed routing - Full route tree computation - overlap elimination - diff --git a/appveyor.yml b/appveyor.yml index 492b378..f10d1aa 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,26 +1,20 @@ image: Visual Studio 2019 platform: x64 -version: 0.1.{build} branches: only: - pyqt_gui + - WIP environment: VCVARS: C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat VCVARSARG: x64 MINICONDA: C:\Miniconda3-x64 INNO: C:\Program Files (x86)\Inno Setup 6 - -build: off - -artifacts: - - path: exe\__main__.dist\ - name: ED_LRR - type: zip - - - path: installer\Output\*.exe - name: Setup - type: file + matrix: + - PY_VERSION: "3.5" + - PY_VERSION: "3.6" + - PY_VERSION: "3.7" + - PY_VERSION: "3.8" install: - set PATH=%MINICONDA%\\Library\\bin;%MINICONDA%\\Scripts;%USERPROFILE%\\.cargo\\bin;%PATH% @@ -30,14 +24,12 @@ install: - if defined VCVARS call "%VCVARS%" %VCVARSARG% - conda activate - conda update -y -n base -c defaults conda - - conda install -y -c conda-forge pycrypto nuitka - - pip install PyQt5 setuptools_rust + - pip install nox test_script: - - python build_gui.py - - pip install .[dev] - - python -m nuitka --plugin-enable=multiprocessing --remove-output --plugin-enable=qt-plugins --standalone --assume-yes-for-downloads --follow-imports --output-dir=exe ed_lrr_gui\__main__.py - - rename exe\__main__.dist\__main__.exe ED_LRR.exe - - cd installer - - iscc /QP ED_LRR.iss + - ps: nox -s test-$env:PY_VERSION +build_script: + - ps: nox -s build-$env:PY_VERSION + - ps: Get-ChildItem .\installer\Output\*.exe | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } + - ps: Get-ChildItem .\dist\* | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name.Insert($_.Name.Length,".zip") } \ No newline at end of file diff --git a/benchmark.py b/benchmark.py deleted file mode 100644 index 6d26792..0000000 --- a/benchmark.py +++ /dev/null @@ -1,39 +0,0 @@ -from datetime import datetime -import time -import json -import statistics as stats -from pprint import pprint -import os -import _ed_lrr - -NUM_LOOPS=5 - -results={} - -def time_run(w,s,file="benchmark.json", loops=None): - global results,NUM_LOOPS - if loops is None: - loops=NUM_LOOPS - for _ in range(loops): - t_start = time.time() - ret = _ed_lrr.route(s,48,None,'bfs',True,False,False,False,0.0,None,r"D:\devel\rust\ED_LRR\stars.csv",w,lambda *args,**kwargs: None) - t_end = time.time() - results.setdefault(w,[]).append({'ret':len(ret),'time':t_end - t_start}) - with open(file, "w") as of: - json.dump(results, of,indent=2) - -t_start = datetime.today() - -for w in [1,2,4,7,8,0]: - time_run(w,['Ix','72']) - -print("Benchmark took:", datetime.today() - t_start) - - -for workers,results in results.items(): - t_total=sum([res['time'] for res in results])/len(results) - avg_len=sum([res['ret'] for res in results])/len(results) - times.append([int(workers),timedelta(seconds=t_total),avg_len]) - -for k,v,l in sorted(times,key=lambda rec:rec[1]): - print(k,v,l) \ No newline at end of file diff --git a/benchmark_out.py b/benchmark_out.py deleted file mode 100644 index b089e73..0000000 --- a/benchmark_out.py +++ /dev/null @@ -1,12 +0,0 @@ -import ujson as json -from datetime import timedelta -data=json.load(open("benchmark.json")) - -times=[] - -for workers,results in data.items(): - t_total=sum([res['time'] for res in results])/len(results) - avg_len=sum([res['ret'] for res in results])/len(results) - times.append([int(workers),timedelta(seconds=t_total),avg_len]) -for k,v,l in sorted(times,key=lambda rec:rec[1]): - print(k,v,l) \ No newline at end of file diff --git a/build_gui.py b/build_gui.py index 831c09d..3bb0668 100644 --- a/build_gui.py +++ b/build_gui.py @@ -1,9 +1,11 @@ +# -*- coding: utf-8 -*- from PyQt5 import uic import os - ui_path = os.path.dirname(os.path.abspath(__file__)) for root, folders, files in os.walk(ui_path): + if "site-packages" in root: + continue if ".history" in folders: folders.remove(".history") for file in files: diff --git a/celery/celery_rabbitmq_setup.ps1 b/celery/celery_rabbitmq_setup.ps1 new file mode 100644 index 0000000..2ca184d --- /dev/null +++ b/celery/celery_rabbitmq_setup.ps1 @@ -0,0 +1,16 @@ +#RABBITMQ +rabbitmqctl stop_app +rabbitmqctl reset +rabbitmqctl start_app +rabbitmqctl add_user ed_lrr ed_lrr +rabbitmqctl add_vhost ed_lrr +rabbitmqctl set_user_tags ed_lrr ed_lrr +rabbitmqctl set_permissions -p ed_lrr ed_lrr ".*" ".*" ".*" +rabbitmqctl set_permissions guest ".*" ".*" ".*" +rabbitmqctl set_permissions -p ed_lrr guest ".*" ".*" ".*" +Write-Host RabbitMQ setup done +#Celery +Write-Host starting Celery +celery worker -l info + +#celery -A celery_test flower --presistent --broker=pyamqp://ed_lrr:ed_lrr@localhost/ed_lrr --broker_api=http://ed_lrr:ed_lrr@localhost:15672/api/ diff --git a/celery/celery_test.py b/celery/celery_test.py new file mode 100644 index 0000000..2bf9857 --- /dev/null +++ b/celery/celery_test.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +from celery import Celery +import _ed_lrr +import os + +os.environ.setdefault("CELERY_CONFIG_MODULE", "celeryconfig") +app = Celery("ed_lrr") +app.config_from_envvar("CELERY_CONFIG_MODULE") + + +@app.task(bind=True) +def route(self, hops, jmp_range): + def callback(state): + print("PRC: ", state.get("prc_done", 0.0)) + self.update_state(state="PROGRESS", meta=state) + + self.update_state(state="RUNNING", meta={}) + return _ed_lrr.route( + hops, + jmp_range, + None, + "bfs", + True, + False, + False, + False, + 0.0, + None, + r"C:\Users\Earthnuker\AppData\Local\ED_LRR\data\stars.csv", + 0, + callback, + ) diff --git a/celery/celery_worker.py b/celery/celery_worker.py new file mode 100644 index 0000000..92db4a6 --- /dev/null +++ b/celery/celery_worker.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +import time +from celery_test import route +import sys + +if len(sys.argv) > 1: + job = route.AsyncResult(sys.argv[1]) + if job.ready(): + print([job, job.state, len(job.info), len(job.result)]) + else: + print([job, job.state, job.info, job.result]) + exit(0) +jobs = [route.delay(["Ix", "Colonia"], 48)] +print(jobs) diff --git a/celery/celeryconfig.py b/celery/celeryconfig.py new file mode 100644 index 0000000..4f19bf6 --- /dev/null +++ b/celery/celeryconfig.py @@ -0,0 +1,18 @@ +import os + +os.environ["FORKED_BY_MULTIPROCESSING"] = "1" +broker_url = "pyamqp://ed_lrr:ed_lrr@localhost/ed_lrr" +broker_api = "http://guest:guest@localhost:15672/api/" +imports = ("celery_test",) +result_backend = "file://celery_results/" +result_persistent = True +task_track_started = True +task_time_limit = 60 * 60 +result_extended = True +result_expires = None +worker_direct = True +worker_max_tasks_per_child = 10 +worker_max_memory_per_child = 4 * 1024 * 1024 # 4GB +worker_state_db = "ed_lrr.state" +worker_send_task_events = True +worker_log_color = True diff --git a/celery_test.py b/celery_test.py deleted file mode 100644 index b831ada..0000000 --- a/celery_test.py +++ /dev/null @@ -1,9 +0,0 @@ -from celery import Celery -import _ed_lrr -app = Celery('ed_lrr',backend = 'db+sqlite:///ed_lrr_results.sqlite', broker='pyamqp://ed_lrr:ed_lrr@localhost/ed_lrr') - -@app.task(bind=True) -def route(self,hops,jmp_range): - def callback(state): - self.update_state(state="PROGRESS", meta=state) - return _ed_lrr.route(hops,jmp_range,None,'bfs',True,False,False,False,0.0,None,r"C:\Users\Earthnuker\AppData\Local\ED_LRR\data\stars.csv",0,callback) \ No newline at end of file diff --git a/celery_worker.py b/celery_worker.py deleted file mode 100644 index c1ad95c..0000000 --- a/celery_worker.py +++ /dev/null @@ -1,18 +0,0 @@ -import time -from celery_test import route - -jobs = [route.delay(["Ix", "Colonia"], 48), route.delay(["Colonia", "Sol"], 48)] -while True: - for job in jobs: - if job.ready(): - print([job, job.state, len(job.info)]) - else: - print([job, job.state, job.info]) - print("="*10) - time.sleep(1) - -# 02c77491-9abd-4a88-ab2c-acdf2981086b -# d56b0ca8-067d-45a6-be9b-bb9e74f196cd - -# celery -A celery_test flower --presistent --broker=pyamqp://ed_lrr:ed_lrr@localhost/ed_lrr --broker_api=http://ed_lrr:ed_lrr@localhost:15672/api/ - diff --git a/docs/filters/multifilter.py b/docs/filters/multifilter.py index 3ae005c..3807c16 100644 --- a/docs/filters/multifilter.py +++ b/docs/filters/multifilter.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import contextlib import csv import datetime diff --git a/docs/src/img_out.py b/docs/src/img_out.py index deb32bc..6a76fbd 100644 --- a/docs/src/img_out.py +++ b/docs/src/img_out.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import heapq import sys diff --git a/ed_lrr_gui/__init__.py b/ed_lrr_gui/__init__.py index 7946e22..b9c32b5 100644 --- a/ed_lrr_gui/__init__.py +++ b/ed_lrr_gui/__init__.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from _ed_lrr import * from .preprocess import Preprocessor diff --git a/ed_lrr_gui/__main__.py b/ed_lrr_gui/__main__.py index 67b6740..b9b5aed 100644 --- a/ed_lrr_gui/__main__.py +++ b/ed_lrr_gui/__main__.py @@ -1,17 +1,19 @@ +# -*- coding: utf-8 -*- import sys import multiprocessing as MP import queue import ctypes import os from datetime import datetime -from math import floor import click from tqdm import tqdm -from click_default_group import DefaultGroup import requests as RQ from urllib.parse import urljoin from ed_lrr_gui import Router, Preprocessor, cfg -from _ed_lrr import find_sys +from _ed_lrr import PyRouter + +from dotenv import load_dotenv +load_dotenv() CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) @@ -43,9 +45,8 @@ def main(ctx): MP.freeze_support() if ctx.invoked_subcommand != "config": os.makedirs(cfg["folders.data_dir"], exist_ok=True) - if ctx.invoked_subcommand is None: - ctx.invoke(gui) - return + if ctx.invoked_subcommand==None: + click.forward(gui_main) return @@ -55,18 +56,20 @@ def main(ctx): @click.option("--debug", "-d", is_flag=True, help="Run using debug server") def web(host, port, debug): "Run web interface." + from gevent import monkey + monkey.patch_all() from gevent.pywsgi import WSGIServer from ed_lrr_gui.web import app with app.test_client() as c: c.get("/") # Force before_first_request hook to run if debug: - app.debug=True + app.debug = True app.run(host=host, port=port, debug=True) - return - print("Listening on {}:{}".format(host, port)) - server = WSGIServer((host, port), app) - server.serve_forever() + else: + print("Listening on {}:{}".format(host, port)) + server = WSGIServer((host, port), app) + server.serve_forever() @main.command() @@ -175,7 +178,7 @@ def download(url, folder): unit_divisor=1024, unit_scale=True, ascii=True, - smoothing=0 + smoothing=0, ) as pbar: with open(download_path, "wb") as of: resp = RQ.get( @@ -224,7 +227,7 @@ def preprocess(systems, bodies, output): preproc.start() state = {} pstate = {} - while not (preproc.queue.empty() and preproc.is_alive() == False): + while not (preproc.queue.empty() and not preproc.is_alive()): try: event = preproc.queue.get(True, 0.1) state.update(event) @@ -309,7 +312,7 @@ def preprocess(systems, bodies, output): "-m", default=cfg["route.mode"], help="Search mode", - type=click.Choice(["bfs","bfs_old", "a-star", "greedy"]), + type=click.Choice(["bfs", "bfs_old", "a-star", "greedy"]), show_default=True, ) @click.option( @@ -358,7 +361,7 @@ def route(**kwargs): kwargs["factor"], None, kwargs["path"], - kwargs["workers"] + kwargs["workers"], ] with click.progressbar( length=100, @@ -429,5 +432,9 @@ def precompute(*args, **kwargs): raise NotImplementedError +def gui_main(): + return gui(False) + + if __name__ == "__main__": main() diff --git a/ed_lrr_gui/config.py b/ed_lrr_gui/config.py index f54dfea..deba23b 100644 --- a/ed_lrr_gui/config.py +++ b/ed_lrr_gui/config.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import pathlib from collections import namedtuple import profig @@ -9,6 +10,8 @@ config_dir.mkdir(parents=True, exist_ok=True) config_file = config_dir / "config.ini" config_file.touch() +config_dir = str(config_dir) + cfg = profig.Config(str(config_file), strict=True) cfg.init( @@ -55,8 +58,8 @@ cfg.init("folders.data_dir", os.path.join(config_dir, "data"), comment="Data dir cfg.init("GUI.theme", "dark", comment="GUI theme to use") -cfg.init("web.port",3777,comment="Port to bind to") -cfg.init("web.host","0.0.0.0",comment="Address to bind to") -cfg.init("web.debug",False,comment="Run using debug server") +cfg.init("web.port", 3777, comment="Port to bind to") +cfg.init("web.host", "0.0.0.0", comment="Address to bind to") +cfg.init("web.debug", False, comment="Run using debug server") cfg.sync() diff --git a/ed_lrr_gui/gui/__init__.py b/ed_lrr_gui/gui/__init__.py index c28a133..3265194 100644 --- a/ed_lrr_gui/gui/__init__.py +++ b/ed_lrr_gui/gui/__init__.py @@ -1 +1,2 @@ +# -*- coding: utf-8 -*- from .main import main diff --git a/ed_lrr_gui/gui/ed_lrr.py b/ed_lrr_gui/gui/ed_lrr.py index 81b99b5..8f20378 100644 --- a/ed_lrr_gui/gui/ed_lrr.py +++ b/ed_lrr_gui/gui/ed_lrr.py @@ -15,7 +15,9 @@ class Ui_ED_LRR(object): ED_LRR.setObjectName("ED_LRR") ED_LRR.setEnabled(True) ED_LRR.resize(577, 500) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth(ED_LRR.sizePolicy().hasHeightForWidth()) @@ -26,10 +28,14 @@ class Ui_ED_LRR(object): ED_LRR.setDocumentMode(False) ED_LRR.setTabShape(QtWidgets.QTabWidget.Rounded) self.centralwidget = QtWidgets.QWidget(ED_LRR) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.centralwidget.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.centralwidget.sizePolicy().hasHeightForWidth() + ) self.centralwidget.setSizePolicy(sizePolicy) self.centralwidget.setObjectName("centralwidget") self.verticalLayout = QtWidgets.QVBoxLayout(self.centralwidget) @@ -49,27 +55,39 @@ class Ui_ED_LRR(object): self.formLayout.setObjectName("formLayout") self.lbl_bodies_dl = QtWidgets.QLabel(self.tab_download) self.lbl_bodies_dl.setObjectName("lbl_bodies_dl") - self.formLayout.setWidget(1, QtWidgets.QFormLayout.LabelRole, self.lbl_bodies_dl) + self.formLayout.setWidget( + 1, QtWidgets.QFormLayout.LabelRole, self.lbl_bodies_dl + ) self.lbl_systems_dl = QtWidgets.QLabel(self.tab_download) self.lbl_systems_dl.setObjectName("lbl_systems_dl") - self.formLayout.setWidget(3, QtWidgets.QFormLayout.LabelRole, self.lbl_systems_dl) + self.formLayout.setWidget( + 3, QtWidgets.QFormLayout.LabelRole, self.lbl_systems_dl + ) self.inp_bodies_dl = QtWidgets.QComboBox(self.tab_download) self.inp_bodies_dl.setEditable(True) self.inp_bodies_dl.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) self.inp_bodies_dl.setObjectName("inp_bodies_dl") - self.formLayout.setWidget(1, QtWidgets.QFormLayout.FieldRole, self.inp_bodies_dl) + self.formLayout.setWidget( + 1, QtWidgets.QFormLayout.FieldRole, self.inp_bodies_dl + ) self.inp_systems_dl = QtWidgets.QComboBox(self.tab_download) self.inp_systems_dl.setEditable(True) self.inp_systems_dl.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) self.inp_systems_dl.setObjectName("inp_systems_dl") - self.formLayout.setWidget(3, QtWidgets.QFormLayout.FieldRole, self.inp_systems_dl) + self.formLayout.setWidget( + 3, QtWidgets.QFormLayout.FieldRole, self.inp_systems_dl + ) self.gridLayout = QtWidgets.QGridLayout() self.gridLayout.setObjectName("gridLayout") self.inp_bodies_dest_dl = QtWidgets.QComboBox(self.tab_download) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.inp_bodies_dest_dl.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.inp_bodies_dest_dl.sizePolicy().hasHeightForWidth() + ) self.inp_bodies_dest_dl.setSizePolicy(sizePolicy) self.inp_bodies_dest_dl.setEditable(False) self.inp_bodies_dest_dl.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) @@ -85,10 +103,14 @@ class Ui_ED_LRR(object): self.btn_systems_dest_browse_dl.setObjectName("btn_systems_dest_browse_dl") self.gridLayout_2.addWidget(self.btn_systems_dest_browse_dl, 0, 1, 1, 1) self.inp_systems_dest_dl = QtWidgets.QComboBox(self.tab_download) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.inp_systems_dest_dl.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.inp_systems_dest_dl.sizePolicy().hasHeightForWidth() + ) self.inp_systems_dest_dl.setSizePolicy(sizePolicy) self.inp_systems_dest_dl.setEditable(False) self.inp_systems_dest_dl.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) @@ -111,57 +133,79 @@ class Ui_ED_LRR(object): self.formLayout_3.setObjectName("formLayout_3") self.lbl_bodies_pp = QtWidgets.QLabel(self.tab_preprocess) self.lbl_bodies_pp.setObjectName("lbl_bodies_pp") - self.formLayout_3.setWidget(0, QtWidgets.QFormLayout.LabelRole, self.lbl_bodies_pp) + self.formLayout_3.setWidget( + 0, QtWidgets.QFormLayout.LabelRole, self.lbl_bodies_pp + ) self.gr_bodies_pp = QtWidgets.QGridLayout() self.gr_bodies_pp.setObjectName("gr_bodies_pp") self.btn_bodies_browse_pp = QtWidgets.QPushButton(self.tab_preprocess) self.btn_bodies_browse_pp.setObjectName("btn_bodies_browse_pp") self.gr_bodies_pp.addWidget(self.btn_bodies_browse_pp, 0, 1, 1, 1) self.inp_bodies_pp = QtWidgets.QComboBox(self.tab_preprocess) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.inp_bodies_pp.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.inp_bodies_pp.sizePolicy().hasHeightForWidth() + ) self.inp_bodies_pp.setSizePolicy(sizePolicy) self.inp_bodies_pp.setEditable(False) self.inp_bodies_pp.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) self.inp_bodies_pp.setObjectName("inp_bodies_pp") self.gr_bodies_pp.addWidget(self.inp_bodies_pp, 0, 0, 1, 1) - self.formLayout_3.setLayout(0, QtWidgets.QFormLayout.FieldRole, self.gr_bodies_pp) + self.formLayout_3.setLayout( + 0, QtWidgets.QFormLayout.FieldRole, self.gr_bodies_pp + ) self.lbl_systems_pp = QtWidgets.QLabel(self.tab_preprocess) self.lbl_systems_pp.setObjectName("lbl_systems_pp") - self.formLayout_3.setWidget(1, QtWidgets.QFormLayout.LabelRole, self.lbl_systems_pp) + self.formLayout_3.setWidget( + 1, QtWidgets.QFormLayout.LabelRole, self.lbl_systems_pp + ) self.gr_systems_pp = QtWidgets.QGridLayout() self.gr_systems_pp.setObjectName("gr_systems_pp") self.btn_systems_browse_pp = QtWidgets.QPushButton(self.tab_preprocess) self.btn_systems_browse_pp.setObjectName("btn_systems_browse_pp") self.gr_systems_pp.addWidget(self.btn_systems_browse_pp, 0, 1, 1, 1) self.inp_systems_pp = QtWidgets.QComboBox(self.tab_preprocess) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.inp_systems_pp.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.inp_systems_pp.sizePolicy().hasHeightForWidth() + ) self.inp_systems_pp.setSizePolicy(sizePolicy) self.inp_systems_pp.setEditable(False) self.inp_systems_pp.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) self.inp_systems_pp.setObjectName("inp_systems_pp") self.gr_systems_pp.addWidget(self.inp_systems_pp, 0, 0, 1, 1) - self.formLayout_3.setLayout(1, QtWidgets.QFormLayout.FieldRole, self.gr_systems_pp) + self.formLayout_3.setLayout( + 1, QtWidgets.QFormLayout.FieldRole, self.gr_systems_pp + ) self.lbl_out_pp = QtWidgets.QLabel(self.tab_preprocess) self.lbl_out_pp.setObjectName("lbl_out_pp") self.formLayout_3.setWidget(2, QtWidgets.QFormLayout.LabelRole, self.lbl_out_pp) self.gr_out_grid_pp = QtWidgets.QGridLayout() self.gr_out_grid_pp.setObjectName("gr_out_grid_pp") self.btn_out_browse_pp = QtWidgets.QPushButton(self.tab_preprocess) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.btn_out_browse_pp.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.btn_out_browse_pp.sizePolicy().hasHeightForWidth() + ) self.btn_out_browse_pp.setSizePolicy(sizePolicy) self.btn_out_browse_pp.setObjectName("btn_out_browse_pp") self.gr_out_grid_pp.addWidget(self.btn_out_browse_pp, 0, 1, 1, 1) self.inp_out_pp = QtWidgets.QComboBox(self.tab_preprocess) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth(self.inp_out_pp.sizePolicy().hasHeightForWidth()) @@ -170,10 +214,14 @@ class Ui_ED_LRR(object): self.inp_out_pp.setInsertPolicy(QtWidgets.QComboBox.InsertAtTop) self.inp_out_pp.setObjectName("inp_out_pp") self.gr_out_grid_pp.addWidget(self.inp_out_pp, 0, 0, 1, 1) - self.formLayout_3.setLayout(2, QtWidgets.QFormLayout.FieldRole, self.gr_out_grid_pp) + self.formLayout_3.setLayout( + 2, QtWidgets.QFormLayout.FieldRole, self.gr_out_grid_pp + ) self.btn_preprocess = QtWidgets.QPushButton(self.tab_preprocess) self.btn_preprocess.setObjectName("btn_preprocess") - self.formLayout_3.setWidget(3, QtWidgets.QFormLayout.LabelRole, self.btn_preprocess) + self.formLayout_3.setWidget( + 3, QtWidgets.QFormLayout.LabelRole, self.btn_preprocess + ) self.tabs.addTab(self.tab_preprocess, "") self.tab_route = QtWidgets.QWidget() self.tab_route.setObjectName("tab_route") @@ -181,14 +229,18 @@ class Ui_ED_LRR(object): self.formLayout_2.setObjectName("formLayout_2") self.lbl_sys_lst = QtWidgets.QLabel(self.tab_route) self.lbl_sys_lst.setObjectName("lbl_sys_lst") - self.formLayout_2.setWidget(0, QtWidgets.QFormLayout.LabelRole, self.lbl_sys_lst) + self.formLayout_2.setWidget( + 0, QtWidgets.QFormLayout.LabelRole, self.lbl_sys_lst + ) self.gr_sys = QtWidgets.QGridLayout() self.gr_sys.setObjectName("gr_sys") self.btn_sys_lst_browse = QtWidgets.QPushButton(self.tab_route) self.btn_sys_lst_browse.setObjectName("btn_sys_lst_browse") self.gr_sys.addWidget(self.btn_sys_lst_browse, 0, 1, 1, 1) self.inp_sys_lst = QtWidgets.QComboBox(self.tab_route) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth(self.inp_sys_lst.sizePolicy().hasHeightForWidth()) @@ -221,32 +273,44 @@ class Ui_ED_LRR(object): self.formLayout_2.setLayout(3, QtWidgets.QFormLayout.FieldRole, self.gr_mode) self.chk_permute = QtWidgets.QCheckBox(self.tab_route) self.chk_permute.setObjectName("chk_permute") - self.formLayout_2.setWidget(4, QtWidgets.QFormLayout.LabelRole, self.chk_permute) + self.formLayout_2.setWidget( + 4, QtWidgets.QFormLayout.LabelRole, self.chk_permute + ) self.gridLayout_4 = QtWidgets.QGridLayout() self.gridLayout_4.setObjectName("gridLayout_4") self.chk_permute_keep_last = QtWidgets.QCheckBox(self.tab_route) self.chk_permute_keep_last.setObjectName("chk_permute_keep_last") self.gridLayout_4.addWidget(self.chk_permute_keep_last, 0, 3, 1, 1) self.chk_permute_keep_first = QtWidgets.QCheckBox(self.tab_route) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Fixed + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.chk_permute_keep_first.sizePolicy().hasHeightForWidth()) + sizePolicy.setHeightForWidth( + self.chk_permute_keep_first.sizePolicy().hasHeightForWidth() + ) self.chk_permute_keep_first.setSizePolicy(sizePolicy) self.chk_permute_keep_first.setTristate(False) self.chk_permute_keep_first.setObjectName("chk_permute_keep_first") self.gridLayout_4.addWidget(self.chk_permute_keep_first, 0, 2, 1, 1) self.lbl_keep = QtWidgets.QLabel(self.tab_route) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth(self.lbl_keep.sizePolicy().hasHeightForWidth()) self.lbl_keep.setSizePolicy(sizePolicy) self.lbl_keep.setObjectName("lbl_keep") self.gridLayout_4.addWidget(self.lbl_keep, 0, 1, 1, 1) - self.formLayout_2.setLayout(4, QtWidgets.QFormLayout.FieldRole, self.gridLayout_4) + self.formLayout_2.setLayout( + 4, QtWidgets.QFormLayout.FieldRole, self.gridLayout_4 + ) self.lst_sys = QtWidgets.QTreeWidget(self.tab_route) - sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) + sizePolicy = QtWidgets.QSizePolicy( + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding + ) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth(self.lst_sys.sizePolicy().hasHeightForWidth()) @@ -353,17 +417,27 @@ class Ui_ED_LRR(object): def retranslateUi(self, ED_LRR): _translate = QtCore.QCoreApplication.translate - ED_LRR.setWindowTitle(_translate("ED_LRR", "Elite: Dangerous Long Range Route Plotter")) + ED_LRR.setWindowTitle( + _translate("ED_LRR", "Elite: Dangerous Long Range Route Plotter") + ) self.lbl_bodies_dl.setText(_translate("ED_LRR", "bodies.json")) self.lbl_systems_dl.setText(_translate("ED_LRR", "systemsWithCoordinates.json")) - self.inp_bodies_dl.setCurrentText(_translate("ED_LRR", "https://www.edsm.net/dump/bodies.json")) - self.inp_systems_dl.setCurrentText(_translate("ED_LRR", "https://www.edsm.net/dump/systemsWithCoordinates.json")) + self.inp_bodies_dl.setCurrentText( + _translate("ED_LRR", "https://www.edsm.net/dump/bodies.json") + ) + self.inp_systems_dl.setCurrentText( + _translate( + "ED_LRR", "https://www.edsm.net/dump/systemsWithCoordinates.json" + ) + ) self.btn_bodies_dest_browse_dl.setText(_translate("ED_LRR", "...")) self.btn_systems_dest_browse_dl.setText(_translate("ED_LRR", "...")) self.btn_download.setText(_translate("ED_LRR", "Download")) self.label.setText(_translate("ED_LRR", "Download path")) self.label_2.setText(_translate("ED_LRR", "Download path")) - self.tabs.setTabText(self.tabs.indexOf(self.tab_download), _translate("ED_LRR", "Download")) + self.tabs.setTabText( + self.tabs.indexOf(self.tab_download), _translate("ED_LRR", "Download") + ) self.lbl_bodies_pp.setText(_translate("ED_LRR", "bodies.json")) self.btn_bodies_browse_pp.setText(_translate("ED_LRR", "...")) self.lbl_systems_pp.setText(_translate("ED_LRR", "systemsWithCoordinates.json")) @@ -371,7 +445,9 @@ class Ui_ED_LRR(object): self.lbl_out_pp.setText(_translate("ED_LRR", "Output")) self.btn_out_browse_pp.setText(_translate("ED_LRR", "...")) self.btn_preprocess.setText(_translate("ED_LRR", "Preprocess")) - self.tabs.setTabText(self.tabs.indexOf(self.tab_preprocess), _translate("ED_LRR", "Preprocess")) + self.tabs.setTabText( + self.tabs.indexOf(self.tab_preprocess), _translate("ED_LRR", "Preprocess") + ) self.lbl_sys_lst.setText(_translate("ED_LRR", "System List")) self.btn_sys_lst_browse.setText(_translate("ED_LRR", "...")) self.btn_add.setText(_translate("ED_LRR", "Add")) @@ -393,8 +469,12 @@ class Ui_ED_LRR(object): self.chk_primary.setText(_translate("ED_LRR", "Primary Stars Only")) self.lbl_mode.setText(_translate("ED_LRR", "Mode")) self.btn_go.setText(_translate("ED_LRR", "GO!")) - self.tabs.setTabText(self.tabs.indexOf(self.tab_route), _translate("ED_LRR", "Route")) - self.tabs.setTabText(self.tabs.indexOf(self.tab_log), _translate("ED_LRR", "Log")) + self.tabs.setTabText( + self.tabs.indexOf(self.tab_route), _translate("ED_LRR", "Route") + ) + self.tabs.setTabText( + self.tabs.indexOf(self.tab_log), _translate("ED_LRR", "Log") + ) self.menu_file.setTitle(_translate("ED_LRR", "File")) self.menuWindow.setTitle(_translate("ED_LRR", "Window")) self.menuStyle.setTitle(_translate("ED_LRR", "Style")) diff --git a/ed_lrr_gui/gui/main.py b/ed_lrr_gui/gui/main.py index daae56d..926d754 100644 --- a/ed_lrr_gui/gui/main.py +++ b/ed_lrr_gui/gui/main.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import csv import gzip import multiprocessing as MP @@ -474,7 +475,7 @@ class ED_LRR(Ui_ED_LRR): greedyness, precomp, path, - os.cpu_count()-1 + os.cpu_count() - 1, ) if not self.current_job: self.bar_status.showMessage("Computing Route...") @@ -492,7 +493,7 @@ class ED_LRR(Ui_ED_LRR): greedyness, precomp, path, - os.cpu_count()-1 + os.cpu_count() - 1, ) else: self.error("there is already a job running!") diff --git a/ed_lrr_gui/gui/widget_route.py b/ed_lrr_gui/gui/widget_route.py index 200d278..09b7ec2 100644 --- a/ed_lrr_gui/gui/widget_route.py +++ b/ed_lrr_gui/gui/widget_route.py @@ -25,7 +25,9 @@ class Ui_diag_route(object): self.lst_route.setAlternatingRowColors(True) self.lst_route.setSelectionMode(QtWidgets.QAbstractItemView.NoSelection) self.lst_route.setVerticalScrollMode(QtWidgets.QAbstractItemView.ScrollPerItem) - self.lst_route.setHorizontalScrollMode(QtWidgets.QAbstractItemView.ScrollPerPixel) + self.lst_route.setHorizontalScrollMode( + QtWidgets.QAbstractItemView.ScrollPerPixel + ) self.lst_route.setItemsExpandable(False) self.lst_route.setAllColumnsShowFocus(False) self.lst_route.setObjectName("lst_route") @@ -53,7 +55,11 @@ class Ui_diag_route(object): self.lst_route.headerItem().setText(0, _translate("diag_route", "Num")) self.lst_route.headerItem().setText(1, _translate("diag_route", "System")) self.lst_route.headerItem().setText(2, _translate("diag_route", "Body")) - self.lst_route.headerItem().setText(3, _translate("diag_route", "Distance (Ls)")) - self.chk_copy.setText(_translate("diag_route", "Auto-copy next hop to clipboard")) + self.lst_route.headerItem().setText( + 3, _translate("diag_route", "Distance (Ls)") + ) + self.chk_copy.setText( + _translate("diag_route", "Auto-copy next hop to clipboard") + ) self.btn_close.setText(_translate("diag_route", "Close")) self.btn_export.setText(_translate("diag_route", "Export")) diff --git a/ed_lrr_gui/html_export.py b/ed_lrr_gui/html_export.py index 1b29db5..3193d9c 100644 --- a/ed_lrr_gui/html_export.py +++ b/ed_lrr_gui/html_export.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import jinja2 import os @@ -10,6 +11,7 @@ def dist(p1, p2): s += (c1 - c2) ** 2 return s ** 0.5 + colors = { "O": "#0000FF", "B": "#140AF0", diff --git a/ed_lrr_gui/html_export_template.html.jinja2 b/ed_lrr_gui/html_export_template.html.jinja2 index 729eafa..2e2a780 100644 --- a/ed_lrr_gui/html_export_template.html.jinja2 +++ b/ed_lrr_gui/html_export_template.html.jinja2 @@ -155,4 +155,4 @@ - \ No newline at end of file + diff --git a/ed_lrr_gui/preprocess.py b/ed_lrr_gui/preprocess.py index a487833..e74e14f 100644 --- a/ed_lrr_gui/preprocess.py +++ b/ed_lrr_gui/preprocess.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import queue from collections import namedtuple from datetime import datetime, timedelta diff --git a/ed_lrr_gui/router.py b/ed_lrr_gui/router.py index be17c00..12d3ade 100644 --- a/ed_lrr_gui/router.py +++ b/ed_lrr_gui/router.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import queue from collections import namedtuple from datetime import datetime, timedelta @@ -20,7 +21,7 @@ class Router(Process): self.queue.put({"status": state}) def run(self): - print("Route(): ",self.args,self.kwargs) + print("Route(): ", self.args, self.kwargs) route = _ed_lrr.route(*self.args, **self.kwargs) self.queue.put({"return": route}) diff --git a/ed_lrr_gui/web/__init__.py b/ed_lrr_gui/web/__init__.py index 5ea9afe..f420cf0 100644 --- a/ed_lrr_gui/web/__init__.py +++ b/ed_lrr_gui/web/__init__.py @@ -1 +1,2 @@ +# -*- coding: utf-8 -*- from .app import app, templates, db diff --git a/ed_lrr_gui/web/app.py b/ed_lrr_gui/web/app.py index 8fccda8..3427d6b 100644 --- a/ed_lrr_gui/web/app.py +++ b/ed_lrr_gui/web/app.py @@ -1,28 +1,22 @@ +# -*- coding: utf-8 -*- from flask import ( Flask, jsonify, - session, render_template, redirect, url_for, send_from_directory, request, flash, - current_app + current_app, ) -from flask.json.tag import JSONTag +from flask.cli import AppGroup import uuid -import pickle import os -import time -import random -import base64 -import gevent +import click from functools import wraps from concurrent.futures.process import BrokenProcessPool from datetime import datetime, timedelta -from decimal import Decimal -from multiprocessing import Queue from webargs import fields, validate from webargs.flaskparser import use_kwargs @@ -50,11 +44,9 @@ from flask_login import ( from flask_debugtoolbar import DebugToolbarExtension -from werkzeug.http import HTTP_STATUS_CODES from sqlalchemy_utils import generic_repr, JSONType, PasswordType, UUIDType -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import scoped_session, sessionmaker, relationship, backref -from sqlalchemy.types import Float, String, DateTime +from sqlalchemy.orm import relationship +from sqlalchemy.types import DateTime from jinja2.exceptions import TemplateNotFound from .forms import RouteForm, LoginForm, RegisterForm, ChangePasswordForm from .utils import prepare_route, BootsrapRenderer, is_safe_url @@ -65,22 +57,24 @@ templates = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates" app = Flask(__name__, template_folder=templates) app.config.from_pyfile("config.py") -executor = Executor(app) -db = SQLAlchemy(app) -bootstrap = Bootstrap(app) -csrf = CSRFProtect(app) -nav = Nav(app) -login_manager = LoginManager(app) +app.executor = executor = Executor(app) +app.db = db = SQLAlchemy(app) +app.bootstrap = bootstrap = Bootstrap(app) +app.csrf = csfr = CSRFProtect(app) +app.nav = nav = Nav(app) +app.login_manager = login_manager = LoginManager(app) login_manager.login_view = "login" login_manager.session_protection = "strong" admin = Admin(app, name="ED_LRR", template_mode="bootstrap3") -app.debug=True -toolbar = DebugToolbarExtension(app) +app.debug = True +app.toolbar = toolbar = DebugToolbarExtension(app) def wants_json_response(): - return request.accept_mimetypes['application/json'] >= \ - request.accept_mimetypes['text/html'] + return ( + request.accept_mimetypes["application/json"] + >= request.accept_mimetypes["text/html"] + ) @app.errorhandler(422) @@ -89,30 +83,34 @@ def wants_json_response(): @app.errorhandler(404) def handle_error(err): if wants_json_response(): - return jsonify(error=str(err),code=err.code), err.code - templates=["error/{}.html".format(err.code),"error/default.html"] + return jsonify(error=str(err), code=err.code), err.code + templates = ["error/{}.html".format(err.code), "error/default.html"] try: print(dir(err)) - return render_template(templates,error=err),err.code + return render_template(templates, error=err), err.code except TemplateNotFound: return err.get_response() + def role_required(*roles): def wrapper(fn): @wraps(fn) def decorated_view(*args, **kwargs): if not current_user.is_authenticated(): - return current_app.login_manager.unauthorized() - has_role=False - user=current_app.login_manager.reload_user() + return current_app.login_manager.unauthorized() + has_role = False + user = current_app.login_manager.reload_user() for role in roles: - has_role|=user.has_role(role) + has_role |= user.has_role(role) if not has_role: return current_app.login_manager.unauthorized() return fn(*args, **kwargs) + return decorated_view + return wrapper + @login_manager.user_loader def load_user(user_name): return User.query.get(user_name) @@ -120,7 +118,7 @@ def load_user(user_name): @login_manager.request_loader def load_user_from_header(header_val): - for api_key in [request.args.get('api_key'),request.headers.get('X-API-Key')]: + for api_key in [request.args.get("api_key"), request.headers.get("X-API-Key")]: if api_key: user = User.query.filter_by(api_key=api_key).one_or_none() if user: @@ -130,20 +128,21 @@ def load_user_from_header(header_val): def left_nav(): - links=[View("Home", "index"),View("Route", "route"),View("Jobs", "status",job_id=None)] - if current_user.has_role('admin') or current_user.has_role('worker_host'): - links.insert(2,View("Workers","worker")) - return Navbar( - "E:D LRR", - *links - ) + links = [ + View("Home", "index"), + View("Route", "route"), + View("Jobs", "status", job_id=None), + ] + if current_user.has_role("admin") or current_user.has_role("worker_host"): + links.insert(2, View("Workers", "worker")) + return Navbar("E:D LRR", *links) def right_nav(): links = [View("Login", "login"), View("Register", "register")] if current_user.is_authenticated: links = [View("Change Password", "change_password"), View("Logout", "logout")] - if current_user.has_role('admin'): + if current_user.has_role("admin"): links = [View("Admin", "admin.index")] + links return Navbar("", *links) @@ -158,16 +157,15 @@ def compute_route(args, kwargs): class AnonymousUser(AnonymousUserMixin): - - def has_role(self,role): + def has_role(self, role): return False @property def roles(self): return [] - + @roles.setter - def __set_roles(self,value): + def __set_roles(self, value): raise NotImplementedError @@ -180,52 +178,68 @@ class Worker(db.Model): UUIDType(binary=False, native=False), primary_key=True, default=uuid.uuid4 ) name = db.Column(db.String, unique=True) - current_job=db.Column(UUIDType(binary=False, native=False),db.ForeignKey("job.id"), nullable=True,default=None) - job = relationship('Job',backref="workers") - last_active = db.Column(DateTime, nullable=True,default=None) - owner_name = db.Column( - db.String, db.ForeignKey("user.name"), nullable=True,index=True + current_job = db.Column( + UUIDType(binary=False, native=False), + db.ForeignKey("job.id"), + nullable=True, + default=None, ) - owner = relationship("User",backref="workers") + job = relationship("Job", backref="workers") + last_active = db.Column(DateTime, nullable=True, default=None) + owner_name = db.Column( + db.String, db.ForeignKey("user.name"), nullable=True, index=True + ) + owner = relationship("User", backref="workers") -user_roles = db.Table('user_roles', - db.Column('user_name', db.String, db.ForeignKey('user.name'),primary_key=True), - db.Column('role_name', db.String, db.ForeignKey('role.name'),primary_key=True) + +user_roles = db.Table( + "user_roles", + db.Column("user_name", db.String, db.ForeignKey("user.name"), primary_key=True), + db.Column("role_name", db.String, db.ForeignKey("role.name"), primary_key=True), ) -class Role(db.Model): - name = db.Column(db.String, unique=True,index=True,primary_key=True) - def __init__(self,name): - self.name=name +class Role(db.Model): + name = db.Column(db.String, unique=True, index=True, primary_key=True) + + def __init__(self, name): + self.name = name def __repr__(self): return self.name + class User(db.Model, UserMixin): - name = db.Column(db.String, unique=True,index=True,primary_key=True) + name = db.Column(db.String, unique=True, index=True, primary_key=True) is_active = db.Column(db.Boolean, default=False) api_key = db.Column( - UUIDType(binary=False, native=False), nullable=True, default=uuid.uuid4,index=True + UUIDType(binary=False, native=False), + nullable=True, + default=uuid.uuid4, + index=True, ) password = db.Column(PasswordType(schemes=["pbkdf2_sha512"], max_length=256)) created = db.Column(DateTime, default=datetime.today) - roles = db.relationship("Role",secondary="user_roles") + roles = db.relationship("Role", secondary="user_roles") - def add_roles(self,roles): + def add_roles(self, roles): for role_name in roles: - role=Role.query.filter_by(name=role_name).one() - if not role in self.roles: + role = Role.query.filter_by(name=role_name).one() + if role not in self.roles: self.roles.append(role) db.session.commit() - def has_role(self,role_name): - return Role.query.join(User.roles).filter(User.name==self.name,Role.name==role_name).count()>0 - return ret - + def has_role(self, role_name): + return ( + Role.query.join(User.roles) + .filter(User.name == self.name, Role.name == role_name) + .count() + > 0 + ) + def reset_api_key(self): - self.api_key=uuid,uuid4() + self.api_key = uuid.uuid4() db.session.add(self) db.session.comiit() @@ -241,24 +255,23 @@ class Job(db.Model): UUIDType(binary=False, native=False), primary_key=True, default=uuid.uuid4 ) user_name = db.Column( - db.String, db.ForeignKey("user.name"), nullable=True,index=True + db.String, db.ForeignKey("user.name"), nullable=True, index=True ) func = db.Column(db.String) args = db.Column(JSONType) kwargs = db.Column(JSONType) state = db.Column(JSONType, default={}) - priority = db.Column(db.Integer, default=0,nullable=True) + priority = db.Column(db.Integer, default=0, nullable=True) created = db.Column(DateTime, default=datetime.today) finished = db.Column(DateTime, nullable=True, default=None) started = db.Column(DateTime, nullable=True, default=None) last_update = db.Column(DateTime, nullable=True, default=None) - user = relationship("User",backref="jobs") + user = relationship("User", backref="jobs") # ============================================================ def __repr__(self): return str(self.id) - @property def future(self): fut = executor.futures._futures.get(self.id) @@ -266,20 +279,26 @@ class Job(db.Model): @property def sort_key(self): - state_priorities={"Queued":0,"Starting":1,"Error":1,"Stalled":1,"Running":1} - status_key=state_priorities.get(self.status[1],-1)+1 - user=1-int(self.user is not None) - return (user,-status_key,self.priority,self.created) + state_priorities = { + "Queued": 0, + "Starting": 1, + "Error": 1, + "Stalled": 1, + "Running": 1, + } + status_key = state_priorities.get(self.status[1], -1) + 1 + user = 1 - int(self.user is not None) + return (user, -status_key, self.priority, self.created) @property def age(self): - dt=datetime.today()-self.created + dt = datetime.today() - self.created return dt - dt % timedelta(seconds=1) @classmethod - def next(cls): - for job in sorted(cls.query.all(),key=lambda v:v.sort_key): - if job.status[1] in ['Done']: + def get_next(cls): + for job in sorted(cls.query.all(), key=lambda v: v.sort_key): + if job.status[1] in ["Done"]: continue return job return None @@ -287,15 +306,15 @@ class Job(db.Model): @property def status(self): - states=[ - ("primary", "Done"), - ("danger", "Error"), - ("info", "Stalled"), - ("success", "Running"), - ("secondary", "Starting"), - ("warning", "Queued") - ] - #return states[self.id.int%len(states)] + # [ + # ("primary", "Done"), + # ("danger", "Error"), + # ("info", "Stalled"), + # ("success", "Running"), + # ("secondary", "Starting"), + # ("warning", "Queued"), + # ] + # return states[self.id.int%len(states)] if self.state.get("result"): return ("primary", "Done") if self.state.get("error"): @@ -391,7 +410,7 @@ class Job(db.Model): ).total_seconds() if time_since_last_upd < 5.0: return - state = dict() + state = {} state.update(self.state) state.update({"progress": cb_state}) self.state = state @@ -403,7 +422,7 @@ class Job(db.Model): def done(self, future): print(self.id, "DONE") - state = dict() + state = {} state.update(self.state) executor.futures.pop(self.id) exc = future.exception() @@ -420,24 +439,25 @@ class Job(db.Model): db.create_all() -for role in ['admin','user','worker_host']: +for role in ["admin", "user", "worker_host"]: if Role.query.filter_by(name=role).one_or_none() is None: db.session.add(Role(role)) -def create_user(name,password,roles,active=False): - user=User.query.filter_by(name=name).one_or_none() + +def create_user(name, password, roles, active=False): + user = User.query.filter_by(name=name).one_or_none() if user: db.session.delete(user) - user=User(name=name,password=password,is_active=active) + user = User(name=name, password=password, is_active=active) user.add_roles(roles) db.session.add(user) db.session.commit() return user -create_user('admin','admin',['admin','user'],True) -create_user('user','user',['user'],True) -create_user('host','host',['user','worker_host'],True) +# create_user("admin", "admin", ["admin", "user"], True) +# create_user("user", "user", ["user"], True) +# create_user("host", "host", ["user", "worker_host"], True) class SQLAView(ModelView): @@ -449,7 +469,7 @@ class SQLAView(ModelView): column_display_pk = True def is_accessible(self): - return current_user.is_authenticated and current_user.has_role('admin') + return current_user.is_authenticated and current_user.has_role("admin") def inaccessible_callback(self, name, **kwargs): return redirect(url_for("login")) @@ -458,7 +478,7 @@ class SQLAView(ModelView): class UserView(SQLAView): from wtforms import PasswordField - column_list = ("name", "active", "password", "api_key","roles") + column_list = ("name", "active", "password", "api_key", "roles") column_formatters = { "password": lambda view, context, model, name: "", "api_key": lambda view, context, model, name: model.api_key or "", @@ -469,9 +489,7 @@ class UserView(SQLAView): class JobView(SQLAView): # Job.id,Job.user,Job.func,Job.args,Job.kwargs,Job.state,Job.created,Job.finished,Job.started,Job.last_update column_list = ("id", "status", "user", "created", "started", "finished") - column_formatters = { - "status": lambda view, context, model, name: model.status[1], - } + column_formatters = {"status": lambda view, context, model, name: model.status[1]} class WorkerView(SQLAView): @@ -485,6 +503,7 @@ class WorkerView(SQLAView): # "status": lambda view, context, model, name: model.status[1], # } + admin.add_view(JobView(Job, db.session)) admin.add_view(UserView(User, db.session)) admin.add_view(SQLAView(Worker, db.session)) @@ -531,7 +550,7 @@ def api_route(_=None, **args): args["factor"], None, r"D:\devel\rust\ED_LRR\stars.csv", - app.config['ROUTE_WORKERS'] + app.config["ROUTE_WORKERS"], ) return jsonify({"id": submit_job(ed_lrr.route, *args)}) @@ -544,7 +563,7 @@ def api_status(): @app.route("/api/whoami") def api_whoami(): - return jsonify({'name':current_user.name}) + return jsonify({"name": current_user.name}) @app.route("/api/status/") @@ -573,21 +592,21 @@ def route(): return render_template("form.html", form=form, title="Plot Route") -@app.route("/status/",defaults={'job_id':None}) +@app.route("/status/", defaults={"job_id": None}) @app.route("/status/") @login_required def status(job_id=None): if job_id is not None: - job=Job.query.get_or_404(str(job_id)) + job = Job.query.get_or_404(str(job_id)) return render_template("job.html", job=job) - return render_template( - "status.html", Job=Job, state=request.args.get("state") - ) + return render_template("status.html", Job=Job, state=request.args.get("state")) + @app.route("/") def index(): return render_template("index.html") + @app.route("/login", methods=["GET", "POST"]) def login(): if current_user.is_authenticated: @@ -602,9 +621,9 @@ def login(): flash("Account is deactivated!", "warning") return redirect(url_for("login")) login_user(user, remember=form.data["remember"]) - next = request.args.get('next') + next = request.args.get("next") if not is_safe_url(next): - next=None + next = None return redirect(next or url_for("status")) return render_template("form.html", form=form, title="Login") @@ -614,7 +633,7 @@ def register(): form = RegisterForm() if form.validate_on_submit(): if User.query.filter_by(name=form.data["username"]).one_or_none() is not None: - flash('Username already exists','danger') + flash("Username already exists", "danger") return render_template("form.html", form=form, title="Register") user = User() user.name = form.data["username"] @@ -641,11 +660,13 @@ def change_password(): return redirect(url_for("status")) return render_template("form.html", form=form, title="Register") + @app.route("/workers/") @login_required def worker(): return render_template("workers.html") + @app.route("/logout") def logout(): logout_user() @@ -654,12 +675,38 @@ def logout(): @app.before_first_request def resume_jobs(): - print(Job.next()) + print("NEXT:", Job.get_next()) with app.test_request_context(): for job in Job.query.all(): if job.status[1] != "Done": print("Restarting {} with state {}".format(job.id, job.status[1])) job.start() + +user_cli = AppGroup('user', help="Manage users") +job_cli = AppGroup('job', help="Manage Jobs") +worker_cli = AppGroup('worker', help="Manage Workers") + + +@app.cli.command("gevent") +def cmd_gevent(): + return + + +@user_cli.command("create") +@click.argument("name") +@click.option("-i", "--inactive", help="Crate account as inactive", is_flag=True, default=False) +@click.option("-r", "--role", help="Assign role to account", default=["user"], multiple=True) +@click.password_option("-p", "--password", help="Password for user") +def cmd_create_user(name, role, password, inactive): + "Create a new user" + create_user(name, password, role, not inactive) + print("User created!") + + +app.cli.add_command(user_cli) +app.cli.add_command(job_cli) +app.cli.add_command(worker_cli) + if __name__ == "__main__": app.run(host="127.0.0.1", port=3777, debug=True) diff --git a/ed_lrr_gui/web/config.py b/ed_lrr_gui/web/config.py index 4811887..71c977f 100644 --- a/ed_lrr_gui/web/config.py +++ b/ed_lrr_gui/web/config.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os SECRET_KEY = "ED_LRR_WEBAPP" @@ -8,11 +9,11 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False ROUTE_WORKERS = 0 EXECUTOR_TYPE = "process" -EXECUTOR_MAX_WORKERS = os.cpu_count()-1 +EXECUTOR_MAX_WORKERS = os.cpu_count() - 1 EXECUTOR_FUTURES_MAX_LENGTH = 500 FLASK_ADMIN_SWATCH = "Darkly" DEBUG_TB_TEMPLATE_EDITOR_ENABLED = True -MAIL_DEFAULT_SENDER = '"ED_LRR Admin" ' \ No newline at end of file +MAIL_DEFAULT_SENDER = '"ED_LRR Admin" ' diff --git a/ed_lrr_gui/web/forms.py b/ed_lrr_gui/web/forms.py index 785d8dc..cd7edae 100644 --- a/ed_lrr_gui/web/forms.py +++ b/ed_lrr_gui/web/forms.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from flask_wtf import FlaskForm from wtforms import ( StringField, @@ -14,6 +15,7 @@ from wtforms.widgets.html5 import NumberInput from wtforms.widgets import TextInput from wtforms.validators import ValidationError + class StringListField(Field): widget = TextInput() @@ -61,7 +63,7 @@ class RouteForm(FlaskForm): default=50, widget=NumberInput(min=0, max=100, step=1), ) - + priority = FloatField( "Priority (0=max, 100=min)", [validators.NumberRange(0, 100)], diff --git a/ed_lrr_gui/web/static/theme.css b/ed_lrr_gui/web/static/theme.css index cb647bb..a94148c 100644 --- a/ed_lrr_gui/web/static/theme.css +++ b/ed_lrr_gui/web/static/theme.css @@ -20,4 +20,4 @@ table { border: 1px solid #eee; width: 512px; height: 512px; -} \ No newline at end of file +} diff --git a/ed_lrr_gui/web/templates/admin/index.html b/ed_lrr_gui/web/templates/admin/index.html index fb7f8ef..a862dea 100644 --- a/ed_lrr_gui/web/templates/admin/index.html +++ b/ed_lrr_gui/web/templates/admin/index.html @@ -2,4 +2,4 @@ {% block body %}

Hello world

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/base.html b/ed_lrr_gui/web/templates/base.html index 1bfa00a..52b69b4 100644 --- a/ed_lrr_gui/web/templates/base.html +++ b/ed_lrr_gui/web/templates/base.html @@ -45,4 +45,4 @@ {# application content needs to be provided in the app_content block #} {% block app_content %}{% endblock %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/error/404.html b/ed_lrr_gui/web/templates/error/404.html index ff69b53..70ce510 100644 --- a/ed_lrr_gui/web/templates/error/404.html +++ b/ed_lrr_gui/web/templates/error/404.html @@ -3,4 +3,4 @@ {% block app_content %}

404 Not Found

-{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/form.html b/ed_lrr_gui/web/templates/form.html index f8ed079..0150748 100644 --- a/ed_lrr_gui/web/templates/form.html +++ b/ed_lrr_gui/web/templates/form.html @@ -13,4 +13,4 @@ {{ wtf.quick_form(form) }} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/index.html b/ed_lrr_gui/web/templates/index.html index 28898b2..accf70d 100644 --- a/ed_lrr_gui/web/templates/index.html +++ b/ed_lrr_gui/web/templates/index.html @@ -7,4 +7,4 @@ Number of Jobs: {{current_user.jobs|count}} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/job.html b/ed_lrr_gui/web/templates/job.html index 224af3e..71c8a83 100644 --- a/ed_lrr_gui/web/templates/job.html +++ b/ed_lrr_gui/web/templates/job.html @@ -128,4 +128,4 @@ {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/status.html b/ed_lrr_gui/web/templates/status.html index d0a9fe2..22226db 100644 --- a/ed_lrr_gui/web/templates/status.html +++ b/ed_lrr_gui/web/templates/status.html @@ -84,4 +84,4 @@ {% endfor %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/templates/workers.html b/ed_lrr_gui/web/templates/workers.html index 9e58735..179908d 100644 --- a/ed_lrr_gui/web/templates/workers.html +++ b/ed_lrr_gui/web/templates/workers.html @@ -11,4 +11,4 @@ {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ed_lrr_gui/web/utils.py b/ed_lrr_gui/web/utils.py index 6f5fe35..76d69d7 100644 --- a/ed_lrr_gui/web/utils.py +++ b/ed_lrr_gui/web/utils.py @@ -1,13 +1,15 @@ +# -*- coding: utf-8 -*- from flask_nav.renderers import Renderer from dominate import tags -from urllib.parse import urlparse,urljoin +from urllib.parse import urlparse, urljoin from flask import request + def is_safe_url(target): ref_url = urlparse(request.host_url) test_url = urlparse(urljoin(request.host_url, target)) - return test_url.scheme in ('http', 'https') and \ - ref_url.netloc == test_url.netloc + return test_url.scheme in ("http", "https") and ref_url.netloc == test_url.netloc + def dist(p1, p2): s = 0 diff --git a/ed_lrr_gui/web/worker.py b/ed_lrr_gui/web/worker.py index 7182efe..cee6ff8 100644 --- a/ed_lrr_gui/web/worker.py +++ b/ed_lrr_gui/web/worker.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import requests as RQ import _ed_lrr as ed_lrr diff --git a/icon/make.py b/icon/make.py index 5e67641..2fccd8f 100644 --- a/icon/make.py +++ b/icon/make.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import svgwrite import random import time diff --git a/icon/out/icon_1.svg b/icon/out/icon_1.svg index 0538a0a..67e6905 100644 --- a/icon/out/icon_1.svg +++ b/icon/out/icon_1.svg @@ -1,2 +1,2 @@ - \ No newline at end of file + diff --git a/icon/out/icon_1_small.svg b/icon/out/icon_1_small.svg index 9a6c926..ddc0a9a 100644 --- a/icon/out/icon_1_small.svg +++ b/icon/out/icon_1_small.svg @@ -1,2 +1,2 @@ - \ No newline at end of file + diff --git a/installer/ED_LRR.iss b/installer/ED_LRR.iss index 2498082..4de1576 100644 --- a/installer/ED_LRR.iss +++ b/installer/ED_LRR.iss @@ -11,7 +11,7 @@ OutputBaseFilename="ED_LRR Setup" ChangesEnvironment = true [Files] -Source: "..\exe\__main__.dist\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs +Source: "{#SourceFolder}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs [Icons] Name: "{group}\ED_LRR"; Filename: "{app}\ED_LRR.exe"; WorkingDir: "{app}" diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..2d99fc3 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,107 @@ +import nox +from nox.logger import logger +import os +import shutil + +to_append = [] + +path = os.environ.get("PATH", "").split(os.pathsep) +while True: + python = shutil.which("python", path=os.pathsep.join(path)) + if python is None: + break + python = os.path.dirname(python) + for elem in path: + if elem.startswith(python): + path.remove(elem) + to_append.append(elem) + +path += to_append + +path = os.pathsep.join(path) +os.environ["PATH"] = path + +versions = ["3.{}".format(s) for s in [5, 6, 7, 8]] + +versions += ["3"] + +nox.options.keywords = "test" + + +@nox.session(venv_backend="conda") +def devenv(session): + """Set up development environment""" + global path + location = os.path.abspath(session._runner.venv.location_name) + session.env["PATH"] = os.pathsep.join([location, path, location]) + session.conda_install("pycrypto", "ujson") + session.install("--no-cache-dir", "-e", ".[all]") + logger.warning(f'Devenv set up, now run "conda activate {location}"') + + +@nox.session(python=versions, venv_backend="conda") +def test(session): + """Run the test suite.""" + global path + location = os.path.abspath(session._runner.venv.location_name) + python = os.path.join(location, "python.exe") + # friendly_name = session._runner.friendly_name + cargo_args = ["--manifest-path", "./rust/Cargo.toml"] + session.env["PATH"] = os.pathsep.join([location, path, location]) + # ================================================ + session.run(python, "-VV", external=True) + session.run("cargo", "clean", *cargo_args, external=True) + session.run("cargo", "clean", *(cargo_args + ["--release"]), external=True) + session.run("cargo", "test", *cargo_args, external=True) + session.conda_install("pycrypto", "ujson") + session.install("--no-cache-dir", "setuptools_rust") + session.install("--no-cache-dir", ".[all]") + session.run("py.test", "-v", *(session.posargs or [])) + # if session.python: + # session.notify(f"build-{session.python}") + + +@nox.session(python=versions, venv_backend="conda") +def build(session): + "Build installer and zip" + global path + location = session._runner.venv.location_name + python = os.path.join(location, "python.exe") + location = os.path.abspath(location) + cargo_args = ["--manifest-path", "./rust/Cargo.toml"] + session.env["PATH"] = os.pathsep.join([location, path, location]) + # ================================================ + session.run(python, "-VV", external=True) + session.run("cargo", "clean", *cargo_args, external=True) + session.run("cargo", "clean", *(cargo_args + ["--release"]), external=True) + session.conda_install("pycrypto", "ujson") + session.install("--no-cache-dir", ".[gui,web,build]") + session.run( + "pyinstaller", + "-y", + "--console", + "--noupx", + "--clean", + "--hidden-import", + "pkg_resources.py2_warn", + "--name", + f"ED_LRR_{session.python}", + "--specpath", + "build", + "ed_lrr_gui/__main__.py", + external=True, + silent=True, + ) + if session.python: + source_path = os.path.abspath(f"./dist/ED_LRR_{session.python}") + else: + source_path = os.path.abspath(f"./dist/ED_LRR") + source_path = source_path.strip(os.sep) + session.run( + "iscc", + f'/FED_LRR_{session.python}', + f'/DSourceFolder={source_path}', + "installer/ED_LRR.iss", + external=True, + silent=True, + ) diff --git a/pyproject.toml b/pyproject.toml index 2b6e07f..ce3a100 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,3 @@ [build-system] -requires = ["setuptools", "wheel", "setuptools-rust"] \ No newline at end of file +requires = ["setuptools", "wheel","setuptools_rust"] +build-backend = "setuptools.build_meta" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index c6c5435..0000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -qt_api=pyqt5 -addopts = --cov=ed_lrr_gui --cov-report html \ No newline at end of file diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 48bb1b3..28922b0 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,19 +1,11 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "ahash" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "const-random 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "aho-corasick" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -21,16 +13,11 @@ name = "atty" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "autocfg" version = "1.0.0" @@ -38,12 +25,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "backtrace" -version = "0.3.43" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -53,7 +40,7 @@ version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -61,7 +48,7 @@ name = "better-panic" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)", "console 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -70,10 +57,15 @@ name = "bincode" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "block-buffer" version = "0.7.3" @@ -81,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -95,13 +87,13 @@ dependencies = [ [[package]] name = "bstr" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -111,7 +103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "byteorder" -version = "1.3.2" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -131,10 +123,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "console" version = "0.9.2" @@ -143,44 +143,27 @@ dependencies = [ "clicolors-control 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "const-random" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "const-random-macro 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "const-random-macro" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam-channel" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "crossbeam-utils" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -190,28 +173,48 @@ name = "csv" version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bstr 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "csv-core 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "bstr 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "csv-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "csv-core" -version = "0.1.6" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "ctor" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "derivative" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "dict_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -228,19 +231,21 @@ version = "0.2.0" dependencies = [ "better-panic 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "derivative 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "dict_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "humantime 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "permutohedron 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "pyo3 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pyo3 0.9.0-alpha.1 (git+https://github.com/PyO3/pyo3)", "rstar 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "simd-json 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -248,14 +253,6 @@ name = "encode_unicode" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "float-cmp" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "fnv" version = "1.0.6" @@ -269,16 +266,6 @@ dependencies = [ "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "getrandom" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", - "wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ghost" version = "0.1.1" @@ -286,33 +273,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "halfbrown" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hashbrown" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "hermit-abi" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -337,7 +306,7 @@ dependencies = [ "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", "unindent 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -346,7 +315,7 @@ name = "inventory" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ctor 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "ctor 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", "ghost 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "inventory-impl 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -358,7 +327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -378,16 +347,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.66" +version = "0.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lock_api" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "memchr" -version = "2.3.0" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "num-traits" @@ -397,29 +376,60 @@ dependencies = [ "autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num_cpus" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "opaque-debug" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "paste" -version = "0.1.6" +name = "parking_lot" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "paste-impl 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "paste" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "paste-impl 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "paste-impl" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -439,7 +449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -452,43 +462,43 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.9.0-alpha.1" +source = "git+https://github.com/PyO3/pyo3#74b22eb651aac14cd64524219943b77cf7e700ac" dependencies = [ "indoc 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "inventory 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "paste 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "pyo3cls 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", - "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "paste 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "pyo3cls 0.9.0-alpha.1 (git+https://github.com/PyO3/pyo3)", + "regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", "unindent 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "pyo3-derive-backend" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.9.0-alpha.1" +source = "git+https://github.com/PyO3/pyo3#74b22eb651aac14cd64524219943b77cf7e700ac" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "pyo3cls" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" +version = "0.9.0-alpha.1" +source = "git+https://github.com/PyO3/pyo3#74b22eb651aac14cd64524219943b77cf7e700ac" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "pyo3-derive-backend 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pyo3-derive-backend 0.9.0-alpha.1 (git+https://github.com/PyO3/pyo3)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -499,14 +509,19 @@ dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "regex" -version = "1.3.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -515,12 +530,12 @@ name = "regex-automata" version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -542,32 +557,37 @@ name = "ryu" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_json" -version = "1.0.45" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -583,31 +603,18 @@ dependencies = [ ] [[package]] -name = "simd-json" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "float-cmp 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "halfbrown 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "spin" -version = "0.5.2" +name = "smallvec" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "strsim" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "syn" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -620,7 +627,7 @@ name = "termios" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -651,11 +658,6 @@ name = "version_check" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "winapi" version = "0.3.8" @@ -676,41 +678,37 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] -"checksum ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" -"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" +"checksum aho-corasick 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "743ad5a418686aad3b87fd14c43badd828cf26e214a00f92a384291cf22e1811" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" -"checksum backtrace 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)" = "7f80256bc78f67e7df7e36d77366f636ed976895d91fe2ab9efa3973e8fe8c4f" +"checksum backtrace 0.3.44 (registry+https://github.com/rust-lang/crates.io-index)" = "e4036b9bf40f3cf16aba72a3d65e8a520fc4bafcdc7079aea8f848c58c5b5536" "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" "checksum better-panic 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d12a680cc74d8c4a44ee08be4a00dedf671b089c2440b2e3fdaa776cd468476" "checksum bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" "checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -"checksum bstr 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "fe8a65814ca90dfc9705af76bb6ba3c6e2534489a72270e797e603783bb4990b" +"checksum bstr 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "502ae1441a0a5adb8fbd38a5955a6416b9493e92b465de5e4a9bde6a539c2c48" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" -"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum clicolors-control 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90082ee5dcdd64dc4e9e0d37fbf3ee325419e39c0092191e0393df65518f741e" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum console 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "45e0f3986890b3acbc782009e2629dfe2baa430ac091519ce3be26164a2ae6c0" -"checksum const-random 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" -"checksum const-random-macro 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" -"checksum crossbeam-channel 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "acec9a3b0b3559f15aee4f90746c4e5e293b701c0f7d3925d24e01645267b68c" -"checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" +"checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061" +"checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum csv 1.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "00affe7f6ab566df61b4be3ce8cf16bc2576bca0963ceb0955e45d514bf9a279" -"checksum csv-core 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9b5cadb6b25c77aeff80ba701712494213f4a8418fcda2ee11b6560c3ad0bf4c" -"checksum ctor 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8ce37ad4184ab2ce004c33bf6379185d3b1c95801cab51026bd271bf68eedc" +"checksum csv-core 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +"checksum ctor 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "47c5e5ac752e18207b12e16b10631ae5f7f68f8805f335f9b817ead83d9ffce1" +"checksum derivative 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1b94d2eb97732ec84b4e25eaf37db890e317b80e921f168c82cb5282473f8151" +"checksum dict_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d7ee5676aa48357bd5e8bcf095167e6678837232a7b85bce424f46cd93a2c60" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum encode_unicode 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -"checksum float-cmp 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "75224bec9bfe1a65e2d34132933f2de7fe79900c96a0174307554244ece8150e" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" -"checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" "checksum ghost 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a36606a68532b5640dc86bb1f33c64b45c4682aad4c50f3937b317ea387f3d6" -"checksum halfbrown 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "f28ee31ba5bb3a3251606db26de2e807552c5f295429d03f756bdc4e00f54c7a" -"checksum hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" -"checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" +"checksum hermit-abi 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "1010591b26bbfe835e9faeabeb11866061cc7dcebffd56ad7d0942d0e61aefd8" "checksum humantime 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b9b6c53306532d3c8e8087b44e6580e10db51a023cf9b433cea2ac38066b92da" "checksum indoc 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "3f9553c1e16c114b8b77ebeb329e5f2876eed62a8d51178c8bc6bff0d65f98f8" "checksum indoc-impl 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b714fc08d0961716390977cdff1536234415ac37b509e34e5a983def8340fb75" @@ -719,41 +717,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" "checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" -"checksum memchr 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3197e20c7edb283f87c071ddfc7a2cca8f8e0b888c242959846a6fce03c72223" +"checksum libc 0.2.67 (registry+https://github.com/rust-lang/crates.io-index)" = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" +"checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" "checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" -"checksum paste 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "423a519e1c6e828f1e73b720f9d9ed2fa643dce8a7737fb43235ce0b41eeaa49" -"checksum paste-impl 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "4214c9e912ef61bf42b81ba9a47e8aad1b2ffaf739ab162bf96d1e011f54e6c5" +"checksum parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +"checksum parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +"checksum paste 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "63e1afe738d71b1ebab5f1207c055054015427dbfc7bbe9ee1266894156ec046" +"checksum paste-impl 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc4a7f6f743211c5aab239640a65091535d97d43d92a52bca435a640892bb" "checksum pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ec91767ecc0a0bbe558ce8c9da33c068066c57ecc8bb8477ef8c1ad3ef77c27" "checksum permutohedron 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b687ff7b5da449d39e418ad391e5e08da53ec334903ddbb921db208908fc372c" "checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" "checksum proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3acb317c6ff86a4e579dfa00fc5e6cca91ecbb4e7eb2df0468805b674eb88548" -"checksum pyo3 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e1bfe257586436fbe1296d917f14a167d4253d0873bf43e2c9b9bdd58a3f9f35" -"checksum pyo3-derive-backend 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4882d8237fd8c7373cc25cb802fe0dab9ff70830fd56f47ef6c7f3f287fcc057" -"checksum pyo3cls 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fdf321cfab555f7411298733c86d21e5136f5ded13f5872fabf9de3337beecda" +"checksum pyo3 0.9.0-alpha.1 (git+https://github.com/PyO3/pyo3)" = "" +"checksum pyo3-derive-backend 0.9.0-alpha.1 (git+https://github.com/PyO3/pyo3)" = "" +"checksum pyo3cls 0.9.0-alpha.1 (git+https://github.com/PyO3/pyo3)" = "" "checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" -"checksum regex 1.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b5508c1941e4e7cb19965abef075d35a9a8b5cdf0846f30b4050e9b55dc55e87" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum regex 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" "checksum regex-automata 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "92b73c2a1770c255c240eaa4ee600df1704a38dc3feaa6e949e7fcd4f8dc09f9" -"checksum regex-syntax 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e734e891f5b408a29efbf8309e656876276f49ab6a6ac208600b4419bd893d90" +"checksum regex-syntax 0.6.14 (registry+https://github.com/rust-lang/crates.io-index)" = "b28dfe3fe9badec5dbf0a79a9cccad2cfc2ab5484bdb3e44cbd1ae8b3ba2be06" "checksum rstar 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0650eaaa56cbd1726fd671150fce8ac6ed9d9a25d1624430d7ee9d196052f6b6" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" "checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8" -"checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" -"checksum serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" -"checksum serde_json 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)" = "eab8f15f15d6c41a154c1b128a22f2dfabe350ef53c40953d84e36155c91192b" +"checksum scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +"checksum serde 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)" = "e707fbbf255b8fc8c3b99abb91e7257a622caeb20a9818cbadbeeede4e0932ff" +"checksum serde_derive 1.0.105 (registry+https://github.com/rust-lang/crates.io-index)" = "ac5d00fc561ba2724df6758a17de23df5914f20e41cb00f94d5b7ae42fffaff8" +"checksum serde_json 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)" = "9371ade75d4c2d6cb154141b9752cf3781ec9c05e0e5cf35060e1e70ee7b9c25" "checksum sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd26bc0e7a2e3a7c959bc494caf58b72ee0c71d67704e9520f736ca7e4853ecf" -"checksum simd-json 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "fb65296b57a8709ea32a87cefc0e1099cdd407ee3aed89114af11c74ab86f7bf" -"checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -"checksum strsim 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" -"checksum syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "af6f3550d8dff9ef7dc34d384ac6f107e5d31c8f57d9f28e0081503f547ac8f5" +"checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +"checksum strsim 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +"checksum syn 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a0294dc449adc58bb6592fff1a23d3e5e6e235afc6a0ffca2657d19e7bbffe5" "checksum termios 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72b620c5ea021d75a735c943269bb07d30c9b77d6ac6b236bc8b5c496ef05625" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" "checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" "checksum unindent 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "63f18aa3b0e35fed5a0048f029558b1518095ffe2a0a31fb87c93dece93a4993" "checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce" -"checksum wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)" = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" "checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 40f1ec1..33b5eef 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -20,18 +20,20 @@ codegen-units = 1 lto = true [dependencies] -pyo3 = { version = "0.8.5", features = ["extension-module"] } +pyo3 = { git = "https://github.com/PyO3/pyo3", features = ["extension-module"] } csv = "1.1.3" -serde = { version = "1.0.104", features = ["derive"] } +serde = { version = "1.0.105", features = ["derive"] } humantime = "2.0.0" permutohedron = "0.2.4" -serde_json = "1.0.45" +serde_json = "1.0.48" fnv = "1.0.6" bincode = "1.2.1" sha3 = "0.8.2" -byteorder = "1.3.2" -strsim = "0.9.3" +byteorder = "1.3.4" +strsim = "0.10.0" rstar = "0.7.1" -crossbeam-channel = "0.4.0" -simd-json = "0.2.3" +crossbeam-channel = "0.4.2" better-panic = "0.2.0" +derivative = "2.0.2" +dict_derive = "0.2.0" +num_cpus = "1.12.0" diff --git a/rust/src/common.rs b/rust/src/common.rs index 0d5f4fe..78009f0 100644 --- a/rust/src/common.rs +++ b/rust/src/common.rs @@ -1,125 +1,145 @@ -use crate::route::Router; -use serde::{Deserialize, Serialize}; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::path::PathBuf; - -pub fn find_matches( - path: &PathBuf, - names: Vec, - exact: bool -) -> Result)>, String> { - let mut best: HashMap)> = HashMap::new(); - for name in &names { - best.insert(name.to_string(), (0.0, None)); - } - let mut reader = match csv::ReaderBuilder::new().from_path(path) { - Ok(rdr) => rdr, - Err(e) => { - return Err(format!("Error opening {}: {}", path.to_str().unwrap(), e).to_string()); - } - }; - let systems = reader.deserialize::(); - for sys in systems { - let sys = sys.unwrap(); - for name in &names { - best.entry(name.clone()).and_modify(|ent| { - if (exact)&&(&sys.system==name) { - *ent = (1.0, Some(sys.clone().build())) - } else { - let d1 = strsim::normalized_levenshtein(&sys.system, &name); - let d2 = strsim::normalized_levenshtein(&sys.body, &name); - if d1 > ent.0 { - *ent = (d1, Some(sys.clone().build())) - } else if d2 > ent.0 { - *ent = (d2, Some(sys.clone().build())) - } - } - - }); - } - } - Ok(best) -} - -#[derive(Debug, Clone,Copy, Serialize, Deserialize)] -pub struct TreeNode { - pub id: u32, - pub pos: [f32; 3], - pub mult: f32, -} - -impl TreeNode { - pub fn get(&self,router:&Router) -> Option { - let mut cache = router.cache.as_ref().unwrap().lock().unwrap(); - cache.get(self.id) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SystemSerde { - pub id: u32, - pub star_type: String, - pub system: String, - pub body: String, - pub mult: f32, - pub distance: u32, - pub x: f32, - pub y: f32, - pub z: f32, -} - -impl SystemSerde { - pub fn build(self) -> System { - System { - id: self.id, - star_type: self.star_type, - system: self.system, - body: self.body, - mult: self.mult, - distance: self.distance, - pos: [self.x, self.y, self.z], - } - } - - pub fn to_node(&self) -> TreeNode { - TreeNode { - id: self.id, - pos: [self.x, self.y, self.z], - mult: self.mult, - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct System { - pub id: u32, - pub star_type: String, - pub system: String, - pub body: String, - pub mult: f32, - pub distance: u32, - pub pos: [f32; 3], -} - -impl System { - pub fn to_node(&self) -> TreeNode { - TreeNode { - id: self.id, - pos: self.pos, - mult: self.mult, - } - } -} - -impl Ord for System { - fn cmp(&self, other: &Self) -> Ordering { - self.id.cmp(&other.id) - } -} - -impl PartialOrd for System { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} +use crate::route::Router; +use dict_derive::IntoPyObject; +use serde::{Deserialize, Serialize}; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::num::ParseIntError; +use std::path::PathBuf; +use std::str::FromStr; + +pub enum SysEntry { + ID(u32), + Name(String) +} + +impl SysEntry { + pub fn parse(s: &str) -> Self { + if let Ok(n) = s.parse() { + SysEntry::ID(n) + } else { + SysEntry::Name(s.to_owned()) + } + } +} + +pub fn find_matches( + path: &PathBuf, + names: Vec, + exact: bool, +) -> Result)>, String> { + let mut best: HashMap)> = HashMap::new(); + if names.is_empty() { + return Ok(best); + } + for name in &names { + best.insert(name.to_string(), (0.0, None)); + } + let mut reader = match csv::ReaderBuilder::new().from_path(path) { + Ok(rdr) => rdr, + Err(e) => { + return Err(format!("Error opening {}: {}", path.to_str().unwrap(), e).to_string()); + } + }; + let systems = reader.deserialize::(); + for sys in systems { + let sys = sys.unwrap(); + for name in &names { + best.entry(name.clone()).and_modify(|ent| { + if (exact) && (&sys.system == name) { + *ent = (1.0, Some(sys.clone().build())) + } else { + let d1 = strsim::normalized_levenshtein(&sys.system, &name); + let d2 = strsim::normalized_levenshtein(&sys.body, &name); + if d1 > ent.0 { + *ent = (d1, Some(sys.clone().build())) + } else if d2 > ent.0 { + *ent = (d2, Some(sys.clone().build())) + } + } + }); + } + } + Ok(best) +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct TreeNode { + pub id: u32, + pub pos: [f32; 3], + pub mult: f32, +} + +impl TreeNode { + pub fn get(&self, router: &Router) -> Option { + let mut cache = router.cache.as_ref().unwrap().lock().unwrap(); + cache.get(self.id) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, IntoPyObject)] +pub struct SystemSerde { + pub id: u32, + pub star_type: String, + pub system: String, + pub body: String, + pub mult: f32, + pub distance: u32, + pub x: f32, + pub y: f32, + pub z: f32, +} + +impl SystemSerde { + pub fn build(self) -> System { + System { + id: self.id, + star_type: self.star_type, + system: self.system, + body: self.body, + mult: self.mult, + distance: self.distance, + pos: [self.x, self.y, self.z], + } + } + + pub fn to_node(&self) -> TreeNode { + TreeNode { + id: self.id, + pos: [self.x, self.y, self.z], + mult: self.mult, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct System { + pub id: u32, + pub star_type: String, + pub system: String, + pub body: String, + pub mult: f32, + pub distance: u32, + pub pos: [f32; 3], +} + +impl System { + pub fn to_node(&self) -> TreeNode { + TreeNode { + id: self.id, + pos: self.pos, + mult: self.mult, + } + } +} + +impl Ord for System { + fn cmp(&self, other: &Self) -> Ordering { + self.id.cmp(&other.id) + } +} + +impl PartialOrd for System { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/rust/src/galaxy.rs b/rust/src/galaxy.rs index 9bd1fbb..5a91d46 100644 --- a/rust/src/galaxy.rs +++ b/rust/src/galaxy.rs @@ -1,68 +1,83 @@ -use serde::Deserialize; -use serde_json::Result; -use serde_json; -use std::fs::File; -use std::io::Seek; -use std::io::{BufRead, BufReader, BufWriter, SeekFrom}; -use std::path::PathBuf; -use std::str; -use std::time::Instant; - -#[derive(Debug, Deserialize)] -struct Coords { - x: f32, - y: f32, - z: f32, -} - -#[derive(Debug, Deserialize)] -struct Body { - name: String, - #[serde(rename = "type")] - body_type: String, - subType: String, - #[serde(rename = "distanceToArrival")] - distance: f32, -} - -#[derive(Debug, Deserialize)] -struct System { - coords: Coords, - name: String, - bodies: Vec, -} - -fn main() -> std::io::Result<()> { - better_panic::install(); - let mut buffer = String::new(); - let mut bz2_reader = std::process::Command::new("bzip2").args( - &["-d","-c",r#"E:\EDSM\galaxy.json.bz2"#] - ).stdout(std::process::Stdio::piped()) - .spawn() - .expect("Failed to spawn execute bzip2!"); - let mut reader = BufReader::new(bz2_reader.stdout.as_mut().expect("Failed to open stdout of child process")); - let mut count = 0; - while let Ok(n) = reader.read_line(&mut buffer) { - if n==0 { - break; - } - buffer = buffer - .trim() - .trim_end_matches(|c| c == ',') - .trim() - .to_string(); - if let Ok(sys) = serde_json::from_str::(&buffer) { - for b in &sys.bodies { - if b.body_type == "Star" { - count += 1; - if (count % 100_000) == 0 { - println!("{}: {:?}", count, b); - } - } - } - } - buffer.clear(); - } - println!("Total: {}", count); - Ok(()) -} +use serde::Deserialize; +use serde_json; +use std::io::{BufRead, BufReader}; +use std::str; + +#[derive(Debug, Deserialize)] +struct Coords { + x: f32, + y: f32, + z: f32, +} + +#[derive(Debug, Deserialize)] +struct Body { + name: String, + #[serde(rename = "type")] + body_type: String, + #[serde(rename = "subType")] + sub_type: String, + #[serde(rename = "distanceToArrival")] + distance: f32, +} + +#[derive(Debug, Deserialize)] +struct System { + coords: Coords, + name: String, + bodies: Vec, +} + + +/* +pub id: u32, +pub star_type: String, +pub system: String, +pub body: String, +pub mult: f32, +pub distance: u32, +pub x: f32, +pub y: f32, +pub z: f32, +*/ + + +fn main() -> std::io::Result<()> { + better_panic::install(); + let mut buffer = String::new(); + let mut bz2_reader = std::process::Command::new("bzip2") + .args(&["-d", "-c", r#"E:\EDSM\galaxy.json.bz2"#]) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("Failed to spawn execute bzip2!"); + let mut reader = BufReader::new( + bz2_reader + .stdout + .as_mut() + .expect("Failed to open stdout of child process"), + ); + let mut count = 0; + while let Ok(n) = reader.read_line(&mut buffer) { + if n == 0 { + break; + } + buffer = buffer + .trim() + .trim_end_matches(|c| c == ',') + .trim() + .to_string(); + if let Ok(sys) = serde_json::from_str::(&buffer) { + for b in &sys.bodies { + if b.body_type == "Star" { + count += 1; + if (count % 100_000) == 0 { + println!("{}: {:?}", count, b); + } + } + } + } + buffer.clear(); + } + println!("Total: {}", count); + Ok(()) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 1508f8a..cfc6679 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,221 +1,382 @@ -#![deny(warnings)] -mod common; -mod preprocess; -mod route; -use std::path::PathBuf; -use common::find_matches; -use pyo3::exceptions::*; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyList}; - -enum SysEntry { - ID(u32), - Name(String), -} - -impl From<&str> for SysEntry { - fn from(s: &str) -> SysEntry { - if let Ok(n) = s.parse() { - SysEntry::ID(n) - } else { - SysEntry::Name(s.to_owned()) - } - } -} - -fn resolve(entries: &Vec, path: &PathBuf) -> Result,String> { - let mut names: Vec = Vec::new(); - let mut ids: Vec = Vec::new(); - let mut ret: Vec = Vec::new(); - for ent in entries { - match ent { - SysEntry::Name(name) => names.push(name.to_owned()), - SysEntry::ID(id) => ids.push(*id), - } - }; - let name_ids = find_matches(path, names, false)?; - for ent in entries { - match ent { - SysEntry::Name(name) => { - let ent_res=name_ids.get(&name.to_owned()).ok_or(format!("System {} not found",name))?; - let sys=ent_res.1.as_ref().ok_or(format!("System {} not found",name))?; - if ent_res.0<0.75 { - println!("WARNING: {} match to {} with low confidence ({}%)",name,sys.system,ent_res.0*100.0); - } - ret.push(sys.id); - } - SysEntry::ID(id) => ret.push(*id), - } - } - return Ok(ret); -} - -#[pymodule] -pub fn _ed_lrr(_py: Python, m: &PyModule) -> PyResult<()> { - better_panic::install(); - /// preprocess(infile_systems, infile_bodies, outfile, callback) - /// -- - /// - /// Preprocess bodies.json and systemsWithCoordinates.json into stars.csv - #[pyfn(m, "preprocess")] - fn ed_lrr_preprocess( - py: Python<'static>, - infile_systems: String, - infile_bodies: String, - outfile: String, - callback: PyObject, - ) -> PyResult { - use preprocess::*; - let state = PyDict::new(py); - let state_dict = PyDict::new(py); - callback.call(py, (state_dict,), None).unwrap(); - let callback_wrapped = move |state: &PreprocessState| { - // println!("SEND: {:?}",state); - state_dict.set_item("file", state.file.clone())?; - state_dict.set_item("total", state.total)?; - state_dict.set_item("count", state.count)?; - state_dict.set_item("done", state.done)?; - state_dict.set_item("message", state.message.clone())?; - callback.call(py, (state_dict,), None) - }; - preprocess_files( - &PathBuf::from(infile_bodies), - &PathBuf::from(infile_systems), - &PathBuf::from(outfile), - &callback_wrapped, - ) - .unwrap(); - Ok(state.to_object(py)) - } - - ///find_sys(sys_names, sys_list_path) - /// -- - /// - /// Find system by name - #[pyfn(m, "find_sys")] - fn find_sys(py: Python, sys_names: Vec, sys_list: String) -> PyResult { - let path = PathBuf::from(sys_list); - match find_matches(&path, sys_names,false) { - Ok(vals) => { - let ret = PyDict::new(py); - for (key, (diff, sys)) in vals { - let ret_dict = PyDict::new(py); - if let Some(val) = sys { - let pos = PyList::new(py, val.pos.iter()); - ret_dict.set_item("star_type", val.star_type.clone())?; - ret_dict.set_item("system", val.system.clone())?; - ret_dict.set_item("body", val.body.clone())?; - ret_dict.set_item("distance", val.distance)?; - ret_dict.set_item("pos", pos)?; - ret_dict.set_item("id", val.id)?; - ret.set_item(key, (diff, ret_dict).to_object(py))?; - } - } - Ok(ret.to_object(py)) - } - Err(e) => Err(PyErr::new::(e)), - } - } - - /// route(hops, range, prune, mode, primary, permute, keep_first, keep_last, greedyness, precomp, path, num_workers, callback) - /// -- - /// - /// Compute a Route using the suplied parameters - #[pyfn(m, "route")] - #[allow(clippy::too_many_arguments)] - fn py_route( - py: Python<'static>, - hops: Vec<&str>, - range: f32, - prune: Option<(usize, f32)>, - mode: String, - primary: bool, - permute: bool, - keep_first: bool, - keep_last: bool, - greedyness: Option, - precomp: Option, - path: String, - num_workers: Option, - callback: PyObject, - ) -> PyResult { - use route::*; - let num_workers=num_workers.unwrap_or(1); - let mode = match Mode::parse(&mode) { - Ok(val) => val, - Err(e) => { - return Err(PyErr::new::(e)); - } - }; - let state_dict = PyDict::new(py); - { - let cb_res = callback.call(py, (state_dict,), None); - if cb_res.is_err() { - println!("Error: {:?}",cb_res); - } - } - let callback_wrapped = move |state: &SearchState| { - state_dict.set_item("mode", state.mode.clone())?; - state_dict.set_item("system", state.system.clone())?; - state_dict.set_item("body", state.body.clone())?; - state_dict.set_item("depth", state.depth)?; - state_dict.set_item("queue_size", state.queue_size)?; - state_dict.set_item("d_rem", state.d_rem)?; - state_dict.set_item("d_total", state.d_total)?; - state_dict.set_item("prc_done", state.prc_done)?; - state_dict.set_item("n_seen", state.n_seen)?; - state_dict.set_item("prc_seen", state.prc_seen)?; - state_dict.set_item("from", state.from.clone())?; - state_dict.set_item("to", state.to.clone())?; - let cb_res=callback.call(py, (state_dict,), None); - if cb_res.is_err() { - println!("Error: {:?}",cb_res); - } - cb_res - }; - let hops: Vec = hops.iter().cloned().map(SysEntry::from).collect(); - println!("Resolving systems..."); - let hops: Vec = match resolve(&hops,&PathBuf::from(&path)) { - Ok(ids) => ids, - Err(err_msg) => { - return Err(PyErr::new::(err_msg)); - } - }; - let opts = RouteOpts { - systems: hops, - range: Some(range), - file_path: PathBuf::from(path), - precomp_file: precomp.map(PathBuf::from), - callback: Box::new(callback_wrapped), - mode, - factor: greedyness, - precompute: false, - permute, - keep_first, - keep_last, - primary, - prune, - workers: num_workers - }; - let none = ().to_object(py); - match route(opts) { - Ok(Some(route)) => { - let hops = route.iter().map(|hop| { - let pos = PyList::new(py, hop.pos.iter()); - let elem = PyDict::new(py); - elem.set_item("star_type", hop.star_type.clone()).unwrap(); - elem.set_item("system", hop.system.clone()).unwrap(); - elem.set_item("body", hop.body.clone()).unwrap(); - elem.set_item("distance", hop.distance).unwrap(); - elem.set_item("pos", pos).unwrap(); - elem - }); - let lst = PyList::new(py, hops); - Ok(lst.to_object(py)) - } - Ok(None) => Ok(none), - Err(e) => Err(PyErr::new::(e)), - } - } - Ok(()) -} +// #![deny(warnings)] +mod common; +mod preprocess; +mod route; + +#[macro_use] +extern crate derivative; +use crate::common::SystemSerde; +use crate::common::{find_matches, SysEntry}; +use crate::route::{Router, SearchState}; +use pyo3::exceptions::*; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyTuple}; +use pyo3::PyObjectProtocol; +use std::path::PathBuf; + +/* + +pub id: u32, +pub star_type: String, +pub system: String, +pub body: String, +pub mult: f32, +pub distance: u32, +pub x: f32, +pub y: f32, +pub z: f32, +*/ + +impl SystemSerde { + fn fill_dict(&self, dict: &PyDict) -> PyResult<()> { + dict.clear(); + dict.set_item("id", self.id)?; + dict.set_item("star_type", self.star_type.clone())?; + dict.set_item("system", self.system.clone())?; + dict.set_item("body", self.body.clone())?; + dict.set_item("mult", self.mult)?; + dict.set_item("distance", self.distance)?; + dict.set_item("x", self.x)?; + dict.set_item("y", self.y)?; + dict.set_item("z", self.z)?; + return Ok(()); + } +} + +#[pyclass(dict)] +#[derive(Derivative)] +#[derivative(Debug)] +#[text_signature = "(callback, workers, /)"] +struct PyRouter { + router: Router, + stars_path: String, +} + +#[pymethods] +impl PyRouter { + #[new] + #[args(callback = "None")] + fn new(callback: Option, py: Python<'static>) -> PyResult { + let cb_func = move |state: &SearchState| { + return match callback.as_ref() { + Some(cb) => cb.call(py, (state.clone(),), None), + None => Ok(py.None()), + }; + }; + + let router = match Router::new(Box::new(cb_func)) { + Ok(router) => router, + Err(err_msg) => { + return Err(PyErr::new::(err_msg)); + } + }; + let ret = PyRouter { + router, + stars_path: String::from(""), + }; + Ok(ret) + } + + #[args(filter_func = "None")] + #[text_signature = "(path, /)"] + fn load(&mut self, path: String, py: Python<'static>) -> PyResult { + self.stars_path = path; + return Ok(py.None()); + } + + #[args(greedyness = "0.5", num_workers = "0", beam_width = "0")] + #[text_signature = "(hops, range, greedyness, beam_width, num_workers, /)"] + fn route( + &mut self, + hops: &PyList, + range: f32, + greedyness: f32, + beam_width: usize, + num_workers: usize, + py: Python, + ) -> PyResult { + let route_res = self.router.load(&PathBuf::from(self.stars_path.clone())); + if let Err(err_msg) = route_res { + return Err(PyErr::new::(err_msg)); + }; + let mut sys_entries: Vec = Vec::new(); + for hop in hops { + if let Ok(id) = hop.extract() { + sys_entries.push(SysEntry::ID(id)); + } else { + sys_entries.push(SysEntry::parse(hop.extract()?)); + } + } + println!("Resolving systems..."); + let ids: Vec = match resolve(&sys_entries, &self.router.path) { + Ok(ids) => ids, + Err(err_msg) => { + return Err(PyErr::new::(err_msg)); + } + }; + match self + .router + .computer_route(&ids, range, greedyness, beam_width, num_workers) + { + // TODO: return list of dicts (or objects) + Ok(route) => Ok(route.len().to_object(py)), + Err(err_msg) => Err(PyErr::new::(err_msg)), + } + } + + #[args(hops = "*")] + #[text_signature = "(sys_1, sys_2, ..., /)"] + fn resolve_systems(&self, hops: &PyTuple, py: Python) -> PyResult { + let mut sys_entries: Vec = Vec::new(); + for hop in hops { + if let Ok(id) = hop.extract() { + sys_entries.push(SysEntry::ID(id)); + } else { + sys_entries.push(SysEntry::parse(hop.extract()?)); + } + } + println!("Resolving systems..."); + let ids: Vec = match resolve(&sys_entries, &PathBuf::from(self.stars_path.clone())) { + Ok(ids) => ids, + Err(err_msg) => { + return Err(PyErr::new::(err_msg)); + } + }; + let ret: Vec<(_, u32)> = hops.into_iter().zip(ids.into_iter()).collect(); + Ok(PyDict::from_sequence(py, ret.to_object(py))?.to_object(py)) + } + + #[staticmethod] + fn preprocess_edsm() -> PyResult<()> { + unimplemented!() + } + + #[staticmethod] + fn preprocess_galaxy() -> PyResult<()> { + unimplemented!() + } +} + +#[pyproto] +impl PyObjectProtocol for PyRouter { + fn __str__(&self) -> PyResult { + Ok(format!("{:?}", &self)) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("{:?}", &self)) + } +} + +fn resolve(entries: &Vec, path: &PathBuf) -> Result, String> { + let mut names: Vec = Vec::new(); + let mut ids: Vec = Vec::new(); + let mut ret: Vec = Vec::new(); + for ent in entries { + match ent { + SysEntry::Name(name) => names.push(name.to_owned()), + SysEntry::ID(id) => ids.push(*id), + } + } + if !path.exists() { + return Err(format!( + "Source file \"{:?}\" does not exist!", + path.display() + )); + } + + let name_ids = find_matches(path, names, false)?; + for ent in entries { + match ent { + SysEntry::Name(name) => { + let ent_res = name_ids + .get(&name.to_owned()) + .ok_or(format!("System {} not found", name))?; + let sys = ent_res + .1 + .as_ref() + .ok_or(format!("System {} not found", name))?; + if ent_res.0 < 0.75 { + println!( + "WARNING: {} match to {} with low confidence ({:.2}%)", + name, + sys.system, + ent_res.0 * 100.0 + ); + } + ret.push(sys.id); + } + SysEntry::ID(id) => ret.push(*id), + } + } + return Ok(ret); +} + +#[pymodule] +pub fn _ed_lrr(_py: Python, m: &PyModule) -> PyResult<()> { + better_panic::install(); + + m.add_class::()?; + + Ok(()) +} + +/* + +/// Preprocess bodies.json and systemsWithCoordinates.json into stars.csv + #[pyfn(m, "preprocess")] + #[text_signature = "(infile_systems, infile_bodies, outfile, callback, /)"] + fn ed_lrr_preprocess( + py: Python<'static>, + infile_systems: String, + infile_bodies: String, + outfile: String, + callback: PyObject, + ) -> PyResult { + use preprocess::*; + let state = PyDict::new(py); + let state_dict = PyDict::new(py); + callback.call(py, (state_dict,), None).unwrap(); + let callback_wrapped = move |state: &PreprocessState| { + // println!("SEND: {:?}",state); + state_dict.set_item("file", state.file.clone())?; + state_dict.set_item("total", state.total)?; + state_dict.set_item("count", state.count)?; + state_dict.set_item("done", state.done)?; + state_dict.set_item("message", state.message.clone())?; + callback.call(py, (state_dict,), None) + }; + preprocess_files( + &PathBuf::from(infile_bodies), + &PathBuf::from(infile_systems), + &PathBuf::from(outfile), + &callback_wrapped, + ) + .unwrap(); + Ok(state.to_object(py)) + } + + /// Find system by name + #[pyfn(m, "find_sys")] + #[text_signature = "(sys_names, sys_list_path, /)"] + fn find_sys(py: Python, sys_names: Vec, sys_list: String) -> PyResult { + let path = PathBuf::from(sys_list); + match find_matches(&path, sys_names, false) { + Ok(vals) => { + let ret = PyDict::new(py); + for (key, (diff, sys)) in vals { + let ret_dict = PyDict::new(py); + if let Some(val) = sys { + let pos = PyList::new(py, val.pos.iter()); + ret_dict.set_item("star_type", val.star_type.clone())?; + ret_dict.set_item("system", val.system.clone())?; + ret_dict.set_item("body", val.body.clone())?; + ret_dict.set_item("distance", val.distance)?; + ret_dict.set_item("pos", pos)?; + ret_dict.set_item("id", val.id)?; + ret.set_item(key, (diff, ret_dict).to_object(py))?; + } + } + Ok(ret.to_object(py)) + } + Err(e) => Err(PyErr::new::(e)), + } + } + + /// Compute a Route using the suplied parameters + #[pyfn(m, "route")] + #[text_signature = "(hops, range, mode, primary, permute, keep_first, keep_last, greedyness, precomp, path, num_workers, callback, /)"] + #[allow(clippy::too_many_arguments)] + fn py_route( + py: Python<'static>, + hops: Vec<&str>, + range: f32, + mode: String, + primary: bool, + permute: bool, + keep_first: bool, + keep_last: bool, + greedyness: Option, + precomp: Option, + path: String, + num_workers: Option, + callback: PyObject, + ) -> PyResult { + use route::*; + let num_workers = num_workers.unwrap_or(1); + let mode = match Mode::parse(&mode) { + Ok(val) => val, + Err(e) => { + return Err(PyErr::new::(e)); + } + }; + let state_dict = PyDict::new(py); + { + let cb_res = callback.call(py, (state_dict,), None); + if cb_res.is_err() { + println!("Error: {:?}", cb_res); + } + } + let callback_wrapped = move |state: &SearchState| { + state_dict.set_item("mode", state.mode.clone())?; + state_dict.set_item("system", state.system.clone())?; + state_dict.set_item("body", state.body.clone())?; + state_dict.set_item("depth", state.depth)?; + state_dict.set_item("queue_size", state.queue_size)?; + state_dict.set_item("d_rem", state.d_rem)?; + state_dict.set_item("d_total", state.d_total)?; + state_dict.set_item("prc_done", state.prc_done)?; + state_dict.set_item("n_seen", state.n_seen)?; + state_dict.set_item("prc_seen", state.prc_seen)?; + state_dict.set_item("from", state.from.clone())?; + state_dict.set_item("to", state.to.clone())?; + let cb_res = callback.call(py, (state_dict,), None); + if cb_res.is_err() { + println!("Error: {:?}", cb_res); + } + cb_res + }; + let hops: Vec = (hops.iter().map(|v| SysEntry::from_str(&v)).collect::,_>>())?; + println!("Resolving systems..."); + let hops: Vec = match resolve(&hops, &PathBuf::from(&path)) { + Ok(ids) => ids, + Err(err_msg) => { + return Err(PyErr::new::(err_msg)); + } + }; + let opts = RouteOpts { + systems: hops, + range: Some(range), + file_path: PathBuf::from(path), + precomp_file: precomp.map(PathBuf::from), + callback: Box::new(callback_wrapped), + mode, + factor: greedyness, + precompute: false, + permute, + keep_first, + keep_last, + primary, + workers: num_workers, + }; + match route(opts) { + Ok(Some(route)) => { + let hops = route.iter().map(|hop| { + let pos = PyList::new(py, hop.pos.iter()); + let elem = PyDict::new(py); + elem.set_item("star_type", hop.star_type.clone()).unwrap(); + elem.set_item("system", hop.system.clone()).unwrap(); + elem.set_item("body", hop.body.clone()).unwrap(); + elem.set_item("distance", hop.distance).unwrap(); + elem.set_item("pos", pos).unwrap(); + elem + }); + let lst = PyList::new(py, hops); + Ok(lst.to_object(py)) + } + Ok(None) => Ok(py.None()), + Err(e) => Err(PyErr::new::(e)), + } + } + +*/ diff --git a/rust/src/preprocess.rs b/rust/src/preprocess.rs index 5363f2f..bb96d0c 100644 --- a/rust/src/preprocess.rs +++ b/rust/src/preprocess.rs @@ -116,7 +116,7 @@ fn process_systems( ret } -fn build_index(path: &PathBuf) -> std::io::Result<()> { +pub fn build_index(path: &PathBuf) -> std::io::Result<()> { let mut wtr = BufWriter::new(File::create(path.with_extension("idx"))?); let mut idx: Vec = Vec::new(); let mut records = (csv::Reader::from_path(path)?).into_deserialize::(); diff --git a/rust/src/route.rs b/rust/src/route.rs index 3b5aab0..7369a9f 100644 --- a/rust/src/route.rs +++ b/rust/src/route.rs @@ -1,8 +1,9 @@ use crate::common::{System, SystemSerde, TreeNode}; use core::cmp::Ordering; -use crossbeam_channel::{ - bounded, unbounded, Receiver, SendError, Sender, TryIter, -}; +use crossbeam_channel::{bounded, unbounded, Receiver, SendError, Sender, TryIter}; +use derivative::Derivative; +use dict_derive::IntoPyObject; +use crate::preprocess::build_index; use fnv::{FnvHashMap, FnvHashSet}; use humantime::format_duration; use permutohedron::LexicalPermutation; @@ -19,9 +20,25 @@ use std::thread; use std::thread::JoinHandle; use std::time::Instant; -const STATUS_INVERVAL: u128 = 500; //ms +const STATUS_INVERVAL: u128 = 5000; //ms -#[derive(Debug)] +struct Weight { + // TODO: implement + star_type: FnvHashMap, + dist_from_start: f32, + dist_to_goal: f32, + dist_to_point: Vec<(f32, [f32; 3])>, +} + +impl Weight { + fn calc(&self, node: &TreeNode, dst: &TreeNode, src: &TreeNode) -> f32 { + let d_start = dist(&node.pos, &src.pos); + let d_goal = dist(&node.pos, &dst.pos); + return 0.0; + } +} + +#[derive(Debug, Clone, IntoPyObject)] pub struct SearchState { pub mode: String, pub system: String, @@ -48,7 +65,6 @@ pub struct RouteOpts { pub keep_last: bool, pub factor: Option, pub mode: Mode, - pub prune: Option<(usize, f32)>, pub systems: Vec, pub callback: Box PyResult>, pub workers: usize, @@ -58,9 +74,20 @@ pub struct RouteOpts { #[allow(non_camel_case_types)] pub enum Mode { BFS, - BFS_old, + BiDir, // TODO: implement bidirectional BFS Greedy, AStar, + Shortest, // TODO: A-Star with distance as weight +} + + +#[derive(Debug)] +#[allow(non_camel_case_types)] +pub enum PrecomputeMode { + Full, + Route_From, + Route_To, + None } impl Mode { @@ -68,9 +95,9 @@ impl Mode { let s = s.to_lowercase(); match s.as_ref() { "bfs" => Ok(Mode::BFS), - "bfs_old" => Ok(Mode::BFS_old), "greedy" => Ok(Mode::Greedy), "astar" => Ok(Mode::AStar), + "bidir" => Ok(Mode::BiDir), val => Err(format!("Invalid Mode: {}", val)), } } @@ -162,9 +189,25 @@ pub struct LineCache { impl LineCache { pub fn new(path: &PathBuf) -> Option>> { + //TODO: Verify match between index file and csv file let idx_path = path.with_extension("idx"); - let cache = - bincode::deserialize_from(&mut BufReader::new(File::open(idx_path).ok()?)).ok()?; + if !idx_path.exists() { + eprintln!("No index found for {:?}, building...", path); + build_index(path).map_err(|e| { + eprintln!("Error creating index for {:?}: {}", path, e); + }).ok()?; + } + let cache = bincode::deserialize_from(&mut BufReader::new( + File::open(idx_path) + .map_err(|e| { + eprintln!("Error opening index for {:?}: {}", path, e); + }) + .ok()?, + )) + .map_err(|e| { + eprintln!("Reading index for {:?}: {}", path, e); + }) + .ok()?; let reader = csv::Reader::from_path(path).ok()?; let ret = Self { reader, cache }; Some(Arc::new(Mutex::new(ret))) @@ -206,6 +249,7 @@ struct WorkUnit { node: TreeNode, depth: usize, parent_id: Option, + range: f32, } #[derive(Debug)] @@ -218,30 +262,8 @@ enum WorkerSet { }, } -fn neighbor_worker( - tree: &LargeNodeRTree, - search_range: f32, - rx: Receiver>, - tx: Sender, -) { - while let Ok(Some(unit)) = rx.recv() { - let range=search_range*unit.node.mult; - tree.locate_within_distance(unit.node.pos, range*range) - .cloned() - .for_each(|nb| { - let wu = WorkUnit { - node: nb, - depth: unit.depth + 1, - parent_id: Some(unit.node.id), - }; - tx.send(wu).unwrap(); - }); - } - drop(tx); -} - impl WorkerSet { - fn new(tree: Arc>, search_range: f32, num_workers: usize) -> Self { + fn new(tree: Arc>, num_workers: usize) -> Self { if num_workers == 0 { return WorkerSet::Empty; } @@ -254,7 +276,7 @@ impl WorkerSet { let tx = result_tx.clone(); let tree = tree.clone(); move || { - neighbor_worker(&tree, search_range, rx, tx); + Self::work(&tree, rx, tx); } }) }) @@ -266,29 +288,62 @@ impl WorkerSet { }; } - fn close(self) -> Result<(),String> { - if let WorkerSet::Workers{mut handles,tx,rx} = self { - let t_start=Instant::now(); + fn work(tree: &LargeNodeRTree, rx: Receiver>, tx: Sender) { + while let Ok(Some(unit)) = rx.recv() { + let range = unit.range * unit.node.mult; + tree.locate_within_distance(unit.node.pos, range * range) + .cloned() + .for_each(|nb| { + let wu = WorkUnit { + node: nb, + depth: unit.depth + 1, + parent_id: Some(unit.node.id), + range: unit.range, + }; + tx.send(wu).unwrap(); + }); + } + drop(tx); + } + + fn resize(self, tree: Arc>, num: usize) -> Result { + self.close()?; + return Ok(WorkerSet::new(tree.clone(), num)); + } + + // fn replace(self, tree: Arc>) -> Result { + // let num=self.num(); + // return self.resize(tree.clone(),num); + // } + + fn close(self) -> Result<(), String> { + if let WorkerSet::Workers { + mut handles, + tx, + rx, + } = self + { + let t_start = Instant::now(); loop { - if (rx.len()==0) && (tx.len()==0) { + if (rx.len() == 0) && (tx.len() == 0) { break; } rx.try_iter().for_each(|_| {}); - }; + } for _ in &handles { match tx.send(None) { - Ok(_) => {}, + Ok(_) => {} Err(e) => { - return Err(format!("{:?}",e)); + return Err(format!("{:?}", e)); } } - }; + } drop(tx); - while let Some(handle)= handles.pop() { + while let Some(handle) = handles.pop() { handle.join().unwrap(); } drop(rx); - println!("cleared in {}",format_duration(t_start.elapsed())); + println!("cleared in {}", format_duration(t_start.elapsed())); } return Ok(()); } @@ -296,12 +351,12 @@ impl WorkerSet { fn queue_size(&self) -> usize { match self { WorkerSet::Empty => 0, - WorkerSet::Workers{rx,tx,..} => tx.len() + rx.len() + WorkerSet::Workers { rx, tx, .. } => tx.len() + rx.len(), } } fn queue_empty(&self) -> bool { - return self.queue_size()==0; + return self.queue_size() == 0; } fn send(&self, wu: WorkUnit) -> Result<(), SendError>> { @@ -331,14 +386,10 @@ impl WorkerSet { // impl Iterator - fn iter(&self) -> Result,String> { + fn iter(&self) -> Result, String> { match self { - WorkerSet::Empty => { - Err("can't iterate on empty WorkerSet".to_string()) - } - WorkerSet::Workers { rx, .. } => { - Ok(rx.try_iter()) - } + WorkerSet::Empty => Err("can't iterate on empty WorkerSet".to_string()), + WorkerSet::Workers { rx, .. } => Ok(rx.try_iter()), } } @@ -351,28 +402,43 @@ impl WorkerSet { // } } +#[derive(Derivative)] +#[derivative(Debug)] pub struct Router { + #[derivative(Debug = "ignore")] tree: Arc>, + #[derivative(Debug = "ignore")] scoopable: FnvHashSet, + #[derivative(Debug = "ignore")] pub route_tree: Option>, + #[derivative(Debug = "ignore")] pub cache: Option>>, - range: f32, - primary_only: bool, - path: PathBuf, - prune: Option<(usize, f32)>, + pub path: PathBuf, + #[derivative(Debug = "ignore")] workers: WorkerSet, - callback: Box PyResult>, + #[derivative(Debug = "ignore")] + pub callback: Box PyResult>, } impl Router { - pub fn new( - path: &PathBuf, - range: f32, - prune: Option<(usize, f32)>, - primary_only: bool, - num_workers: usize, - callback: Box PyResult>, - ) -> Result { + pub fn new(callback: Box PyResult>) -> Result { + let ret = Self { + tree: Arc::new(LargeNodeRTree::default()), + scoopable: FnvHashSet::default(), + route_tree: None, + cache: None, + callback, + workers: WorkerSet::Empty, + path: PathBuf::from(""), + }; + Ok(ret) + } + + pub fn load(&mut self, path: &PathBuf) -> Result<(), String> { + if self.path == path.to_path_buf() { + return Ok(()); + } + self.tree = Arc::new(LargeNodeRTree::default()); // clear R*-Tree let mut scoopable = FnvHashSet::default(); let mut reader = match csv::ReaderBuilder::new().from_path(path) { Ok(rdr) => rdr, @@ -386,9 +452,9 @@ impl Router { .deserialize::() .filter_map(|res| { let sys = res.expect("Failed to read"); - if primary_only && sys.distance != 0 { + if sys.distance != 0 { return None; - }; + } if sys.mult > 1.0f32 { scoopable.insert(sys.id); } else { @@ -408,37 +474,28 @@ impl Router { format_duration(t_load.elapsed()) ); let t_load = Instant::now(); - let tree = Arc::new(LargeNodeRTree::bulk_load_with_params(systems)); - let workers = WorkerSet::new(Arc::clone(&tree), range, num_workers); + self.tree = Arc::new(LargeNodeRTree::bulk_load_with_params(systems)); + self.path = path.to_path_buf(); + self.cache = LineCache::new(&path.to_path_buf()); + self.scoopable = scoopable; println!("R*-Tree built in {}", format_duration(t_load.elapsed())); - let ret = Self { - tree, - scoopable, - route_tree: None, - range, - primary_only, - cache: LineCache::new(path), - path: path.clone(), - callback, - prune, - workers, - }; - Ok(ret) + Ok(()) + } + + pub fn start_workers(&mut self, num: usize) -> Result<(), String> { + let mut w = WorkerSet::Empty; + std::mem::swap(&mut self.workers, &mut w); + self.workers = w.resize(self.tree.clone(), num)?; + Ok(()) } pub fn from_file( filename: &PathBuf, callback: Box PyResult>, - ) -> Result<(PathBuf, Self), String> { + ) -> Result<(PathBuf, f32, Self), String> { let mut reader = BufReader::new(match File::open(&filename) { Ok(fh) => fh, - Err(e) => { - return Err(format!( - "Error opening file {}: {}", - filename.display(), - e - )) - } + Err(e) => return Err(format!("Error opening file {}: {}", filename.display(), e)), }); println!("Loading {}", filename.display()); let (primary, range, file_hash, path, route_tree): ( @@ -449,13 +506,7 @@ impl Router { FnvHashMap, ) = match bincode::deserialize_from(&mut reader) { Ok(res) => res, - Err(e) => { - return Err(format!( - "Error loading file {}: {}", - filename.display(), - e - )) - } + Err(e) => return Err(format!("Error loading file {}: {}", filename.display(), e)), }; if hash_file(&path) != file_hash { return Err("File hash mismatch!".to_string()); @@ -463,16 +514,14 @@ impl Router { let cache = LineCache::new(&path); Ok(( path.clone(), + range, Self { tree: Arc::new(RTree::default()), scoopable: FnvHashSet::default(), route_tree: Some(route_tree), - range, cache, - primary_only: primary, path, callback, - prune: None, workers: WorkerSet::Empty, }, )) @@ -482,8 +531,16 @@ impl Router { self.tree.locate_within_distance(*center, radius * radius) } - fn neighbours(&self, node: &TreeNode) -> impl Iterator { - self.points_in_sphere(&node.pos, node.mult * self.range) + fn neighbours(&self, node: &TreeNode, range: f32) -> impl Iterator { + self.points_in_sphere(&node.pos, node.mult * range) + } + + fn neighbours_r(&self, node: &TreeNode, range: f32) -> impl Iterator { + let pos = node.pos.clone(); + self.points_in_sphere(&node.pos, range * 4.0) + .filter(move |s| { + return s.dist2(&pos) < (range * s.mult) * (range * s.mult); + }) } fn valid(&self, id: u32) -> bool { @@ -492,11 +549,13 @@ impl Router { } pub fn best_multiroute( - &self, + &mut self, waypoints: &[System], + range: f32, keep: (bool, bool), - mode: Mode, factor: f32, + beam_width: usize, + num_workers: usize, ) -> Result, String> { let mut best_score: f32 = std::f32::MAX; let mut waypoints = waypoints.to_owned(); @@ -529,16 +588,33 @@ impl Router { } } println!("Best permutation: {:?}", best_permutation_waypoints); - self.multiroute(&best_permutation_waypoints, mode, factor) + self.multiroute( + &best_permutation_waypoints, + range, + factor, + beam_width, + num_workers, + ) } pub fn multiroute( - &self, + &mut self, waypoints: &[System], - mode: Mode, + range: f32, factor: f32, + beam_width: usize, + num_workers: usize, ) -> Result, String> { + if self.tree.size() == 0 { + return Err("No Systems loaded, pleased load some with the 'load' method!".to_string()); + } + if factor < 0.0 || factor > 1.0 { + return Err("Factor needs to be between 0.0 (BFS) and 1.0 (Greedy Search)".to_string()); + } let mut route: Vec = Vec::new(); + if factor == 0.0 { + self.start_workers(num_workers)?; + } for pair in waypoints.windows(2) { match pair { [src, dst] => { @@ -549,16 +625,11 @@ impl Router { ); println!( "Jump Range: {} Ly, Distance: {} Ly, Estimated Jumps: {}", - self.range, + range, d_total, - d_total / self.range + d_total / range ); - let block = match mode { - Mode::BFS => self.route_bfs(&src, &dst)?, - Mode::BFS_old => self.route_bfs_old(&src, &dst)?, - Mode::Greedy => self.route_greedy(&src, &dst)?, - Mode::AStar => self.route_astar(&src, &dst, factor)?, - }; + let block = self.route_astar(&src, &dst, factor, beam_width, range)?; if route.is_empty() { for sys in block.iter() { route.push(sys.clone()); @@ -582,9 +653,14 @@ impl Router { src: &System, dst: &System, factor: f32, + beam_width: usize, + range: f32, ) -> Result, String> { if factor == 0.0 { - return self.route_bfs(src, dst); + return self.route_bfs(src, dst, range, beam_width); + } + if factor == 1.0 { + return self.route_greedy(src, dst, range, beam_width); } let src_name = src.system.clone(); let dst_name = dst.system.clone(); @@ -614,8 +690,8 @@ impl Router { let mut found = false; let mut queue: Vec<(usize, usize, TreeNode)> = Vec::new(); queue.push(( - 0, // depth - (start_sys.distp(goal_sys) / self.range) as usize, // h + 0, // depth + (start_sys.distp(goal_sys) / range) as usize, // h start_sys.to_node(), )); seen.insert(start_sys.id); @@ -647,7 +723,7 @@ impl Router { break; } queue.extend( - self.neighbours(&node) + self.neighbours(&node, range) .filter(|nb| (self.valid(nb.id) || (nb.id == goal_sys.id))) .filter(|nb| seen.insert(nb.id)) .map(|nb| { @@ -656,14 +732,14 @@ impl Router { if d_g < d_rem { d_rem = d_g; } - (depth + 1, (d_g / self.range) as usize, *nb) + (depth + 1, (d_g / (range * 4.0)) as usize, *nb) }), ); queue.sort_by(|b, a| { let (a_0, a_1) = (a.0 as f32, a.1 as f32); let (b_0, b_1) = (b.0 as f32, b.1 as f32); - let v_a = a_0 + a_1 * factor; - let v_b = b_0 + b_1 * factor; + let v_a = a_0 * (1.0 - factor) + a_1 * factor; + let v_b = b_0 * (1.0 - factor) + b_1 * factor; fcmp(v_a, v_b) }); // queue.reverse(); @@ -693,7 +769,13 @@ impl Router { Ok(v) } - pub fn route_greedy(&self, src: &System, dst: &System) -> Result, String> { + pub fn route_greedy( + &self, + src: &System, + dst: &System, + range: f32, + beam_width: usize, + ) -> Result, String> { let src_name = src.system.clone(); let dst_name = dst.system.clone(); let start_sys = src; @@ -750,7 +832,7 @@ impl Router { break; } queue.extend( - self.neighbours(&node) + self.neighbours(&node, range) .filter(|nb| (self.valid(nb.id) || (nb.id == goal_sys.id))) .filter(|nb| seen.insert(nb.id)) .map(|nb| { @@ -762,8 +844,7 @@ impl Router { (d_g, depth + 1, nb.clone()) }), ); - queue.sort_by(|a, b| fcmp(a.0, b.0).then(a.1.cmp(&b.1))); - queue.reverse(); + queue.sort_by(|a, b| fcmp(b.0, a.0).then(b.1.cmp(&a.1))); } if queue.is_empty() { break; @@ -787,7 +868,18 @@ impl Router { Ok(v) } - pub fn precompute(&mut self, src: &System) -> Result<(), String> { + pub fn precompute_all(&mut self, range:f32) -> Result<(),String> { + // TODO: implement all-pairs shortest path based on optimized BFS + unimplemented!(); + } + + pub fn precompute_to(&mut self, dst: &System, range: f32) -> Result<(), String> { + // TODO: -> precompute to + unimplemented!(); + } + + pub fn precompute(&mut self, src: &System, range: f32) -> Result<(), String> { + // TODO: -> precompute from let total = self.tree.size() as f32; let t_start = Instant::now(); let mut prev = FnvHashMap::default(); @@ -809,7 +901,7 @@ impl Router { std::io::stdout().flush().unwrap(); while let Some((d, sys)) = queue.pop_front() { queue_next.extend( - self.neighbours(&sys) + self.neighbours(&sys, range) // .filter(|&nb| self.valid(nb)) .filter(|&nb| seen.insert(nb.id)) .map(|nb| { @@ -822,17 +914,23 @@ impl Router { depth += 1; } self.route_tree = Some(prev); + let file_hash = hash_file(&self.path); + let file_hash_hex = file_hash + .iter() + .map(|v| format!("{:02x}", v)) + .collect::>() + .join(""); let ofn = format!( - "{}_{}{}.router", + "{}_{}_{}.router", src.system.replace("*", "").replace(" ", "_"), - self.range, - if self.primary_only { "_primary" } else { "" } + range, + file_hash_hex ); let mut out_fh = BufWriter::new(File::create(&ofn).unwrap()); let data = ( - self.primary_only, - self.range, - hash_file(&self.path), + self.tree.size(), + range, + file_hash, self.path.clone(), self.route_tree.as_ref().unwrap(), ); @@ -913,6 +1011,9 @@ impl Router { } pub fn route_to(&self, dst: &System) -> Result, String> { + if self.route_tree.is_none() { + return Err("Can't computer route without a precomputed route-tree".to_owned()); + } let prev = self.route_tree.as_ref().unwrap(); if !prev.contains_key(&dst.id) { return Err(format!("System-ID {} not found", dst.id).to_string()); @@ -943,9 +1044,15 @@ impl Router { Ok(v) } - pub fn route_bfs(&self, start_sys: &System, goal_sys: &System) -> Result, String> { + pub fn route_bfs( + &self, + start_sys: &System, + goal_sys: &System, + range: f32, + beam_width: usize, + ) -> Result, String> { if self.workers.is_empty() { - return self.route_bfs_old(start_sys, goal_sys); + return self.route_bfs_serial(start_sys, goal_sys, range, beam_width); } println!("Running BFS with {} worker(s)", self.workers.num()); let t_start = Instant::now(); @@ -959,8 +1066,9 @@ impl Router { node: start_sys.to_node(), parent_id: None, depth: 0, + range, }; - if wu.node.id==goal_sys.id { + if wu.node.id == goal_sys.id { return Ok(vec![goal_sys.clone()]); } let mut found = false; @@ -968,7 +1076,7 @@ impl Router { let d_total = dist(&start_sys.pos, &goal_sys.pos); let mut d_rem = d_total; let mut state = SearchState { - mode: "BFS".into(), + mode: format!("BFS_parallel({})", self.workers.num()), depth: 0, queue_size: 0, d_rem, @@ -1019,14 +1127,18 @@ impl Router { } }) .collect(); - if nbs.is_empty() && workers.queue_empty() && seen.len()>1 { + if nbs.is_empty() && workers.queue_empty() && seen.len() > 1 { break; } - // nbs.sort_by(|a, b| { - // let d_a=a.node.dist2(&goal_sys.pos); - // let d_b=b.node.dist2(&goal_sys.pos); - // return a.depth.cmp(&b.depth).then(fcmp(d_a,d_b)); - // }); + if beam_width != 0 && nbs.len() > beam_width { + nbs.sort_by(|a, b| { + let d_a = a.node.dist2(&goal_sys.pos); + let d_b = b.node.dist2(&goal_sys.pos); + return fcmp(d_a, d_b); + }); + nbs = nbs.iter().take(beam_width).cloned().collect(); + } + while let Some(wu) = nbs.pop() { if let Some(parent_id) = wu.parent_id { prev.insert(wu.node.id, parent_id); @@ -1065,25 +1177,24 @@ impl Router { Ok(v) } - pub fn route_bfs_old( + pub fn route_bfs_serial( &self, start_sys: &System, goal_sys: &System, + range: f32, + beam_width: usize, ) -> Result, String> { - if start_sys.id==goal_sys.id { + if start_sys.id == goal_sys.id { return Ok(vec![goal_sys.clone()]); } let t_start = Instant::now(); println!("Running BFS"); - if self.prune.is_some() { - println!("WARNING: search pruning is not implemented!"); - } let src_name = start_sys.system.clone(); let dst_name = goal_sys.system.clone(); let d_total = dist(&start_sys.pos, &goal_sys.pos); let mut d_rem = d_total; let mut state = SearchState { - mode: "BFS_old".into(), + mode: "BFS_serial".into(), depth: 0, queue_size: 0, d_rem, @@ -1120,7 +1231,7 @@ impl Router { state.d_rem = d_rem; state.n_seen = seen.len(); state.prc_seen = ((seen.len() * 100) as f32) / total; - { + if !queue.is_empty() { let s = queue.get(0).unwrap().get(&self).unwrap(); state.system = s.system.clone(); state.body = s.body.clone(); @@ -1134,7 +1245,7 @@ impl Router { t_last = Instant::now(); } let valid_nbs = self - .neighbours(&node) + .neighbours(&node, range) .filter(|nb| (self.valid(nb.id) || (nb.id == goal_sys.id))) .filter(|nb| seen.insert(nb.id)) .map(|nb| { @@ -1147,7 +1258,25 @@ impl Router { }); queue_next.extend(valid_nbs); } - std::mem::swap(&mut queue, &mut queue_next); + + if beam_width != 0 { + let mut q = Vec::new(); + while let Some(v) = queue_next.pop_front() { + q.push(v); + } + q.sort_by(|a, b| { + let d_a = a.dist2(&goal_sys.pos); + let d_b = b.dist2(&goal_sys.pos); + return fcmp(d_a, d_b); + }); + queue.clear(); + for v in q.iter().take(beam_width).cloned() { + queue.push_back(v); + } + queue_next.clear(); + } else { + std::mem::swap(&mut queue, &mut queue_next); + } if found { break; } @@ -1179,67 +1308,21 @@ impl Router { } } -pub fn route(opts: RouteOpts) -> Result>, String> { - // TODO: implement pruning (check if dist to target improved by at least $n*jump_range$ Ly in the last $m$ steps) - if opts.systems.is_empty() { - if opts.precomp_file.is_some() { - return Err("Error: Please specify exatly one system".to_owned()); - } else if opts.precompute { - return Err("Error: Please specify at least one system".to_owned()); - } else { - return Err("Error: Please specify at least two systems".to_owned()); - }; +impl Router { + pub fn computer_route( + &mut self, + sys_ids: &[u32], + range: f32, + factor: f32, + beam_width: usize, + num_workers: usize, + ) -> Result, String> { + let id_map = self.get_systems_by_ids(sys_ids)?; + let hops: Vec = sys_ids + .iter() + .map(|id| id_map.get(&id).unwrap()) + .cloned() + .collect(); + self.multiroute(&hops, range, factor, beam_width, num_workers) } - let mut router: Router = if opts.precomp_file.is_some() { - let (_, ret) = - Router::from_file(&opts.precomp_file.clone().unwrap(), Box::new(opts.callback))?; - ret - } else if opts.range.is_some() { - Router::new( - &opts.file_path, - opts.range.unwrap(), - opts.prune, - opts.primary, - opts.workers, - Box::new(opts.callback), - )? - } else { - Router::new( - &opts.file_path, - opts.range.unwrap(), - opts.prune, - opts.primary, - opts.workers, - opts.callback, - )? - }; - let mut systems: Vec = Vec::new(); - let sys_ht = router.get_systems_by_ids(&opts.systems)?; - for sys in opts.systems { - systems.push(sys_ht.get(&sys).unwrap().clone()); - } - if opts.precompute { - for sys in systems { - router.route_tree = None; - router.precompute(&sys)?; - } - return Ok(None); - } - let route = if router.route_tree.is_some() { - router.route_to(systems.first().unwrap())? - } else if opts.permute { - router.best_multiroute( - &systems, - (opts.keep_first, opts.keep_last), - opts.mode, - opts.factor.unwrap_or(1.0), - )? - } else { - router.multiroute(&systems, opts.mode, opts.factor.unwrap_or(1.0))? - }; - router.workers.close()?; - if route.is_empty() { - return Err("No route found!".to_string()); - } - Ok(Some(route)) } diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..596fb22 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +[flake8] +exclude = .nox,.git,__pycache__,build,dist,.history +verbose = 1 + +[tool:pytest] +qt_api = pyqt5 +console_output_style = progress +addopts = --benchmark-skip --flake8 +testpaths = tests +python_files = test_*.py +python_functions = test_* \ No newline at end of file diff --git a/setup.py b/setup.py index e1c9f3a..29e0c2a 100644 --- a/setup.py +++ b/setup.py @@ -1,82 +1,115 @@ -import sys -import distutils.cmd -import distutils.log +# -*- coding: utf-8 -*- from setuptools import find_packages, setup from setuptools_rust import Binding, RustExtension, Strip - -with open("README.md", "r") as fh: +with open('README.md', 'r') as fh: long_description = fh.read() +extras_require = { + 'build': ['pyinstaller', 'pywin32'], + 'test': [ + 'pytest', + 'pytest-cov', + 'pytest-dependency', + 'pytest-benchmark[histogram]', + 'pytest-metadata', + 'pytest-flake8', + 'pytest-flask', + 'pytest-mock', + 'pytest-flask-sqlalchemy', + 'pytest-steps', + 'flake8-bugbear', + 'flake8-comprehensions', + 'cohesion', + 'hypothesis', + 'flaky' + ], + 'dev': [ + 'black; python_version >= "3.6"', + 'jinja2', + 'tsp', + 'flake8', + 'flake8-bugbear', + 'flake8-comprehensions', + 'cohesion', + 'pre-commit', + 'ipython', + 'flask-konch', + 'setuptools_rust' + ], + 'gui': ['PyQt5', 'pyperclip'], + 'web': [ + 'flask', + 'gevent', + 'webargs', + 'flask-executor', + 'flask-wtf', + 'flask-user', + 'flask-debugtoolbar', + 'flask-bootstrap4', + 'flask-sqlalchemy', + 'flask-nav', + 'flask-admin', + 'sqlalchemy_utils[password]', + 'python-dotenv', + ], +} +extras_require['all'] = sorted(set(sum(extras_require.values(), []))) + setup( - name="ed_lrr_gui", - version_format="{tag}.dev{commitcount}+{gitsha}", - author="Daniel Seiller", - author_email="earthnuker@gmail.com", - description="Elite: Dangerous long range route plotter", + use_scm_version={'write_to': '__version__.py'}, + name='ed_lrr_gui', + author='Daniel Seiller', + author_email='earthnuker@gmail.com', + description='Elite: Dangerous long range route plotter', long_description=long_description, - long_description_content_type="text/markdown", - url="none yet", + long_description_content_type='text/markdown', + url='https://gitlab.com/Earthnuker/ed_lrr/-/tree/pyqt_gui', rust_extensions=[ RustExtension( - "_ed_lrr", - path="rust/Cargo.toml", + '_ed_lrr', + path='rust/Cargo.toml', binding=Binding.PyO3, - rustc_flags=["--emit=asm"], + rustc_flags=['--emit=asm'], strip=Strip.No, debug=False, native=True, + quiet=True, ) ], packages=find_packages(), - entry_points={"console_scripts": ["ed_lrr = ed_lrr_gui.__main__:main"]}, - install_requires=[ - "appdirs", - "PyYAML", - "requests", - "python-dateutil", - "pyperclip", - "click", - "tqdm", - "PyQt5", - "click-default-group", - "profig", - "ujson", - "colorama", - "svgwrite", - ], - setup_requires=[ - "setuptools", - "setuptools-rust", - "wheel", - "pytest-runner", - "setuptools-git-version", - ], - extras_require={ - "test": ["pytest", "pytest-cov", "pytest-dependency"], - "dev": ["black", "jinja2", "tsp"], - "web": [ - "flask", - "gevent", - "webargs", - "flask-executor", - "flask-wtf", - "flask-user", - "flask-debugtoolbar", - "flask-bootstrap4", - "flask-sqlalchemy", - "flask-nav", - "flask-admin", - "sqlalchemy_utils[password]", - ], + entry_points={ + 'console_scripts': ['ed_lrr = ed_lrr_gui.__main__:main'], + 'gui_scripts': ['ed_lrr_gui = ed_lrr_gui.__main__:gui_main'] }, + install_requires=[ + 'appdirs', + 'PyYAML', + 'requests', + 'python-dateutil', + 'click', + 'tqdm', + 'click-default-group', + 'profig', + 'ujson', + 'colorama', + 'svgwrite', + ], + setup_requires=['setuptools', 'setuptools-rust', + 'setuptools-scm', 'wheel'], + dependency_links=['https://github.com/Nuitka/Nuitka/archive/develop.zip'], + extras_require=extras_require, classifiers=[ - "License :: OSI Approved :: MIT License", - "Programming Language :: Rust", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: Implementation :: CPython", - "Operating System :: OS Independent", + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Rust', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: Implementation :: CPython', + 'Operating System :: Windows', + 'Operating System :: Linux', ], include_package_data=True, zip_safe=False, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..8e003d2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,120 @@ +import pytest +import random +import csv +from tempfile import mkstemp +from pathlib import Path + + +def get_mult(star_type): + if star_type.startswith("Neutron"): + return 4.0 + if star_type.startswith("White Dwarf"): + return 1.5 + return 1.0 + + +def gen_pos(p_distrib): + p = [] + for v in p_distrib: + v = random.triangular(-v, v) + p.append(v) + return p + + +def make_stars(num, p_distrib): + star_types = [ + "A (Blue-White) Star", + "A (Blue-White super giant) Star", + "B (Blue-White) Star", + "B (Blue-White super giant) Star", + "Black Hole", + "CJ Star", + "CN Star", + "C Star", + "F (White) Star", + "F (White super giant) Star", + "G (White-Yellow) Star", + "G (White-Yellow super giant) Star", + "Herbig Ae/Be Star", + "K (Yellow-Orange giant) Star", + "K (Yellow-Orange) Star", + "L (Brown dwarf) Star", + "M (Red dwarf) Star", + "M (Red giant) Star", + "M (Red super giant) Star", + "MS-type Star", + "Neutron Star", + "O (Blue-White) Star", + "star_type", + "S-type Star", + "Supermassive Black Hole", + "T (Brown dwarf) Star", + "T Tauri Star", + "White Dwarf (DAB) Star", + "White Dwarf (DA) Star", + "White Dwarf (DAV) Star", + "White Dwarf (DAZ) Star", + "White Dwarf (DB) Star", + "White Dwarf (DBV) Star", + "White Dwarf (DBZ) Star", + "White Dwarf (DC) Star", + "White Dwarf (DCV) Star", + "White Dwarf (DQ) Star", + "White Dwarf (D) Star", + "Wolf-Rayet C Star", + "Wolf-Rayet NC Star", + "Wolf-Rayet N Star", + "Wolf-Rayet O Star", + "Wolf-Rayet Star", + "Y (Brown dwarf) Star", + ] + id_n = 0 + while id_n < num: + name = "System {}".format(id_n) + body = "System {} Star {}".format(id_n, 0) + distance = 0 + star_type = random.choice(star_types) + mult = get_mult(star_type) + x, y, z = gen_pos(p_distrib) + s_type = random.choice(star_types) + record = [id_n, s_type, name, body, mult, distance] + record.extend((x, y, z)) + yield record + id_n += 1 + for sub_id in range(random.randint(0, 4)): + star_type = random.choice(star_types) + mult = get_mult(star_type) + distance = random.randint(100, 10000) + body = "System {} Star {}".format(id_n, sub_id + 1) + s_type = random.choice(star_types) + record = [id_n, s_type, name, body, mult, distance] + record.extend((x, y, z)) + yield record + id_n += 1 + + +@pytest.fixture(scope="module") +def stars_path(): + num_stars = int(1e7) + p_distrib = [5000, 5000, 500] + tmpfile, filename = mkstemp(suffix=".csv", prefix="stars_", text=True) + filename = Path(filename) + tmpfile = open(tmpfile, "w", encoding="utf-8") + fields = ["id", "star_type", "system", "body", "mult"] + fields += ["distance", "x", "y", "z"] + csv_writer = csv.DictWriter(tmpfile, fields) + rows = (dict(zip(fields, row)) for row in make_stars(num_stars, p_distrib)) + csv_writer.writeheader() + csv_writer.writerows(rows) + tmpfile.close() + while True: + sys_ids = random.choices(range(num_stars), k=10) + if len(set(sys_ids)) == len(sys_ids): + break + rand_sys = list(map("System {}".format, sys_ids)) + yield str(filename.resolve()), rand_sys + if filename.exists(): + filename.unlink() + idx = filename.with_suffix(".idx") + if idx.exists(): + idx.unlink() diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py new file mode 100644 index 0000000..31d8f89 --- /dev/null +++ b/tests/test_benchmark.py @@ -0,0 +1,45 @@ +import pytest +import os +from math import log2, ceil + + +def resolve_systems(r, *names): + ret = [] + mapping = r.resolve_systems(*names) + for name in names: + ret.append(mapping.get(name)) + return ret + + +@pytest.fixture(scope="module") +def ed_lrr_router(stars_path): + stars_path, systems = stars_path + from ed_lrr_gui import PyRouter + + r = PyRouter(lambda status: None) + r.load(stars_path) + system_ids = resolve_systems(r, *systems) + return r, system_ids + + +argvalues = [(0, 0)] +argvalues += [(2 ** n, 0) for n in range(17)] +argvalues += [(0, g) for g in (0.25, 0.5, 0.75, 1)] +ids = [] +for width, greedyness in argvalues: + ids.append("beam_width:{}-greedyness:{}".format(width, greedyness)) + +n_workers = [0] +n_workers += [2**n for n in range(ceil(log2(os.cpu_count()))+1)] + + +@pytest.mark.parametrize("workers", n_workers, + ids=lambda v: "workers:{}".format(v)) +@pytest.mark.parametrize(argnames=("width", "greedyness"), + argvalues=argvalues, ids=ids) +@pytest.mark.parametrize("r_range", [48.0], ids=lambda v: "range:{}".format(v)) +def test_benchmark(benchmark, ed_lrr_router, + r_range, workers, greedyness, width): + r, system_ids = ed_lrr_router + args = system_ids, r_range, greedyness, width, workers + benchmark(r.route, *args) diff --git a/tests/test_ed_lrr.py b/tests/test_ed_lrr.py index 2f2e3f9..70e1962 100644 --- a/tests/test_ed_lrr.py +++ b/tests/test_ed_lrr.py @@ -1,24 +1,78 @@ +# -*- coding: utf-8 -*- +import random import pytest - -stars_csv = "D:\\devel\\rust\\ED_LRR\\stars.csv" +import os +from flaky import flaky -@pytest.mark.dependency() -def test_import(): - import _ed_lrr +if not hasattr(random, "choices"): + + def choices(population, *, k=1): + return [random.choice(population) for _ in range(k)] + + random.choices = choices -@pytest.mark.dependency(depends=["test_import"]) -def test_search_works(): - import _ed_lrr +@pytest.fixture(scope="module") +def py_router(stars_path): + from ed_lrr_gui import PyRouter - system_names = ["Ix", "Sol", "Colonia", "Sagittarius A*"] - systems = _ed_lrr.find_sys(system_names, stars_csv) - print(systems) + stars_path, names = stars_path + router = PyRouter(lambda status: None) + router.load(stars_path) + resolved_systems = router.resolve_systems(*names) + for name in names: + err = "Failed to resolve {}".format(name) + assert name in resolved_systems, err + yield router, resolved_systems -@pytest.mark.dependency(depends=["test_import"]) -def test_zero_range_fails(): - import _ed_lrr +class Test_ED_LRR(object): # noqa: H601 + @pytest.mark.dependency() + @flaky(max_runs=10, min_passes=5) + def test_load_and_resolve(self, stars_path): + stars_path, names = stars_path + from ed_lrr_gui import PyRouter - # _ed_lrr.route() + router = PyRouter(lambda status: None) + router.load(stars_path) + resolved_systems = router.resolve_systems(*names) + for name in names: + err = "Failed to resolve {}".format(name) + assert name in resolved_systems, err + + @pytest.mark.dependency(depends=["Test_ED_LRR::test_load_and_resolve"]) + @flaky(max_runs=10, min_passes=5) + def test_zero_range_fails(self, py_router): + r, resolved_systems = py_router + waypoints = random.choices(list(resolved_systems.values()), k=2) + err = pytest.raises(RuntimeError, r.route, waypoints, 0) + err.match(r"No route from .* to .* found!") + + beam_widths = [256, 512, 1024, 0] + greedyness = [0, 0.5, 1] + n_workers = [0, os.cpu_count()] + ranges = [30, 50] + + @pytest.mark.dependency(depends=["Test_ED_LRR::test_load_and_resolve"]) + @flaky(max_runs=10, min_passes=2) + @pytest.mark.parametrize("workers", n_workers, + ids=lambda v: "workers:{}".format(v)) + @pytest.mark.parametrize("jump_range", ranges, + ids=lambda v: "range:{}".format(v)) + @pytest.mark.parametrize( + "greedyness", greedyness, ids=lambda v: "greedyness:{}".format(v) + ) + @pytest.mark.parametrize( + "beam_width", beam_widths, ids=lambda v: "beam_width:{}".format(v) + ) + def test_route(self, py_router, jump_range, + workers, greedyness, beam_width): + r, resolved_systems = py_router + waypoints = random.choices(list(resolved_systems.values()), k=2) + args = waypoints, jump_range, greedyness, beam_width, workers + err = "Failed to route for waypoints: {}".format(waypoints) + route_len = r.route(*args) + assert route_len != 0, err + del r + # TODO: verify hop distance and inclusion of waypoints diff --git a/tests/test_gui.py b/tests/test_gui.py index 6eb59d5..48bcb58 100644 --- a/tests/test_gui.py +++ b/tests/test_gui.py @@ -1,18 +1,22 @@ -import pytest +# -*- coding: utf-8 -*- +# import pytest -@pytest.mark.dependency() -def test_import(): - import ed_lrr_gui - from ed_lrr_gui.main import main - import ed_lrr_gui.gui as ED_LRR_GUI +# class Test_GUI(object): +# pass + +# @pytest.mark.dependency() +# def test_import(): +# import ed_lrr_gui +# from ed_lrr_gui.main import main +# import ed_lrr_gui.gui as ED_LRR_GUI -@pytest.mark.dependency(depends=["test_import"]) -def test_search_works(): - import ed_lrr_gui +# @pytest.mark.dependency(depends=["test_import"]) +# def test_search_works(): +# import ed_lrr_gui -@pytest.mark.dependency(depends=["test_import"]) -def test_zero_range_fails(): - import ed_lrr_gui +# @pytest.mark.dependency(depends=["test_import"]) +# def test_zero_range_fails(): +# import ed_lrr_gui diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 0000000..283f85d --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,21 @@ +import pytest + + +@pytest.fixture +def app(): + from ed_lrr_gui.web import app as flask_app + yield flask_app + + +@pytest.fixture +def db(): + from ed_lrr_gui.web import db as flask_db + yield flask_db + + +class Test_DB(object): # noqa: H601 + pass + + +class Test_Webapp(object): # noqa: H601 + pass diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 5fbf146..0000000 --- a/tox.ini +++ /dev/null @@ -1,50 +0,0 @@ -[tox] -envlist = py35-build,py36-build,py37-build,py38-build -requires = - tox-conda - -[testenv] -platform = win32 -description = Build ED_LRR -recreate = True -skip_install = True -skipsdist = True -deps = - PyQt5 - setuptools_rust - build: https://github.com/Nuitka/Nuitka/archive/develop.zip -conda_deps = - pycrypto - ujson - dev: ipython -passenv = - CARGO_HOME - RUSTUP_HOME - INCLUDE - LIB - MSSdk - DISTUTILS_USE_SDK -whitelist_externals = - cargo - iscc - cmd - powershell - conda -conda_channels = - conda-forge -commands = - python build_gui.py - - build: python -m pip install .[web] - build: python -m nuitka --remove-output --plugin-enable=multiprocessing --plugin-enable=qt-plugins --plugin-enable=gevent --standalone --assume-yes-for-downloads --follow-imports --output-dir=exe ed_lrr_gui\__main__.py - build: cmd /c rename exe\__main__.dist\__main__.exe ED_LRR.exe - build: iscc /F"ED_LRR_{envname}" /Qp installer/ED_LRR.iss - - install: python -m pip install -e .[dev,web,test] - - dev: python -m pip install -e .[dev,web,test] - dev: cmd /c echo install done now run "conda activate {envdir}" - - test: python -m pip install .[dev,web,test] - test: cargo test - test: pytest \ No newline at end of file