Earthnuker/ED_LRR
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Big update, AppVeyor_Test

Browse source
This commit is contained in:
Daniel S. 2020-03-28 14:53:52 +01:00
parent aec570d055
commit e71faf0b92
65 changed files with 2141 additions and 1355 deletions
Download patch file Download diff file Expand all files Collapse all files

1
ed_lrr_gui/web/__init__.py
View file

@ -1 +1,2 @@
# -*- coding: utf-8 -*-
from .app import app, templates, db

275
ed_lrr_gui/web/app.py
View file

@ -1,28 +1,22 @@
# -*- coding: utf-8 -*-
from flask import (
Flask,
jsonify,
session,
render_template,
redirect,
url_for,
send_from_directory,
request,
flash,
current_app
current_app,
)
from flask.json.tag import JSONTag
from flask.cli import AppGroup
import uuid
import pickle
import os
import time
import random
import base64
import gevent
import click
from functools import wraps
from concurrent.futures.process import BrokenProcessPool
from datetime import datetime, timedelta
from decimal import Decimal
from multiprocessing import Queue
from webargs import fields, validate
from webargs.flaskparser import use_kwargs
@ -50,11 +44,9 @@ from flask_login import (
from flask_debugtoolbar import DebugToolbarExtension
from werkzeug.http import HTTP_STATUS_CODES
from sqlalchemy_utils import generic_repr, JSONType, PasswordType, UUIDType
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import scoped_session, sessionmaker, relationship, backref
from sqlalchemy.types import Float, String, DateTime
from sqlalchemy.orm import relationship
from sqlalchemy.types import DateTime
from jinja2.exceptions import TemplateNotFound
from .forms import RouteForm, LoginForm, RegisterForm, ChangePasswordForm
from .utils import prepare_route, BootsrapRenderer, is_safe_url
@ -65,22 +57,24 @@ templates = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates"
app = Flask(__name__, template_folder=templates)
app.config.from_pyfile("config.py")
executor = Executor(app)
db = SQLAlchemy(app)
bootstrap = Bootstrap(app)
csrf = CSRFProtect(app)
nav = Nav(app)
login_manager = LoginManager(app)
app.executor = executor = Executor(app)
app.db = db = SQLAlchemy(app)
app.bootstrap = bootstrap = Bootstrap(app)
app.csrf = csfr = CSRFProtect(app)
app.nav = nav = Nav(app)
app.login_manager = login_manager = LoginManager(app)
login_manager.login_view = "login"
login_manager.session_protection = "strong"
admin = Admin(app, name="ED_LRR", template_mode="bootstrap3")
app.debug=True
toolbar = DebugToolbarExtension(app)
app.debug = True
app.toolbar = toolbar = DebugToolbarExtension(app)
def wants_json_response():
return request.accept_mimetypes['application/json'] >= \
request.accept_mimetypes['text/html']
return (
request.accept_mimetypes["application/json"]
>= request.accept_mimetypes["text/html"]
)
@app.errorhandler(422)
@ -89,30 +83,34 @@ def wants_json_response():
@app.errorhandler(404)
def handle_error(err):
if wants_json_response():
return jsonify(error=str(err),code=err.code), err.code
templates=["error/{}.html".format(err.code),"error/default.html"]
return jsonify(error=str(err), code=err.code), err.code
templates = ["error/{}.html".format(err.code), "error/default.html"]
try:
print(dir(err))
return render_template(templates,error=err),err.code
return render_template(templates, error=err), err.code
except TemplateNotFound:
return err.get_response()
def role_required(*roles):
def wrapper(fn):
@wraps(fn)
def decorated_view(*args, **kwargs):
if not current_user.is_authenticated():
return current_app.login_manager.unauthorized()
has_role=False
user=current_app.login_manager.reload_user()
return current_app.login_manager.unauthorized()
has_role = False
user = current_app.login_manager.reload_user()
for role in roles:
has_role|=user.has_role(role)
has_role |= user.has_role(role)
if not has_role:
return current_app.login_manager.unauthorized()
return fn(*args, **kwargs)
return decorated_view
return wrapper
@login_manager.user_loader
def load_user(user_name):
return User.query.get(user_name)
@ -120,7 +118,7 @@ def load_user(user_name):
@login_manager.request_loader
def load_user_from_header(header_val):
for api_key in [request.args.get('api_key'),request.headers.get('X-API-Key')]:
for api_key in [request.args.get("api_key"), request.headers.get("X-API-Key")]:
if api_key:
user = User.query.filter_by(api_key=api_key).one_or_none()
if user:
@ -130,20 +128,21 @@ def load_user_from_header(header_val):
def left_nav():
links=[View("Home", "index"),View("Route", "route"),View("Jobs", "status",job_id=None)]
if current_user.has_role('admin') or current_user.has_role('worker_host'):
links.insert(2,View("Workers","worker"))
return Navbar(
"E:D LRR",
*links
)
links = [
View("Home", "index"),
View("Route", "route"),
View("Jobs", "status", job_id=None),
]
if current_user.has_role("admin") or current_user.has_role("worker_host"):
links.insert(2, View("Workers", "worker"))
return Navbar("E:D LRR", *links)
def right_nav():
links = [View("Login", "login"), View("Register", "register")]
if current_user.is_authenticated:
links = [View("Change Password", "change_password"), View("Logout", "logout")]
if current_user.has_role('admin'):
if current_user.has_role("admin"):
links = [View("Admin", "admin.index")] + links
return Navbar("", *links)
@ -158,16 +157,15 @@ def compute_route(args, kwargs):
class AnonymousUser(AnonymousUserMixin):
def has_role(self,role):
def has_role(self, role):
return False
@property
def roles(self):
return []
@roles.setter
def __set_roles(self,value):
def __set_roles(self, value):
raise NotImplementedError
@ -180,52 +178,68 @@ class Worker(db.Model):
UUIDType(binary=False, native=False), primary_key=True, default=uuid.uuid4
)
name = db.Column(db.String, unique=True)
current_job=db.Column(UUIDType(binary=False, native=False),db.ForeignKey("job.id"), nullable=True,default=None)
job = relationship('Job',backref="workers")
last_active = db.Column(DateTime, nullable=True,default=None)
owner_name = db.Column(
db.String, db.ForeignKey("user.name"), nullable=True,index=True
current_job = db.Column(
UUIDType(binary=False, native=False),
db.ForeignKey("job.id"),
nullable=True,
default=None,
)
owner = relationship("User",backref="workers")
job = relationship("Job", backref="workers")
last_active = db.Column(DateTime, nullable=True, default=None)
owner_name = db.Column(
db.String, db.ForeignKey("user.name"), nullable=True, index=True
)
owner = relationship("User", backref="workers")
user_roles = db.Table('user_roles',
db.Column('user_name', db.String, db.ForeignKey('user.name'),primary_key=True),
db.Column('role_name', db.String, db.ForeignKey('role.name'),primary_key=True)
user_roles = db.Table(
"user_roles",
db.Column("user_name", db.String, db.ForeignKey("user.name"), primary_key=True),
db.Column("role_name", db.String, db.ForeignKey("role.name"), primary_key=True),
)
class Role(db.Model):
name = db.Column(db.String, unique=True,index=True,primary_key=True)
def __init__(self,name):
self.name=name
class Role(db.Model):
name = db.Column(db.String, unique=True, index=True, primary_key=True)
def __init__(self, name):
self.name = name
def __repr__(self):
return self.name
class User(db.Model, UserMixin):
name = db.Column(db.String, unique=True,index=True,primary_key=True)
name = db.Column(db.String, unique=True, index=True, primary_key=True)
is_active = db.Column(db.Boolean, default=False)
api_key = db.Column(
UUIDType(binary=False, native=False), nullable=True, default=uuid.uuid4,index=True
UUIDType(binary=False, native=False),
nullable=True,
default=uuid.uuid4,
index=True,
)
password = db.Column(PasswordType(schemes=["pbkdf2_sha512"], max_length=256))
created = db.Column(DateTime, default=datetime.today)
roles = db.relationship("Role",secondary="user_roles")
roles = db.relationship("Role", secondary="user_roles")
def add_roles(self,roles):
def add_roles(self, roles):
for role_name in roles:
role=Role.query.filter_by(name=role_name).one()
if not role in self.roles:
role = Role.query.filter_by(name=role_name).one()
if role not in self.roles:
self.roles.append(role)
db.session.commit()
def has_role(self,role_name):
return Role.query.join(User.roles).filter(User.name==self.name,Role.name==role_name).count()>0
return ret
def has_role(self, role_name):
return (
Role.query.join(User.roles)
.filter(User.name == self.name, Role.name == role_name)
.count()
> 0
)
def reset_api_key(self):
self.api_key=uuid,uuid4()
self.api_key = uuid.uuid4()
db.session.add(self)
db.session.comiit()
@ -241,24 +255,23 @@ class Job(db.Model):
UUIDType(binary=False, native=False), primary_key=True, default=uuid.uuid4
)
user_name = db.Column(
db.String, db.ForeignKey("user.name"), nullable=True,index=True
db.String, db.ForeignKey("user.name"), nullable=True, index=True
)
func = db.Column(db.String)
args = db.Column(JSONType)
kwargs = db.Column(JSONType)
state = db.Column(JSONType, default={})
priority = db.Column(db.Integer, default=0,nullable=True)
priority = db.Column(db.Integer, default=0, nullable=True)
created = db.Column(DateTime, default=datetime.today)
finished = db.Column(DateTime, nullable=True, default=None)
started = db.Column(DateTime, nullable=True, default=None)
last_update = db.Column(DateTime, nullable=True, default=None)
user = relationship("User",backref="jobs")
user = relationship("User", backref="jobs")
# ============================================================
def __repr__(self):
return str(self.id)
@property
def future(self):
fut = executor.futures._futures.get(self.id)
@ -266,20 +279,26 @@ class Job(db.Model):
@property
def sort_key(self):
state_priorities={"Queued":0,"Starting":1,"Error":1,"Stalled":1,"Running":1}
status_key=state_priorities.get(self.status[1],-1)+1
user=1-int(self.user is not None)
return (user,-status_key,self.priority,self.created)
state_priorities = {
"Queued": 0,
"Starting": 1,
"Error": 1,
"Stalled": 1,
"Running": 1,
}
status_key = state_priorities.get(self.status[1], -1) + 1
user = 1 - int(self.user is not None)
return (user, -status_key, self.priority, self.created)
@property
def age(self):
dt=datetime.today()-self.created
dt = datetime.today() - self.created
return dt - dt % timedelta(seconds=1)
@classmethod
def next(cls):
for job in sorted(cls.query.all(),key=lambda v:v.sort_key):
if job.status[1] in ['Done']:
def get_next(cls):
for job in sorted(cls.query.all(), key=lambda v: v.sort_key):
if job.status[1] in ["Done"]:
continue
return job
return None
@ -287,15 +306,15 @@ class Job(db.Model):
@property
def status(self):
states=[
("primary", "Done"),
("danger", "Error"),
("info", "Stalled"),
("success", "Running"),
("secondary", "Starting"),
("warning", "Queued")
]
#return states[self.id.int%len(states)]
# [
# ("primary", "Done"),
# ("danger", "Error"),
# ("info", "Stalled"),
# ("success", "Running"),
# ("secondary", "Starting"),
# ("warning", "Queued"),
# ]
# return states[self.id.int%len(states)]
if self.state.get("result"):
return ("primary", "Done")
if self.state.get("error"):
@ -391,7 +410,7 @@ class Job(db.Model):
).total_seconds()
if time_since_last_upd < 5.0:
return
state = dict()
state = {}
state.update(self.state)
state.update({"progress": cb_state})
self.state = state
@ -403,7 +422,7 @@ class Job(db.Model):
def done(self, future):
print(self.id, "DONE")
state = dict()
state = {}
state.update(self.state)
executor.futures.pop(self.id)
exc = future.exception()
@ -420,24 +439,25 @@ class Job(db.Model):
db.create_all()
for role in ['admin','user','worker_host']:
for role in ["admin", "user", "worker_host"]:
if Role.query.filter_by(name=role).one_or_none() is None:
db.session.add(Role(role))
def create_user(name,password,roles,active=False):
user=User.query.filter_by(name=name).one_or_none()
def create_user(name, password, roles, active=False):
user = User.query.filter_by(name=name).one_or_none()
if user:
db.session.delete(user)
user=User(name=name,password=password,is_active=active)
user = User(name=name, password=password, is_active=active)
user.add_roles(roles)
db.session.add(user)
db.session.commit()
return user
create_user('admin','admin',['admin','user'],True)
create_user('user','user',['user'],True)
create_user('host','host',['user','worker_host'],True)
# create_user("admin", "admin", ["admin", "user"], True)
# create_user("user", "user", ["user"], True)
# create_user("host", "host", ["user", "worker_host"], True)
class SQLAView(ModelView):
@ -449,7 +469,7 @@ class SQLAView(ModelView):
column_display_pk = True
def is_accessible(self):
return current_user.is_authenticated and current_user.has_role('admin')
return current_user.is_authenticated and current_user.has_role("admin")
def inaccessible_callback(self, name, **kwargs):
return redirect(url_for("login"))
@ -458,7 +478,7 @@ class SQLAView(ModelView):
class UserView(SQLAView):
from wtforms import PasswordField
column_list = ("name", "active", "password", "api_key","roles")
column_list = ("name", "active", "password", "api_key", "roles")
column_formatters = {
"password": lambda view, context, model, name: "",
"api_key": lambda view, context, model, name: model.api_key or "",
@ -469,9 +489,7 @@ class UserView(SQLAView):
class JobView(SQLAView):
# Job.id,Job.user,Job.func,Job.args,Job.kwargs,Job.state,Job.created,Job.finished,Job.started,Job.last_update
column_list = ("id", "status", "user", "created", "started", "finished")
column_formatters = {
"status": lambda view, context, model, name: model.status[1],
}
column_formatters = {"status": lambda view, context, model, name: model.status[1]}
class WorkerView(SQLAView):
@ -485,6 +503,7 @@ class WorkerView(SQLAView):
# "status": lambda view, context, model, name: model.status[1],
# }
admin.add_view(JobView(Job, db.session))
admin.add_view(UserView(User, db.session))
admin.add_view(SQLAView(Worker, db.session))
@ -531,7 +550,7 @@ def api_route(_=None, **args):
args["factor"],
None,
r"D:\devel\rust\ED_LRR\stars.csv",
app.config['ROUTE_WORKERS']
app.config["ROUTE_WORKERS"],
)
return jsonify({"id": submit_job(ed_lrr.route, *args)})
@ -544,7 +563,7 @@ def api_status():
@app.route("/api/whoami")
def api_whoami():
return jsonify({'name':current_user.name})
return jsonify({"name": current_user.name})
@app.route("/api/status/<uuid:job_id>")
@ -573,21 +592,21 @@ def route():
return render_template("form.html", form=form, title="Plot Route")
@app.route("/status/",defaults={'job_id':None})
@app.route("/status/", defaults={"job_id": None})
@app.route("/status/<uuid:job_id>")
@login_required
def status(job_id=None):
if job_id is not None:
job=Job.query.get_or_404(str(job_id))
job = Job.query.get_or_404(str(job_id))
return render_template("job.html", job=job)
return render_template(
"status.html", Job=Job, state=request.args.get("state")
)
return render_template("status.html", Job=Job, state=request.args.get("state"))
@app.route("/")
def index():
return render_template("index.html")
@app.route("/login", methods=["GET", "POST"])
def login():
if current_user.is_authenticated:
@ -602,9 +621,9 @@ def login():
flash("Account is deactivated!", "warning")
return redirect(url_for("login"))
login_user(user, remember=form.data["remember"])
next = request.args.get('next')
next = request.args.get("next")
if not is_safe_url(next):
next=None
next = None
return redirect(next or url_for("status"))
return render_template("form.html", form=form, title="Login")
@ -614,7 +633,7 @@ def register():
form = RegisterForm()
if form.validate_on_submit():
if User.query.filter_by(name=form.data["username"]).one_or_none() is not None:
flash('Username already exists','danger')
flash("Username already exists", "danger")
return render_template("form.html", form=form, title="Register")
user = User()
user.name = form.data["username"]
@ -641,11 +660,13 @@ def change_password():
return redirect(url_for("status"))
return render_template("form.html", form=form, title="Register")
@app.route("/workers/")
@login_required
def worker():
return render_template("workers.html")
@app.route("/logout")
def logout():
logout_user()
@ -654,12 +675,38 @@ def logout():
@app.before_first_request
def resume_jobs():
print(Job.next())
print("NEXT:", Job.get_next())
with app.test_request_context():
for job in Job.query.all():
if job.status[1] != "Done":
print("Restarting {} with state {}".format(job.id, job.status[1]))
job.start()
user_cli = AppGroup('user', help="Manage users")
job_cli = AppGroup('job', help="Manage Jobs")
worker_cli = AppGroup('worker', help="Manage Workers")
@app.cli.command("gevent")
def cmd_gevent():
return
@user_cli.command("create")
@click.argument("name")
@click.option("-i", "--inactive", help="Crate account as inactive", is_flag=True, default=False)
@click.option("-r", "--role", help="Assign role to account", default=["user"], multiple=True)
@click.password_option("-p", "--password", help="Password for user")
def cmd_create_user(name, role, password, inactive):
"Create a new user"
create_user(name, password, role, not inactive)
print("User created!")
app.cli.add_command(user_cli)
app.cli.add_command(job_cli)
app.cli.add_command(worker_cli)
if __name__ == "__main__":
app.run(host="127.0.0.1", port=3777, debug=True)

5
ed_lrr_gui/web/config.py
View file

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
import os
SECRET_KEY = "ED_LRR_WEBAPP"
@ -8,11 +9,11 @@ SQLALCHEMY_TRACK_MODIFICATIONS = False
ROUTE_WORKERS = 0
EXECUTOR_TYPE = "process"
EXECUTOR_MAX_WORKERS = os.cpu_count()-1
EXECUTOR_MAX_WORKERS = os.cpu_count() - 1
EXECUTOR_FUTURES_MAX_LENGTH = 500
FLASK_ADMIN_SWATCH = "Darkly"
DEBUG_TB_TEMPLATE_EDITOR_ENABLED = True
MAIL_DEFAULT_SENDER = '"ED_LRR Admin" <ed_lrr@gmail.com>'
MAIL_DEFAULT_SENDER = '"ED_LRR Admin" <ed_lrr@gmail.com>'

4
ed_lrr_gui/web/forms.py
View file

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
from flask_wtf import FlaskForm
from wtforms import (
StringField,
@ -14,6 +15,7 @@ from wtforms.widgets.html5 import NumberInput
from wtforms.widgets import TextInput
from wtforms.validators import ValidationError
class StringListField(Field):
widget = TextInput()
@ -61,7 +63,7 @@ class RouteForm(FlaskForm):
default=50,
widget=NumberInput(min=0, max=100, step=1),
)
priority = FloatField(
"Priority (0=max, 100=min)",
[validators.NumberRange(0, 100)],

2
ed_lrr_gui/web/static/theme.css
View file

@ -20,4 +20,4 @@ table {
border: 1px solid #eee;
width: 512px;
height: 512px;
}
}

2
ed_lrr_gui/web/templates/admin/index.html
View file

@ -2,4 +2,4 @@
{% block body %}
<p>Hello world</p>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/base.html
View file

@ -45,4 +45,4 @@
{# application content needs to be provided in the app_content block #}
{% block app_content %}{% endblock %}
</div>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/error/404.html
View file

@ -3,4 +3,4 @@
{% block app_content %}
<h1>404 Not Found</h1>
<p><a href="{{ url_for('index') }}"><button type="button" class="btn btn-secondary">Back</button></a></p>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/form.html
View file

@ -13,4 +13,4 @@
{{ wtf.quick_form(form) }}
</div>
</div>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/index.html
View file

@ -7,4 +7,4 @@
Number of Jobs: {{current_user.jobs|count}}
</div>
</div>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/job.html
View file

@ -128,4 +128,4 @@
{% endif %}
</div>
</div>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/status.html
View file

@ -84,4 +84,4 @@
{% endfor %}
</table>
</div>
{% endblock %}
{% endblock %}

2
ed_lrr_gui/web/templates/workers.html
View file

@ -11,4 +11,4 @@
{% endif %}
</div>
</div>
{% endblock %}
{% endblock %}

8
ed_lrr_gui/web/utils.py
View file

@ -1,13 +1,15 @@
# -*- coding: utf-8 -*-
from flask_nav.renderers import Renderer
from dominate import tags
from urllib.parse import urlparse,urljoin
from urllib.parse import urlparse, urljoin
from flask import request
def is_safe_url(target):
ref_url = urlparse(request.host_url)
test_url = urlparse(urljoin(request.host_url, target))
return test_url.scheme in ('http', 'https') and \
ref_url.netloc == test_url.netloc
return test_url.scheme in ("http", "https") and ref_url.netloc == test_url.netloc
def dist(p1, p2):
s = 0

1
ed_lrr_gui/web/worker.py
View file

@ -1,3 +1,4 @@
# -*- coding: utf-8 -*-
import requests as RQ
import _ed_lrr as ed_lrr