fmartingr
/
jeeves
Archived
1
0
Fork 0

Removed Django

This commit is contained in:
Felipe Martín 2019-10-13 13:09:43 +00:00
parent 9565fa4640
commit 790089df7d
26 changed files with 3 additions and 665 deletions

View File

@ -1,2 +0,0 @@
JEEVES_DEBUG=on
DATABASE_URL=sqlite:///local.db

View File

@ -1,11 +0,0 @@
FROM python:3
ENV PYTHONUNBUFFERED 1
WORKDIR /opt/app
ADD . /opt/app
RUN pip install poetry && \
poetry install
CMD ["poetry", "run", "python", "manage.py", "runserver", "0.0.0.0:8000"]

View File

@ -1,9 +1,6 @@
quicksetup: quicksetup:
poetry install poetry install
poetry run pre-commit install poetry run pre-commit install
cp .env.example .env
poetry run python manage.py migrate
poetry run python manage.py loaddata local_data
clean: test:
if [ -f "local.db" ]; then rm local.db; fi; poetry run pytest . --cov=jeeves.core

View File

@ -1,28 +0,0 @@
version: "3"
services:
backend:
build: .
restart: always
volumes:
- .:/opt/app
ports:
- 8000:8000
depends_on:
- redis
env_file:
- .env
redis:
image: redis:5
worker:
build: .
restart: always
command: poetry run python manage.py rundramatiq
volumes:
- .:/opt/app
depends_on:
- redis
env_file:
- .env

View File

@ -1 +0,0 @@
default_app_config = "jeeves.db.apps.DBAppConfig"

View File

@ -1,13 +0,0 @@
from django.contrib import admin
# Register your models here.
from .models import Run, Flow, Task
class TaskAdmin(admin.ModelAdmin):
list_display = ("name", "type", "pk")
admin.site.register(Flow)
admin.site.register(Run)
admin.site.register(Task, TaskAdmin)

View File

@ -1,10 +0,0 @@
from django.apps import AppConfig
from jeeves.core.registry import TaskRegistry
class DBAppConfig(AppConfig):
name = "jeeves.db"
def ready(self):
TaskRegistry.autodiscover()

View File

@ -1,20 +0,0 @@
[
{
"model": "auth.user",
"pk": 1,
"fields": {
"password": "pbkdf2_sha256$150000$8GIlObHdvhdv$phdSFNeTHkkN6MJnzBCcuC+qo5RI0gAQdLhAmWyFSR0=",
"last_login": "2019-09-23T18:48:28.585Z",
"is_superuser": true,
"username": "admin",
"first_name": "",
"last_name": "",
"email": "admin@admin.com",
"is_staff": true,
"is_active": true,
"date_joined": "2019-09-23T18:48:20.713Z",
"groups": [],
"user_permissions": []
}
}
]

View File

@ -1,93 +0,0 @@
# Generated by Django 2.2.5 on 2019-09-23 20:56
import uuid
import django.db.models.deletion
from django.db import models, migrations
import jeeves.db.models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Flow",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("modified_at", models.DateTimeField(auto_now=True)),
("name", models.CharField(max_length=64)),
("_definition", models.TextField()),
],
options={"abstract": False},
bases=(models.Model, jeeves.db.models.DefinitionMixin),
),
migrations.CreateModel(
name="Task",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("modified_at", models.DateTimeField(auto_now=True)),
("type", models.CharField(max_length=32)),
("_definition", models.TextField()),
],
options={"abstract": False},
bases=(jeeves.db.models.DefinitionMixin, models.Model),
),
migrations.CreateModel(
name="Run",
fields=[
(
"id",
models.UUIDField(
default=uuid.uuid4,
editable=False,
primary_key=True,
serialize=False,
),
),
("created_at", models.DateTimeField(auto_now_add=True)),
("modified_at", models.DateTimeField(auto_now=True)),
(
"status",
models.CharField(
choices=[("pending", "Pending"), ("finished", "Finished")],
default="pending",
max_length=32,
),
),
("success", models.BooleanField(default=True)),
("_definition", models.TextField()),
("_result", models.TextField(default="{}")),
(
"flow",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="db.Flow"
),
),
],
options={"abstract": False},
bases=(jeeves.db.models.DefinitionMixin, models.Model),
),
]

View File

@ -1,17 +0,0 @@
# Generated by Django 2.2.5 on 2019-09-25 21:00
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [("db", "0001_initial")]
operations = [
migrations.AddField(
model_name="task",
name="name",
field=models.CharField(default="Hey", max_length=128),
preserve_default=False,
)
]

View File

@ -1,14 +0,0 @@
# Generated by Django 2.2.5 on 2019-09-25 21:23
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [("db", "0002_task_name")]
operations = [
migrations.AlterField(
model_name="task", name="type", field=models.CharField(max_length=128)
)
]

View File

@ -1,21 +0,0 @@
# Generated by Django 2.2.5 on 2019-09-26 16:13
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [("db", "0003_auto_20190925_2123")]
operations = [
migrations.AddField(
model_name="run",
name="stderr",
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name="run",
name="stdout",
field=models.TextField(blank=True, null=True),
),
]

View File

@ -1,21 +0,0 @@
# Generated by Django 2.2.5 on 2019-09-26 16:29
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [("db", "0004_auto_20190926_1613")]
operations = [
migrations.AlterField(
model_name="run",
name="stderr",
field=models.TextField(blank=True, default=""),
),
migrations.AlterField(
model_name="run",
name="stdout",
field=models.TextField(blank=True, default=""),
),
]

View File

@ -1,18 +0,0 @@
# Generated by Django 2.2.5 on 2019-09-29 17:46
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [("db", "0005_auto_20190926_1629")]
operations = [
migrations.RenameField(model_name="run", old_name="stderr", new_name="output"),
migrations.RemoveField(model_name="run", name="stdout"),
migrations.AlterField(
model_name="flow",
name="_definition",
field=models.TextField(default='{"tasks": []}'),
),
]

View File

@ -1,172 +0,0 @@
import copy
import json
import uuid
import logging
from django.db import models
from jeeves.core.registry import TaskRegistry
logger = logging.getLogger(__name__)
class BaseModel(models.Model):
"""
Abstract model used when creating models in the project.
Uses an UUID4 as primary key field and adds a created_at, modified_at
fields.
"""
id = models.UUIDField(default=uuid.uuid4, primary_key=True, editable=False)
created_at = models.DateTimeField(auto_now_add=True)
modified_at = models.DateTimeField(auto_now=True)
class Meta:
abstract = True
class DefinitionMixin:
@property
def definition(self):
return json.loads(self._definition)
@definition.setter
def definition(self, value: dict):
self._definition = json.dumps(value)
class Flow(BaseModel, DefinitionMixin):
name = models.CharField(max_length=64)
_definition = models.TextField(default=json.dumps({"tasks": []}))
def serialize(self):
definition = copy.deepcopy(self.definition)
definition["tasks"] = [task.serialize() for task in self.tasks]
return {"name": self.name, "definition": definition}
@property
def tasks(self):
for task_ref in self.definition["tasks"]:
try:
task = Task.objects.from_reference(task_ref)
yield task
except Task.DoesNotExist:
logger.warning(
f"Task {task_ref} does not exist on db, removing from definition"
)
# TODO: Dirty cleanup of dangling tasks in definitions
definition = self.definition
definition["tasks"].remove(task_ref)
self.definition = definition
self.save()
def execute(self, foreground=False):
from jeeves.db.tasks import start_execution
execution = Run.from_flow(self)
if not foreground:
start_execution.send(execution_id=str(execution.pk))
else:
start_execution(execution_id=str(execution.pk))
return Run.objects.get(pk=str(execution.pk)) # Reload model instance
class TaskManager(models.Manager):
def from_reference(self, reference):
return self.get_queryset().get(pk=reference)
class Task(DefinitionMixin, BaseModel):
name = models.CharField(max_length=128)
type = models.CharField(max_length=128)
_definition = models.TextField()
objects = TaskManager()
def serialize(self):
return {"type": self.type, "definition": self.definition}
@property
def instance(self):
return TaskRegistry.get_task(self.type, **self.definition)
def run(self):
runner = self.instance
return FlowStep(str(self.pk), *runner.execute())
class Run(DefinitionMixin, BaseModel):
PENDING = "pending"
FINISHED = "finished"
RUN_STATUS = ((PENDING, PENDING.capitalize()), (FINISHED, FINISHED.capitalize()))
status = models.CharField(max_length=32, choices=RUN_STATUS, default=PENDING)
success = models.BooleanField(default=True)
output = models.TextField(blank=True, default="")
flow = models.ForeignKey(Flow, on_delete=models.CASCADE)
_definition = models.TextField()
_result = models.TextField(default="{}")
@property
def duration(self):
# TODO: Proper ended_at
return self.modified_at - self.created_at
@property
def result(self):
return json.loads(self._result)
@result.setter
def result(self, value: dict):
self._result = json.dumps(value)
@classmethod
def from_flow(cls, flow: Flow):
execution = cls()
execution.definition = flow.serialize()
execution.flow = flow
execution.save()
return execution
class FlowStep:
task: str
executed: bool = True
error: bool = False
output: str
def __init__(self, task_id, error, output):
self.task = task_id
self.error = error
self.output = output
def __repr__(self):
return f"<FlowStep error={self.error}>"
def serialize(self):
return {
"task": self.task,
"executed": self.executed,
"error": self.error,
"output": self.output,
}
class Result:
def __init__(self):
self.steps = []
def __repr__(self):
return f"""<Result
last={self.last_flow_step}>
"""
def serialize(self):
return {"steps": [step.serialize() for step in self.steps]}
def add_step(self, flow_step):
self.steps.append(flow_step)
@property
def last_flow_step(self):
return self.steps[-1]

View File

@ -1,22 +0,0 @@
import json
import dramatiq
from jeeves.db.models import Run, Result
@dramatiq.actor
def start_execution(execution_id):
run = Run.objects.get(pk=execution_id)
result = Result()
for task in run.flow.tasks:
flow_step = task.run()
result.add_step(flow_step)
run.output += flow_step.output
if flow_step.error:
run.success = False
break
run._result = json.dumps(result.serialize())
run.status = Run.FINISHED
run.save()
return run

View File

@ -1 +0,0 @@
urlpatterns = []

View File

View File

@ -1,5 +0,0 @@
from django.apps import AppConfig
class FrontendConfig(AppConfig):
name = "frontend"

View File

@ -1,127 +0,0 @@
import os
import environ
env = environ.Env()
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
environ.Env.read_env(env_file=os.path.join(BASE_DIR, ".env"))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = env.str("SECRET_KEY", default="1234567890")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env.bool("JEEVES_DEBUG", default=False)
ALLOWED_HOSTS = env.list("JEEVES_ALLOWED_HOSTS", default=["*"])
# Application definition
INSTALLED_APPS = [
# Django
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
# Third party
"django_dramatiq",
# Own
"jeeves.db",
"jeeves.frontend",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "jeeves.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
]
},
},
{"BACKEND": "django.template.backends.jinja2.Jinja2", "APP_DIRS": True, "DIRS": []},
]
WSGI_APPLICATION = "jeeves.wsgi.application"
# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
DATABASES = {"default": env.db()}
# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"
},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
]
# Internationalization
# https://docs.djangoproject.com/en/2.2/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = "/static/"
# Dramatiq
DRAMATIQ_BROKER = {
"BROKER": "dramatiq.brokers.redis.RedisBroker",
"OPTIONS": {"url": "redis://redis:6379/0"},
"MIDDLEWARE": [
"dramatiq.middleware.Prometheus",
"dramatiq.middleware.AgeLimit",
"dramatiq.middleware.TimeLimit",
"dramatiq.middleware.Callbacks",
"dramatiq.middleware.Retries",
"django_dramatiq.middleware.AdminMiddleware",
"django_dramatiq.middleware.DbConnectionsMiddleware",
],
}
# Defines which database should be used to persist Task objects when the
# AdminMiddleware is enabled. The default value is "default".
DRAMATIQ_TASKS_DATABASE = "default"

View File

@ -1,22 +0,0 @@
"""jeeves URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.urls import path, include
from django.contrib import admin
urlpatterns = [
path("admin/", admin.site.urls),
path("", include("jeeves.frontend.urls")),
]

View File

@ -1,16 +0,0 @@
"""
WSGI config for jeeves project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jeeves.settings")
application = get_wsgi_application()

View File

@ -1,21 +0,0 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jeeves.settings")
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == "__main__":
main()

View File

@ -9,12 +9,8 @@ jeeves = "jeeves.cli:main"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.7" python = "^3.7"
django = "^2.2"
django-environ = "^0.4.5"
jinja2 = "^2.10" jinja2 = "^2.10"
pydantic = "^0.32.2" pydantic = "^0.32.2"
django_dramatiq = "^0.8.0"
dramatiq = {version = "^1.7", extras = ["redis"]}
click = "^7.0" click = "^7.0"
toml = "^0.10.0" toml = "^0.10.0"

View File

@ -11,6 +11,6 @@ include_trailing_comma = True
length_sort = 1 length_sort = 1
lines_between_types = 0 lines_between_types = 0
line_length = 88 line_length = 88
known_third_party = click,django,dramatiq,environ,factory,pydantic,pytest,toml known_third_party = click,django,factory,pydantic,pytest,toml
sections = FUTURE, STDLIB, DJANGO, THIRDPARTY, FIRSTPARTY, LOCALFOLDER sections = FUTURE, STDLIB, DJANGO, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
no_lines_before = LOCALFOLDER no_lines_before = LOCALFOLDER