fmartingr
/
jeeves
Archived
1
0
Fork 0
Code Issues Releases Activity

Basic async

This commit is contained in:
Felipe Martin 2019-09-30 18:13:37 +02:00
parent 07cecc71db
commit 831fba65df
Signed by: fmartingr
GPG Key ID: 716BC147715E716F
8 changed files with 157 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docker-compose.yml
View File

@ -13,3 +13,12 @@ services:
redis:
image: redis:5
worker:
build: .
restart: always
command: poetry run python manage.py rundramatiq
volumes:
- .:/opt/app
depends_on:
- redis

33
jeeves/core/models.py
View File

@ -45,6 +45,13 @@ class Flow(BaseModel, DefinitionMixin):
self.definition = definition
self.save()
def execute(self):
from jeeves.core.tasks import start_execution
execution = Run.from_flow(self)
start_execution.send(execution_id=str(execution.pk))
return execution
class TaskManager(models.Manager):
def from_reference(self, reference):
@ -65,9 +72,9 @@ class Task(DefinitionMixin, BaseModel):
def instance(self):
return TaskRegistry.get_task(self.type, **self.definition)
def run(self, message=None):
def run(self):
runner = self.instance
return FlowStep(str(self.pk), *runner.execute(message))
return FlowStep(str(self.pk), *runner.execute())
class Run(DefinitionMixin, BaseModel):
@ -95,22 +102,12 @@ class Run(DefinitionMixin, BaseModel):
self._result = json.dumps(value)
@classmethod
def execute_flow(cls, flow: Flow):
run = cls()
result = Result()
run.definition = flow.serialize()
run.flow = flow
for task in run.flow.tasks:
flow_step = task.run()
result.add_step(flow_step)
run.output += flow_step.output
if flow_step.error:
run.success = False
break
run._result = json.dumps(result.serialize())
run.status = Run.FINISHED
run.save()
return run
def from_flow(cls, flow: Flow):
execution = cls()
execution.definition = flow.serialize()
execution.flow = flow
execution.save()
return execution
class FlowStep:

22
jeeves/core/tasks.py Normal file
View File

@ -0,0 +1,22 @@
import json
import dramatiq
from jeeves.core.models import Run, Result
@dramatiq.actor
def start_execution(execution_id):
run = Run.objects.get(pk=execution_id)
result = Result()
for task in run.flow.tasks:
flow_step = task.run()
result.add_step(flow_step)
run.output += flow_step.output
if flow_step.error:
run.success = False
break
run._result = json.dumps(result.serialize())
run.status = Run.FINISHED
run.save()
return run

6
jeeves/frontend/views.py
View File

@ -49,8 +49,10 @@ class FlowsCRUD:
@classmethod
def run(cls, request, uuid):
flow = Flow.objects.get(pk=uuid)
run = Run.execute_flow(flow)
messages.add_message(request, messages.SUCCESS, f"Flow running: {str(run.pk)}")
execution = flow.execute()
messages.add_message(
request, messages.SUCCESS, f"Flow running: {str(execution.pk)}"
)
return redirect("flow-execution-list", uuid=uuid)
@classmethod

20
jeeves/settings.py
View File

@ -31,6 +31,7 @@ INSTALLED_APPS = [
"django.contrib.messages",
"django.contrib.staticfiles",
# Third party
"django_dramatiq",
# Own
"jeeves.core",
"jeeves.frontend",
@ -105,3 +106,22 @@ USE_TZ = True
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = "/static/"
# Dramatiq
DRAMATIQ_BROKER = {
"BROKER": "dramatiq.brokers.redis.RedisBroker",
"OPTIONS": {"url": "redis://redis:6379/0"},
"MIDDLEWARE": [
"dramatiq.middleware.Prometheus",
"dramatiq.middleware.AgeLimit",
"dramatiq.middleware.TimeLimit",
"dramatiq.middleware.Callbacks",
"dramatiq.middleware.Retries",
"django_dramatiq.middleware.AdminMiddleware",
"django_dramatiq.middleware.DbConnectionsMiddleware",
],
}
# Defines which database should be used to persist Task objects when the
# AdminMiddleware is enabled. The default value is "default".
DRAMATIQ_TASKS_DATABASE = "default"

91
poetry.lock generated
View File

@ -1,5 +1,5 @@
[[package]]
category = "main"
category = "dev"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
name = "appdirs"
optional = false
@ -35,7 +35,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "1.3.0"
[[package]]
category = "main"
category = "dev"
description = "Classes Without Boilerplate"
name = "attrs"
optional = false
@ -52,7 +52,7 @@ python-versions = "*"
version = "0.1.0"
[[package]]
category = "main"
category = "dev"
description = "The uncompromising code formatter."
name = "black"
optional = false
@ -77,7 +77,7 @@ version = "2.0.1"
six = "*"
[[package]]
category = "main"
category = "dev"
description = "Composable command line interface toolkit"
name = "click"
optional = false
@ -114,6 +114,18 @@ version = "2.2.5"
pytz = "*"
sqlparse = "*"
[[package]]
category = "main"
description = "A Django app for Dramatiq."
name = "django-dramatiq"
optional = false
python-versions = ">=3.5"
version = "0.8.0"
[package.dependencies]
django = ">=1.11"
dramatiq = ">=0.18.0"
[[package]]
category = "main"
description = "Django-environ allows you to utilize 12factor inspired environment variables to configure your Django application."
@ -122,6 +134,18 @@ optional = false
python-versions = "*"
version = "0.4.5"
[[package]]
category = "main"
description = "Background Processing for Python 3."
name = "dramatiq"
optional = false
python-versions = ">=3.5"
version = "1.7.0"
[package.dependencies]
prometheus-client = ">=0.2,<0.8"
redis = ">=2.0,<4.0"
[[package]]
category = "dev"
description = "Discover and load entry points from installed packages."
@ -229,6 +253,25 @@ version = "0.15.1"
[package.dependencies]
parso = ">=0.5.0"
[[package]]
category = "main"
description = "A small but fast and easy to use stand-alone template engine written in pure python."
name = "jinja2"
optional = false
python-versions = "*"
version = "2.10.1"
[package.dependencies]
MarkupSafe = ">=0.23"
[[package]]
category = "main"
description = "Safely add untrusted strings to HTML/XML markup."
name = "markupsafe"
optional = false
python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"
version = "1.1.1"
[[package]]
category = "dev"
description = "McCabe checker, plugin for flake8"
@ -327,6 +370,14 @@ six = "*"
toml = "*"
virtualenv = ">=15.2"
[[package]]
category = "main"
description = "Python client for the Prometheus monitoring system."
name = "prometheus-client"
optional = false
python-versions = "*"
version = "0.7.1"
[[package]]
category = "dev"
description = "Library for building powerful interactive command lines in Python"
@ -365,6 +416,14 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.5.0"
[[package]]
category = "main"
description = "Data validation and settings management using python 3.6 type hinting"
name = "pydantic"
optional = false
python-versions = ">=3.6"
version = "0.32.2"
[[package]]
category = "dev"
description = "passive checker of Python programs"
@ -396,7 +455,7 @@ description = "pytest: simple powerful testing with Python"
name = "pytest"
optional = false
python-versions = ">=3.5"
version = "5.1.3"
version = "5.2.0"
[package.dependencies]
atomicwrites = ">=1.0"
@ -450,6 +509,15 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "5.1.2"
[[package]]
category = "main"
description = "Python client for Redis key-value store"
marker = "extra == \"redis\""
name = "redis"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "3.3.8"
[[package]]
category = "dev"
description = "a python refactoring library..."
@ -475,7 +543,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.3.0"
[[package]]
category = "main"
category = "dev"
description = "Python Library for Tom's Obvious, Minimal Language"
name = "toml"
optional = false
@ -524,7 +592,7 @@ version = "0.6.0"
more-itertools = "*"
[metadata]
content-hash = "40f0c0a3f3e7900fd92973ba513b9d4449452e66cf462d9cba9c5299f961c3d5"
content-hash = "d8b854354df97bc8300f8e8de3b745dc3ff1e98f4e99a8c792aa3348955061c7"
python-versions = "^3.7"
[metadata.hashes]
@ -540,7 +608,9 @@ click = ["2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", "5b
colorama = ["05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", "f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48"]
decorator = ["86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", "f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6"]
django = ["148a4a2d1a85b23883b0a4e99ab7718f518a83675e4485e44dc0c1d36988c5fa", "deb70aa038e59b58593673b15e9a711d1e5ccd941b5973b30750d5d026abfd56"]
django-dramatiq = ["492ef6e216ca6ce1c5b09447b2be554ad299207d625649f2e934d28f96af4db9", "5f4f11027413cb3043254a5fa91f0e79db7482a3a1e4b80e55fcdb3f7206689e"]
django-environ = ["6c9d87660142608f63ec7d5ce5564c49b603ea8ff25da595fd6098f6dc82afde", "c57b3c11ec1f319d9474e3e5a79134f40174b17c7cc024bbb2fad84646b120c4"]
dramatiq = ["6d4b21b52baa275c4664126237b3f0029545815b84553904814e43b3b2f58b31", "9cec395ac460eb804e3cca6021cb274011b085843fa85d48d94e9339f7bc2ac3"]
entrypoints = ["589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", "c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"]
flake8 = ["19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548", "8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696"]
identify = ["4f1fe9a59df4e80fcb0213086fcf502bc1765a01ea4fe8be48da3b65afd2a017", "d8919589bd2a5f99c66302fec0ef9027b12ae150b0b0213999ad3f695fc7296e"]
@ -550,6 +620,8 @@ ipython = ["c4ab005921641e40a68e405e286e7a1fcc464497e14d81b6914b4fd95e5dee9b", "
ipython-genutils = ["72dd37233799e619666c9f639a9da83c34013a73e8bbc79a7a6348d93c61fab8", "eb2e116e75ecef9d4d228fdc66af54269afa26ab4463042e33785b887c628ba8"]
isort = ["54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", "6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"]
jedi = ["786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27", "ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e"]
jinja2 = ["065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", "14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b"]
markupsafe = ["00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", "09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", "09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", "24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", "43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", "535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", "62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", "6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", "717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", "7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", "88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", "8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", "98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", "9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", "9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", "ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", "b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", "b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", "b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", "ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", "c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", "cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", "e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7"]
mccabe = ["ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", "dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"]
more-itertools = ["409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", "92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"]
nodeenv = ["ad8259494cf1c9034539f6cced78a1da4840a4b157e23640bc4a0c0546b0cb7a"]
@ -559,18 +631,21 @@ pexpect = ["2094eefdfcf37a1fdbfb9aa090862c1a4878e5c7e0e7e7088bdb511c558e5cd1", "
pickleshare = ["87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", "9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"]
pluggy = ["0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", "fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34"]
pre-commit = ["1d3c0587bda7c4e537a46c27f2c84aa006acc18facf9970bf947df596ce91f3f", "fa78ff96e8e9ac94c748388597693f18b041a181c94a4f039ad20f45287ba44a"]
prometheus-client = ["71cd24a2b3eb335cb800c7159f423df1bd4dcd5171b234be15e3f31ec9f622da"]
prompt-toolkit = ["11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780", "2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1", "977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55"]
ptyprocess = ["923f299cc5ad920c68f2bc0bc98b75b9f838b93b599941a6b63ddbc2476394c0", "d7cc528d76e76342423ca640335bd3633420dc1366f258cb31d05e865ef5ca1f"]
py = ["64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", "dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53"]
pycodestyle = ["95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", "e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c"]
pydantic = ["18598557f0d9ab46173045910ed50458c4fb4d16153c23346b504d7a5b679f77", "6a9335c968e13295430a208487e74d69fef40168b72dea8d975765d14e2da660", "6f5eb88fe4c21380aa064b7d249763fc6306f0b001d7e7d52d80866d1afc9ed3", "bc6c6a78647d7a65a493e1107572d993f26a652c49183201e3c7d23924bf7311", "e1a63b4e6bf8820833cb6fa239ffbe8eec57ccdd7d66359eff20e68a83c1deeb", "ede2d65ae33788d4e26e12b330b4a32c53cb14131c65bca3a59f037c73f6ee7a"]
pyflakes = ["17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", "d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2"]
pygments = ["71e430bc85c88a430f000ac1d9b331d2407f681d6f6aec95e8bcfbc3df5b0127", "881c4c157e45f30af185c1ffe8d549d48ac9127433f2c380c24b84572ad66297"]
pyparsing = ["6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", "d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4"]
pytest = ["813b99704b22c7d377bbd756ebe56c35252bb710937b46f207100e843440b3c2", "cc6620b96bc667a0c8d4fa592a8c9c94178a1bd6cc799dbb057dfd9286d31a31"]
pytest = ["13c1c9b22127a77fc684eee24791efafcef343335d855e3573791c68588fe1a5", "d8ba7be9466f55ef96ba203fc0f90d0cf212f2f927e69186e1353e30bc7f62e5"]
pytest-django = ["264fb4c506db5d48a6364c311a0b00b7b48a52715bad8839b2d8bee9b99ed6bb", "4adfe5fb3ed47f0ba55506dd3daf688b1f74d5e69148c10ad2dd2f79f40c0d62"]
pytest-tldr = ["008b7d53cabbce9d49ee7a92754ed4adafc056bc49ae01b257c2ffb1f5ce2408", "dca4a464a002f389677f4c42f5b9c815aae43219d73ecfe6b7fffe2d190e38eb"]
pytz = ["26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32", "c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7"]
pyyaml = ["0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", "01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", "5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", "5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", "7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", "7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", "87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", "9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", "a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", "b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", "b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", "bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", "f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8"]
redis = ["98a22fb750c9b9bb46e75e945dc3f61d0ab30d06117cbb21ff9cd1d315fedd3b", "c504251769031b0dd7dd5cf786050a6050197c6de0d37778c80c08cb04ae8275"]
rope = ["6b728fdc3e98a83446c27a91fc5d56808a004f8beab7a31ab1d7224cecc7d969", "c5c5a6a87f7b1a2095fb311135e2a3d1f194f5ecb96900fdd0a9100881f48aaf", "f0dcf719b63200d492b85535ebe5ea9b29e0d0b8aebeb87fe03fc1a65924fdaf"]
six = ["3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", "d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"]
sqlparse = ["40afe6b8d4b1117e7dff5504d7a8ce07d9a1b15aeeade8a2d10f130a834f8177", "7c3dca29c022744e95b547e867cee89f4fce4373f3549ccd8797d8eb52cdb873"]

3
pyproject.toml
View File

@ -10,6 +10,9 @@ django = "^2.2"
django-environ = "^0.4.5"
jinja2 = "^2.10"
pydantic = "^0.32.2"
django_dramatiq = "^0.8.0"
dramatiq = {version = "^1.7", extras = ["redis"]}
[tool.poetry.dev-dependencies]
black = {version = "^18.3-alpha.0", allows-prereleases = true}
flake8 = "^3.7"

2
setup.cfg
View File

@ -11,6 +11,6 @@ include_trailing_comma = True
length_sort = 1
lines_between_types = 0
line_length = 88
known_third_party = django,environ,pydantic,pytest
known_third_party = django,dramatiq,environ,pydantic,pytest
sections = FUTURE, STDLIB, DJANGO, THIRDPARTY, FIRSTPARTY, LOCALFOLDER
no_lines_before = LOCALFOLDER