Taskgraph (#534)

escapewindow · web-flow · commit da8e6dff06dd · 2022-04-20T10:17:05.000-07:00
* add py310 to tox

* migrate to taskgraph

* no need to specify 3.9.7

* fix tox kind

* let's not hammer github api
diff --git a/.taskcluster.yml b/.taskcluster.yml
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -24,3 +24,4 @@ recursive-exclude * __pycache__
 recursive-exclude * *.py[co]
 recursive-exclude docs *
 recursive-exclude tests *
+recursive-exclude taskcluster *
diff --git a/src/scriptworker/task.py b/src/scriptworker/task.py
@@ -16,10 +16,10 @@
 from copy import deepcopy
 
 import aiohttp
-import taskcluster
 import taskcluster.exceptions
 from taskcluster.exceptions import TaskclusterFailure
 
+import taskcluster
 from scriptworker.constants import get_reversed_statuses
 from scriptworker.exceptions import ScriptWorkerTaskException, WorkerShutdownDuringTask
 from scriptworker.github import (
diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml
@@ -0,0 +1,37 @@
+---
+trust-domain: scriptworker
+
+task-priority: high
+
+taskgraph:
+    register: scriptworker_taskgraph:register
+    repositories:
+        scriptworker:
+            name: "Scriptworker"
+            project-regex: scriptworker$
+            default-repository: https://github.com/mozilla-releng/scriptworker
+            default-ref: master
+            type: git
+
+workers:
+    aliases:
+        b-linux:
+            provisioner: scriptworker-{level}
+            implementation: docker-worker
+            os: linux
+            worker-type: 'b-linux'
+        images:
+            provisioner: scriptworker-{level}
+            implementation: docker-worker
+            os: linux
+            worker-type: 'images'
+
+notify:
+    email:
+        - release+scriptworker@mozilla.com
+    prefix: "[scriptworker]"
+
+private-artifact-prefix: releng/scriptworker
+
+scriptworker:
+    scope-prefix: project:scriptworker:releng
diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml
@@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.docker_image:transforms
+    - taskgraph.transforms.cached_tasks:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    python3.10:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.10"
+    python3.9:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.9"
+    python3.8:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.8"
+    python3.7:
+        definition: python
+        args:
+            PYTHON_VERSION: "3.7"
diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml
@@ -0,0 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - tox
+
+transforms:
+    - taskgraph.transforms.code_review:transforms
+    - taskgraph.transforms.task:transforms
+
+jobs:
+    complete:
+        description: PR Summary Task
+        run-on-tasks-for: [github-pull-request]
+        worker-type: succeed
diff --git a/taskcluster/ci/tox/kind.yml b/taskcluster/ci/tox/kind.yml
@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - docker-image
+
+transforms:
+    - scriptworker_taskgraph.transforms.tox:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    description: "{name} tox-{targets}"
+    run-on-tasks-for: ["action", "github-pull-request", "github-push"]
+    attributes:
+        code-review: true
+    worker-type: b-linux
+    worker:
+        docker-image: {in-tree: 'python{python-version}'}
+        max-run-time: 1800
+    run:
+        using: run-task
+        cache-dotcache: false
+        checkout:
+            scriptworker: {}
+        # sparse-profile: str/none
+        # workdir:
+        cwd: '{checkout}'
+        command:
+            - sh
+            - -lxce
+            - >-
+              tox -e {targets}
+
+jobs:
+    py37:
+        python-version: "3.7"
+        targets: py37,check
+        env:
+            NO_TESTS_OVER_WIRE: "1"
+    py38:
+        python-version: "3.8"
+        targets: py38,check
+        env:
+            NO_TESTS_OVER_WIRE: "1"
+    py39:
+        python-version: "3.9"
+        targets: py39,check
+        env:
+            NO_TESTS_OVER_WIRE: "1"
+    py310:
+        python-version: "3.10"
+        targets: py310,check
+        env:
+            NO_TESTS_OVER_WIRE: "1"
+    py38-cot:
+        python-version: "3.8"
+        targets: py38-cot
+        env:
+            NO_CREDENTIALS_TESTS: "1"
+        scopes:
+            - secrets:get:repo:github.com/mozilla-releng/scriptworker:github
+    py39-cot:
+        python-version: "3.9"
+        targets: py39-cot
+        env:
+            NO_CREDENTIALS_TESTS: "1"
+        scopes:
+            - secrets:get:repo:github.com/mozilla-releng/scriptworker:github
diff --git a/taskcluster/docker/REGISTRY b/taskcluster/docker/REGISTRY
@@ -0,0 +1 @@
+mozilla-releng
diff --git a/taskcluster/docker/python/Dockerfile b/taskcluster/docker/python/Dockerfile
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+ARG  PYTHON_VERSION
+FROM python:$PYTHON_VERSION
+
+# Add worker user
+RUN mkdir /builds && \
+    groupadd -g 1000 -o worker && \
+    useradd -d /builds/worker -s /bin/bash -m worker -g 1000 -o -u 1000 && \
+    mkdir /builds/worker/artifacts && \
+    chown worker:worker /builds/worker/artifacts
+
+RUN pip install tox
+
+# %include-run-task
+
+ENV SHELL=/bin/bash \
+    HOME=/builds/worker \
+    PATH=/builds/worker/.local/bin:$PATH
+
+VOLUME /builds/worker/checkouts
+VOLUME /builds/worker/.cache
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
diff --git a/taskcluster/requirements.in b/taskcluster/requirements.in
@@ -0,0 +1,4 @@
+# For instructions on managing dependencies, see:
+# https://taskcluster-taskgraph.readthedocs.io/en/latest/howto/bootstrap-taskgraph.html
+
+taskcluster-taskgraph
diff --git a/taskcluster/requirements.txt b/taskcluster/requirements.txt
@@ -0,0 +1,97 @@
+#
+# This file is autogenerated by pip-compile with python 3.6
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in
+#
+appdirs==1.4.4 \
+    --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \
+    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128
+    # via taskcluster-taskgraph
+attrs==21.4.0 \
+    --hash=sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4 \
+    --hash=sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd
+    # via taskcluster-taskgraph
+certifi==2021.10.8 \
+    --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872 \
+    --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569
+    # via requests
+charset-normalizer==2.0.12 \
+    --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \
+    --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df
+    # via requests
+idna==3.3 \
+    --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \
+    --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d
+    # via requests
+json-e==4.4.3 \
+    --hash=sha256:8ed3974faa887ca96a7987298f6550cf2ad35472419a980766b3abe48258de0a
+    # via taskcluster-taskgraph
+pyyaml==6.0 \
+    --hash=sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293 \
+    --hash=sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b \
+    --hash=sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57 \
+    --hash=sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b \
+    --hash=sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4 \
+    --hash=sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07 \
+    --hash=sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba \
+    --hash=sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9 \
+    --hash=sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287 \
+    --hash=sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513 \
+    --hash=sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0 \
+    --hash=sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0 \
+    --hash=sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92 \
+    --hash=sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f \
+    --hash=sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2 \
+    --hash=sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc \
+    --hash=sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c \
+    --hash=sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86 \
+    --hash=sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4 \
+    --hash=sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c \
+    --hash=sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34 \
+    --hash=sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b \
+    --hash=sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c \
+    --hash=sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb \
+    --hash=sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737 \
+    --hash=sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3 \
+    --hash=sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d \
+    --hash=sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53 \
+    --hash=sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78 \
+    --hash=sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803 \
+    --hash=sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a \
+    --hash=sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174 \
+    --hash=sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5
+    # via taskcluster-taskgraph
+redo==2.0.4 \
+    --hash=sha256:81066955041c853b0e6491eb65a0877dce45131c4cfa3d42d923fc2aa8f7a043
+    # via taskcluster-taskgraph
+requests==2.27.1 \
+    --hash=sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61 \
+    --hash=sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d
+    # via
+    #   requests-unixsocket
+    #   taskcluster-taskgraph
+requests-unixsocket==0.3.0 \
+    --hash=sha256:28304283ea9357d45fff58ad5b11e47708cfbf5806817aa59b2a363228ee971e \
+    --hash=sha256:c685c680f0809e1b2955339b1e5afc3c0022b3066f4f7eb343f43a6065fc0e5d
+    # via taskcluster-taskgraph
+slugid==2.0.0 \
+    --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297 \
+    --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c
+    # via taskcluster-taskgraph
+taskcluster-taskgraph==1.1.7 \
+    --hash=sha256:648e2cd5a92ce3412d830d1a28fb41ba3415956f44fe6013a53d920fc313ebdf \
+    --hash=sha256:7025856ed543c8e4a42fae580d579feadb53a54ce91b6e6b994e678449be3aee
+    # via -r requirements.in
+taskcluster-urls==13.0.1 \
+    --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \
+    --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \
+    --hash=sha256:f66dcbd6572a6216ab65949f0fa0b91f2df647918028436c384e6af5cd12ae2b
+    # via taskcluster-taskgraph
+urllib3==1.26.8 \
+    --hash=sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed \
+    --hash=sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c
+    # via requests
+voluptuous==0.12.2 \
+    --hash=sha256:4db1ac5079db9249820d49c891cb4660a6f8cae350491210abce741fabf56513
+    # via taskcluster-taskgraph
diff --git a/taskcluster/scriptworker_taskgraph/__init__.py b/taskcluster/scriptworker_taskgraph/__init__.py
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from importlib import import_module
+
+
+def register(graph_config):
+    """
+    Import all modules that are siblings of this one, triggering decorators in
+    the process.
+    """
+    _import_modules(
+        [
+            "transforms",
+        ]
+    )
+
+
+def _import_modules(modules):
+    for module in modules:
+        import_module(f".{module}", package=__name__)
diff --git a/taskcluster/scriptworker_taskgraph/transforms/__init__.py b/taskcluster/scriptworker_taskgraph/transforms/__init__.py
diff --git a/taskcluster/scriptworker_taskgraph/transforms/tox.py b/taskcluster/scriptworker_taskgraph/transforms/tox.py
diff --git a/tests/test_context.py b/tests/test_context.py
diff --git a/tox.ini b/tox.ini
