Add: run e2e test for local, git, github and gitlab storage backends (#348)

And while at it, cache the Python/Playwright installation files,
to have everything up and running a lot quicker.

Author: TrueBrain

.coveragerc

@@ -0,0 +1,5 @@
+[run]
+branch = True
+concurrency = multiprocessing
+parallel = True
+source = truewiki

.github/workflows/end-to-end.yml

@@ -12,31 +12,46 @@ jobs:
   playwright:
     name: Playwright
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        storage: [local, git, github, gitlab]
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
     - name: Set up Python 3.8
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: 3.8
+        cache: 'pip'
+        cache-dependency-path: |
+          e2e/requirements.base
+          requirements.txt
+    - uses: actions/cache@v3
+      name: Cache Playwright
+      id: playwright-cache
+      with:
+        path: '~/.cache/ms-playwright'
+        key: '${{ runner.os }}-playwright'
     - name: Set up packages
       run: |
         python -m pip install --upgrade pip
         pip install -r e2e/requirements.base
         pip install -r requirements.txt
         playwright install
-    - name: Playwright
+    - name: Set up remote SSH keys
       run: |
-        COVERAGE_FILE="$(pwd)/.coverage" pytest e2e
-    - name: Coverage report
+        mkdir -p ~/.ssh
+        ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts
+        ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts
+    - name: Playwright (storage=${{ matrix.storage }})
       run: |
+        COVERAGE_RCFILE="$(pwd)/.coveragerc" COVERAGE_FILE="$(pwd)/.coverage" pytest e2e -v --storage ${{ matrix.storage }}
+        coverage combine
         coverage report -m
-        coverage html
         coverage xml
-    - name: Archive coverage report
-      uses: actions/upload-artifact@v3
-      with:
-        name: coverage-report
-        path: htmlcov
+      env:
+        TRUEWIKI_STORAGE_GITHUB_PRIVATE_KEY: ${{ secrets.E2E_STORAGE_GITHUB_PRIVATE_KEY }}
+        TRUEWIKI_STORAGE_GITLAB_PRIVATE_KEY: ${{ secrets.E2E_STORAGE_GITLAB_PRIVATE_KEY }}
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3

.gitignore

@@ -2,6 +2,7 @@ __pycache__/
 *.pyc
 /.cache_metadata.json
 /.coverage
+/.coverage.*
 /.env/
 /data/
 /cache/

README.md

@@ -286,7 +286,8 @@ This is located in the `e2e` folder, and requires some additional tooling to be
 ```bash
 .env/bin/pip install -r e2e/requirements.txt
 .env/bin/playwright install
-COVERAGE_FILE="$(pwd)/.coverage" pytest e2e
+COVERAGE_RCFILE="$(pwd)/.coveragerc" COVERAGE_FILE="$(pwd)/.coverage" pytest e2e -v --storage local
+coverage combine
 coverage report -m
 ```
 

e2e/conftest.py

@@ -4,6 +4,7 @@
 import pytest
 import shutil
 import signal
+import time
 
 from ctypes.util import find_library
 from tempfile import TemporaryDirectory
@@ -21,7 +22,7 @@ async def set_death_signal():
     libc.prctl(PR_SET_PDEATHSIG, signal.SIGTERM, 0, 0, 0)
 
 
-async def _run_server():
+async def _run_server(storage):
     global temp_folder, python_proc
 
     # Make sure we can find our way to the code in the temporary folder.
@@ -43,18 +44,22 @@ async def _run_server():
         f.write("end-to-end test file")
     # Make sure a few languages exist.
     os.makedirs("data/Page/en")
+    with open("data/Page/en/.keep", "w") as f:
+        pass
     os.makedirs("data/Page/de")
+    with open("data/Page/de/.keep", "w") as f:
+        pass
 
     # Run TrueWiki, with coverage enabled.
-    command = ["coverage", "run", "--branch", "--source", "truewiki"]
+    command = ["coverage", "run"]
     command.extend(
         [
             "-m",
             "truewiki",
             "--port",
             "8080",
             "--storage",
-            "local",
+            storage,
             "--user",
             "developer",
             "--frontend-url",
@@ -63,6 +68,35 @@ async def _run_server():
             "cache/",
         ]
     )
+
+    if storage == "github":
+        branch_name = "e2e-" + str(time.time()).replace(".", "-")
+
+        command.extend(
+            [
+                "--storage-github-url",
+                "git@github.com:TrueBrain/truewiki-e2e-test.git",
+                "--storage-github-history-url",
+                "https://github.com/TrueBrain/truewiki-e2e-test",
+                "--storage-github-branch",
+                branch_name,
+            ]
+        )
+
+    if storage == "gitlab":
+        branch_name = "e2e-" + str(time.time()).replace(".", "-")
+
+        command.extend(
+            [
+                "--storage-gitlab-url",
+                "git@gitlab.com:TrueBrain/truewiki-e2e-test.git",
+                "--storage-gitlab-history-url",
+                "https://gitlab.com/TrueBrain/truewiki-e2e-test",
+                "--storage-gitlab-branch",
+                branch_name,
+            ]
+        )
+
     python_proc = await asyncio.create_subprocess_exec(
         command[0],
         *command[1:],
@@ -72,17 +106,34 @@ async def _run_server():
     # Make sure we switch back to the project folder.
     os.chdir(cwd)
 
-    # Wait for the startup line to be shown.
-    await python_proc.stdout.readline()
+    # Wait till the server reports "running".
+    lines = []
+    while python_proc.returncode is None and not python_proc.stdout.at_eof():
+        line = await python_proc.stdout.readline()
+        if line is None:
+            break
+        lines.append(line)
+
+        if "Running on http://" in line.decode():
+            break
+
+    # If either the process closed or the stdout closed, the server didn't
+    # actual start. And yes, it can happen stdout is already closed, but
+    # the returncode isn't in yet.
+    if python_proc.returncode is not None or python_proc.stdout.at_eof():
+        print((b"\n".join(lines) + await python_proc.stdout.read()).decode())
+        raise Exception("Failed to start TrueWiki server")
+
+    return None
 
 
 @pytest.fixture(scope="session", autouse=True)
-def run_server():
+def run_server(request):
     # Create a new event loop for our server.
     loop = asyncio.new_event_loop()
 
     # Start the server.
-    loop.run_until_complete(_run_server())
+    loop.run_until_complete(_run_server(request.config.getoption("--storage")))
 
     # Give control back to PyTest.
     yield
@@ -93,6 +144,10 @@ def run_server():
     temp_folder.cleanup()
 
 
+def pytest_addoption(parser):
+    parser.addoption("--storage", action="store", default="local", help="Storage backend")
+
+
 @pytest.fixture(autouse=True)
 def timeout(page: Page):
     page.set_default_timeout(1000)