Merge branch 'develop' into synpy-1636-move-to-electron

Author: BryanFauble

.github/scripts/delete_evaluations.py

@@ -4,7 +4,7 @@
 from synapseclient import Evaluation, Synapse
 
 syn = Synapse()
-syn.login()
+syn.login(profile=None)
 
 # Maximum number of concurrent deletion operations
 MAX_CONCURRENT_DELETIONS = 5

.github/scripts/delete_projects.py

@@ -4,7 +4,7 @@
 from synapseclient import Synapse
 
 syn = Synapse()
-syn.login()
+syn.login(profile=None)
 
 # Maximum number of concurrent deletion operations
 MAX_CONCURRENT_DELETIONS = 5

.github/scripts/delete_teams.py

@@ -4,7 +4,7 @@
 from synapseclient import Synapse
 
 syn = Synapse()
-syn.login()
+syn.login(profile=None)
 
 # Maximum number of concurrent team deletions
 MAX_CONCURRENT_DELETIONS = 5

.github/scripts/empty_trash.py

@@ -4,7 +4,7 @@
 from synapseclient import Synapse
 
 syn = Synapse()
-syn.login()
+syn.login(profile=None)
 
 # Maximum number of concurrent deletion operations
 MAX_CONCURRENT_DELETIONS = 5

.github/workflows/build.yml

@@ -34,27 +34,27 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.13'
-    - uses: pre-commit/action@v3.0.1
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - uses: pre-commit/action@v3.0.1
 
   # run unit (and integration tests if account secrets available) on our build matrix
   test:
     needs: [pre-commit]
 
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-22.04, macos-13, windows-2022]
 
         # if changing the below change the run-integration-tests versions and the check-deploy versions
         # Make sure that we are running the integration tests on the first and last versions of the matrix
-        python: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
     runs-on: ${{ matrix.os }}
 
@@ -83,15 +83,15 @@ jobs:
           path: |
             ${{ steps.get-dependencies.outputs.site_packages_loc }}
             ${{ steps.get-dependencies.outputs.site_bin_dir }}
-          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v24
+          key: ${{ runner.os }}-${{ matrix.python }}-build-${{ env.cache-name }}-${{ hashFiles('setup.py') }}-v29
 
       - name: Install py-dependencies
         if: steps.cache-dependencies.outputs.cache-hit != 'true'
         shell: bash
         run: |
           python -m pip install --upgrade pip
 
-          pip install -e ".[boto3,pandas,pysftp,tests]"
+          pip install -e ".[boto3,pandas,pysftp,tests,curator]"
 
           # ensure that numpy c extensions are installed on windows
           # https://stackoverflow.com/a/59346525
@@ -108,7 +108,7 @@ jobs:
           pytest -sv --cov-append --cov=. --cov-report xml tests/unit
       - name: Check for Secret availability
         id: secret-check
-        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) || contains(fromJSON('["3.13"]'), matrix.python) }}
+        if: ${{ contains(fromJSON('["3.10"]'), matrix.python) || contains(fromJSON('["3.14"]'), matrix.python) }}
         # perform secret check & put boolean result as an output
         shell: bash
         run: |
@@ -124,17 +124,48 @@ jobs:
             echo "synapse_pat_available=true" >> $GITHUB_OUTPUT;
           fi
 
+      - name: Download Failed Tests from Previous Attempt
+        if: ${{ github.run_attempt > 1 && (contains(fromJSON('["3.10"]'), matrix.python) || contains(fromJSON('["3.14"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true'}}
+        uses: actions/download-artifact@v4
+        with:
+          name: failed-tests-${{ matrix.os }}-${{ matrix.python }}
+        continue-on-error: true
+
       # run integration tests iff the decryption keys for the test configuration are available.
       # they will not be available in pull requests from forks.
       # run integration tests on the oldest and newest supported versions of python.
       # we don't run on the entire matrix to avoid a 3xN set of concurrent tests against
       # the target server where N is the number of supported python versions.
       - name: run-integration-tests
+        id: integration_tests
         shell: bash
 
         # keep versions consistent with the first and last from the strategy matrix
-        if: ${{ (contains(fromJSON('["3.9"]'), matrix.python) || contains(fromJSON('["3.13"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true'}}
+        if: ${{ (contains(fromJSON('["3.10"]'), matrix.python) || contains(fromJSON('["3.14"]'), matrix.python)) && steps.secret-check.outputs.secrets_available == 'true'}}
         run: |
+          # Set SYNAPSE_PROFILE based on OS and Python version
+          if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
+            if [ "${{ matrix.python }}" == "3.10" ]; then
+              export SYNAPSE_PROFILE="TestUbuntuMinimumPython"
+            elif [ "${{ matrix.python }}" == "3.14" ]; then
+              export SYNAPSE_PROFILE="TestUbuntuMaximumPython"
+            fi
+          elif [ "${{ startsWith(matrix.os, 'windows') }}" == "true" ]; then
+            if [ "${{ matrix.python }}" == "3.10" ]; then
+              export SYNAPSE_PROFILE="TestWindowsMinimumPython"
+            elif [ "${{ matrix.python }}" == "3.14" ]; then
+              export SYNAPSE_PROFILE="TestWindowsMaximumPython"
+            fi
+          elif [ "${{ startsWith(matrix.os, 'macos') }}" == "true" ]; then
+            if [ "${{ matrix.python }}" == "3.10" ]; then
+              export SYNAPSE_PROFILE="TestMacosMinimumPython"
+            elif [ "${{ matrix.python }}" == "3.14" ]; then
+              export SYNAPSE_PROFILE="TestMacosMaximumPython"
+            fi
+          fi
+
+          echo "Using SYNAPSE_PROFILE: $SYNAPSE_PROFILE"
+
           # decrypt the encrypted test synapse configuration
           openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
           mv test.synapseConfig ~/.synapseConfig
@@ -170,21 +201,96 @@ jobs:
           # Setup ignore patterns based on Python version
           IGNORE_FLAGS="--ignore=tests/integration/synapseclient/test_command_line_client.py"
 
-          if [ "${{ matrix.python }}" == "3.9" ]; then
-            # For min Python version, ignore async tests
-            IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/async/"
-            echo "Running integration tests for Min Python version (3.9) - ignoring async tests"
-          elif [ "${{ matrix.python }}" == "3.13" ]; then
-            # For max Python version, ignore synchronous tests
-            IGNORE_FLAGS="$IGNORE_FLAGS --ignore=tests/integration/synapseclient/models/synchronous/"
-            echo "Running integration tests for Max Python version (3.13) - ignoring synchronous tests"
-          fi
+          # Check if we should run only failed tests from previous attempt
+          if [[ -f failed_tests.txt && -s failed_tests.txt ]]; then
+            echo "::notice::Retry attempt ${{ github.run_attempt }} detected - running only previously failed tests"
+            cat failed_tests.txt
 
-          # use loadscope to avoid issues running tests concurrently that share scoped fixtures
-          pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration -n 8 $IGNORE_FLAGS --dist loadscope
+            # Run only the failed tests
+            pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml \
+              --junit-xml=test-results.xml \
+              -n 8 --dist loadscope \
+              $(cat failed_tests.txt | tr '\n' ' ')
+          else
+            echo "::notice::First attempt or no previous failures - running full integration test suite"
+
+            # use loadscope to avoid issues running tests concurrently that share scoped fixtures
+            pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml \
+              --junit-xml=test-results.xml \
+              tests/integration -n 8 $IGNORE_FLAGS --dist loadscope
+          fi
 
           # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
           pytest -sv --reruns 3 --cov-append --cov=. --cov-report xml tests/integration/synapseclient/test_command_line_client.py
+
+      - name: Extract Failed Tests
+        if: always() && steps.integration_tests.outcome == 'failure'
+        shell: bash
+        run: |
+          python -c "
+          import xml.etree.ElementTree as ET
+          import os
+          tree = ET.parse('test-results.xml')
+          root = tree.getroot()
+          failed = []
+          for testcase in root.iter('testcase'):
+              if testcase.find('failure') is not None or testcase.find('error') is not None:
+                  classname = testcase.get('classname')
+                  name = testcase.get('name')
+                  file_attr = testcase.get('file')
+
+                  # Use the file attribute if available, otherwise convert classname
+                  if file_attr:
+                      # file attribute is already in the correct format
+                      test_path = f'{file_attr}::{classname.split(\".\")[-1]}::{name}'
+                  else:
+                      # Convert classname from dot notation to file path
+                      # e.g., 'tests.integration.foo.test_bar.TestClass' -> 'tests/integration/foo/test_bar.py::TestClass'
+                      parts = classname.split('.')
+                      # Find the test file (usually starts with 'test_')
+                      module_parts = []
+                      class_parts = []
+                      found_test_file = False
+                      for part in parts:
+                          if not found_test_file:
+                              module_parts.append(part)
+                              if part.startswith('test_'):
+                                  found_test_file = True
+                          else:
+                              class_parts.append(part)
+
+                      file_path = '/'.join(module_parts) + '.py'
+                      if class_parts:
+                          test_path = f'{file_path}::{\"::\".join(class_parts)}::{name}'
+                      else:
+                          test_path = f'{file_path}::{name}'
+
+                  failed.append(test_path)
+
+          with open('failed_tests.txt', 'w') as f:
+              f.write('\n'.join(failed))
+          print(f'Found {len(failed)} failed tests')
+          for test in failed:
+              print(f'  - {test}')
+          print(f'Current attempt: ${{ github.run_attempt }}')
+          "
+
+      - name: Upload Failed Tests for Next Attempt
+        if: always() && steps.integration_tests.outcome == 'failure' && github.run_attempt < 3
+        uses: actions/upload-artifact@v4
+        with:
+          name: failed-tests-${{ matrix.os }}-${{ matrix.python }}
+          path: failed_tests.txt
+          retention-days: 2
+          overwrite: true
+
+      - name: Fail job if integration tests failed after all retries
+        if: always() && steps.integration_tests.outcome == 'failure'
+        shell: bash
+        run: |
+          echo "::error::Integration tests failed after ${{ github.run_attempt }} attempt(s)"
+          exit 1
+
       - name: Upload coverage report
         id: upload_coverage_report
         uses: actions/upload-artifact@v4
@@ -201,12 +307,12 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+          fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
       - name: Check coverage-report artifact existence
         id: check_coverage_report
         uses: LIT-Protocol/artifact-exists-action@v0
         with:
-          name: "coverage-report"
+          name: 'coverage-report'
       - name: Download coverage report
         uses: actions/download-artifact@v4
         if: steps.check_coverage_report.outputs.exists == 'true'
@@ -216,21 +322,21 @@ jobs:
         id: check_coverage_xml
         uses: andstor/file-existence-action@v3
         with:
-          files: "coverage.xml"
+          files: 'coverage.xml'
         # This is a workaround described in https://community.sonarsource.com/t/sonar-on-github-actions-with-python-coverage-source-issue/36057
       - name: Override Coverage Source Path for Sonar
         if: steps.check_coverage_xml.outputs.files_exists == 'true'
         run: sed -i "s/<source>\/home\/runner\/work\/synapsePythonClient<\/source>/<source>\/github\/workspace<\/source>/g" coverage.xml
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@master
+        uses: SonarSource/sonarqube-scan-action@v5.3.1
         if: ${{ always() }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
   # on a GitHub release, build the pip package and upload it as a GitHub release asset
   package:
-    needs: [test,pre-commit]
+    needs: [test, pre-commit]
 
     runs-on: ubuntu-22.04
 
@@ -245,7 +351,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.10
 
       - name: set-release-env
         shell: bash
@@ -342,7 +448,6 @@ jobs:
       #     asset_path: dist/${{ steps.build-package.outputs.bdist-package-name }}
       #     asset_content_type: application/zip
 
-
   # build standalone desktop client artifacts for Windows and macOS on release
   build-electron-desktop-clients:
     needs: [test, pre-commit]
@@ -482,7 +587,7 @@ jobs:
         os: [ubuntu-24.04, macos-13, windows-2022]
 
         # python versions should be consistent with the strategy matrix and the runs-integration-tests versions
-        python: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python: ['3.10', '3.11', '3.12', '3.13', '3.14']
 
     runs-on: ${{ matrix.os }}
 

.github/workflows/test-cleanup.yml

@@ -13,6 +13,17 @@ jobs:
     permissions:
       contents: read
 
+    strategy:
+      matrix:
+        synapse_profile:
+          - "TestUbuntuMinimumPython"
+          - "TestUbuntuMaximumPython"
+          - "TestWindowsMinimumPython"
+          - "TestWindowsMaximumPython"
+          - "TestMacosMinimumPython"
+          - "TestMacosMaximumPython"
+          - "" # Empty string for default profile (no SYNAPSE_PROFILE set)
+
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4
@@ -67,17 +78,33 @@ jobs:
         mv test.synapseConfig ~/.synapseConfig
 
     - name: Run evaluation deletion script
+      shell: bash
       run: |
+        if [ -n "${{ matrix.synapse_profile }}" ]; then
+          export SYNAPSE_PROFILE="${{ matrix.synapse_profile }}"
+        fi
         python .github/scripts/delete_evaluations.py
 
     - name: Run project deletion script
+      shell: bash
       run: |
+        if [ -n "${{ matrix.synapse_profile }}" ]; then
+          export SYNAPSE_PROFILE="${{ matrix.synapse_profile }}"
+        fi
         python .github/scripts/delete_projects.py
 
     - name: Run team deletion script
+      shell: bash
       run: |
+        if [ -n "${{ matrix.synapse_profile }}" ]; then
+          export SYNAPSE_PROFILE="${{ matrix.synapse_profile }}"
+        fi
         python .github/scripts/delete_teams.py
 
     - name: Run empty trash script
+      shell: bash
       run: |
+        if [ -n "${{ matrix.synapse_profile }}" ]; then
+          export SYNAPSE_PROFILE="${{ matrix.synapse_profile }}"
+        fi
         python .github/scripts/empty_trash.py

.gitignore

@@ -6,6 +6,7 @@ virtualenvs
 examples_temp
 .DS_Store
 deploy.sh
+.venv/
 
 junk/
 nose.cfg
@@ -33,6 +34,7 @@ coverage.xml
 .ipynb_checkpoints
 *.ipynb
 .env
+test.synapseConfig
 *.spec
 
 synapse-electron/node_modules