Expanding playbook to add more pulsars per Galaxy instance

galaxy.yml

@@ -308,7 +308,7 @@
         name: "tmpreaper in galaxy temp dir"
         minute: "7"
         hour: "*"
-        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_mutable_data_dir }}/tmp >> {{ galaxy_mutable_data_dir }}/cleaning_galaxy_temp_dir.log"
+        job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_mutable_data_dir }}/tmp/ >> {{ galaxy_mutable_data_dir }}/cleaning_galaxy_temp_dir.log"
       become: yes
       become_user: "{{ galaxy_user_name }}"
 #    - name: Creates entry in crontab "tmpreaper call in galaxy jobs dir"

group_vars/galaxyservers.yml

@@ -87,6 +87,16 @@ galaxy_job_config:
       amqp_publish_retry: true
       amqp_publish_retry_max_retries: 60
       manager: _default_
+    pulsar2_tpv_runner:
+      load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner
+      galaxy_url: "https://{{ rabbitmq_hostname }}{{ csnt_galaxy_url_prefix }}"
+      amqp_url: "pyamqp://{{ pulsar.user_name }}:{{ rabbitmq_users_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar2?ssl=1"
+      amqp_acknowledge: true
+      amqp_ack_republish_time: 1200
+      amqp_consumer_timeout: 2
+      amqp_publish_retry: true
+      amqp_publish_retry_max_retries: 60
+      manager: _default_
   handling:
     assign: ['db-skip-locked']
   execution:
@@ -453,7 +463,7 @@ rabbitmq_users:
 
 # TUS
 galaxy_tusd_port: 1080
-galaxy_tus_upload_store: "{{ galaxy_mutable_data_dir }}/tus" # /mnt/data/tus
+galaxy_tus_upload_store: "{{ galaxy_mutable_data_dir }}/tus/" # /mnt/data/tus
 
 #Redis
 galaxy_additional_venv_packages:

host_vars/repeatexplorer-elixir.cerit-sc.cz/vars.yml

@@ -119,13 +119,61 @@ galaxy_config_templates:
     dest: "{{ galaxy_config.galaxy.auth_config_file }}"
   - src: "{{ lookup('first_found', ['templates/'+inventory_hostname+'/config/file_source_templates.yml.j2', 'templates/galaxy/config/file_source_templates.yml.j2']) }}"
     dest: "{{ galaxy_config_dir }}/file_source_templates.yml"
+  - src: templates/galaxy/config/local_data_table_conf.xml.j2
+    dest: "{{ galaxy_config_dir }}/local_data_table_conf.xml"
+#  - src: templates/galaxy/config/llm_hub_config.yaml
+#    dest: "{{ galaxy_config_dir }}/llm_hub_config.yaml"
+
 
 galaxy_extra_dirs:
   - "{{ galaxy_mutable_data_dir }}"
   - "{{ tpv_config_dir }}"
   - "{{ galaxy_config_dir }}/plugins/activities"
   - "{{ galaxy_server_dir }}/static/images/umbr_programs_icons"
 
+rabbitmq_vhosts:
+  - name: pulsar
+  - name: pulsar2
+  - name: galaxy_internal
+
+rabbitmq_users:
+  - user: guest
+    state: absent
+  - user: debian
+    password: "{{ rabbitmq_users_password.mqadmin }}"
+    tags: administrator
+    permissions:
+          - vhost: /
+            configure_priv: .*
+            read_priv: .*
+            write_priv: .*
+  - user: "{{ pulsar.user_name }}"
+    password: "{{ rabbitmq_users_password.pulsar }}"
+    permissions:
+          - vhost: pulsar
+            configure_priv: .*
+            read_priv: .*
+            write_priv: .*
+          - vhost: pulsar2
+            configure_priv: .*
+            read_priv: .*
+            write_priv: .*
+  - user: galaxy
+    password: "{{ vault_rabbitmq_password_galaxy }}"
+    permissions:
+          - vhost: galaxy_internal
+            configure_priv: .*
+            read_priv: .*
+            write_priv: .*
+  - user: flower
+    password: "{{ vault_rabbitmq_password_flower }}"
+    tags: administrator
+    permissions:
+          - vhost: galaxy_internal
+            configure_priv: .*
+            read_priv: .*
+            write_priv: .*
+
 #tusd_instances:
 #      - name: uploads
 #        # user that tusd will run as

requirements.yml

@@ -7,7 +7,7 @@
 - src: galaxyproject.postgresql_objects
   version: 1.2.0
 - src: usegalaxy_eu.certbot
-  version: 0.1.11
+  version: 0.1.13
 - src: galaxyproject.tusd
   version: 0.0.1
 - src: galaxyproject.cvmfs

templates/galaxy/config/tpv_rules_meta.yml.j2

@@ -284,6 +284,10 @@ roles:
     scheduling:
       require:
         - highmem
+  pulsar2:
+    scheduling:
+      require:
+        - pulsar2
 
 destinations:
   tpv_local:
@@ -317,7 +321,7 @@ destinations:
   # DEMON: this is the common pulsar configuration
   tpv_pulsar_common:
     abstract: true
-    runner: pulsar_tpv_runner
+    runner: pulsar2_tpv_runner
     # DEMON: sum of cores in one VM (i.e., elmo5-*)
     max_accepted_cores: 32
     # DEMON: sum of memory in one VM (i.e., elmo5-*)
@@ -367,7 +371,6 @@ destinations:
   # DEMON: this one uses singularity exclusively (consider it as a default configuration)
   tpv_pulsar:
     inherits: tpv_pulsar_common
-    runner: pulsar_tpv_runner
     params:
       singularity_enabled: true
       # DEMON: We need to enable /tmp for writing and it could be possible either by using the tmp_dir: treu or by adding explicit volume like $SCRATCHDIR:/tmp:rw (which should be more benefitial for us)
@@ -387,14 +390,12 @@ destinations:
   # DEMON: Pulsar for TIAAS. It could schedule to a private small queue to ensure the participants of some course will not have to wait
   tpv_pulsar_training:
     inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
     scheduling:
       require:
         - training
   # DEMON: For tools/users that need more memory for their jobs. Current approach is to add the user to highmem role for some time. Better approach would be to implement rescheduling for PBS over Pulsar runner
   tpv_pulsar_highmem:
     inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
     max_accepted_mem: 512
     max_mem: 512
     scheduling:
@@ -403,7 +404,6 @@ destinations:
   # DEMON: For jobs that need whole computing node exclusively (typically RepeatExplorer, DANTE, etc.)
   tpv_pulsar_excl_jobs:
     inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
     params:
       submit_native_specification: >-
         -l place=exclhost
@@ -417,22 +417,19 @@ destinations:
   # DEMON: For testing purposes, set whatever you want here when needed
   tpv_pulsar_test:
     inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
     scheduling:
       require:
         - test
   # DEMON: Some JAVA tools just ignore the traditional way of limiting the allowed memory
   tpv_pulsar_nasty_java:
     inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
     params:
       singularity_run_extra_arguments: '--env _JAVA_OPTIONS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR"'
     scheduling:
       require:
         - nasty-java
   tpv_pulsar_rosettafold:
     inherits: tpv_pulsar
-    runner: pulsar_tpv_runner
     params:
       singularity_enabled: false
       submit_native_specification: >-
@@ -461,7 +458,6 @@ destinations:
     inherits: tpv_pulsar
     max_accepted_gpus: 1
     max_gpus: 1
-    runner: pulsar_tpv_runner
     params:
       singularity_run_extra_arguments: '--nv'
       singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data/2.3:ro'
@@ -484,3 +480,29 @@ destinations:
     scheduling:
       require:
         - repex
+
+  tpv_pulsar2:
+    inherits: tpv_pulsar
+    runner: pulsar_tpv_runner
+    params:
+      ssh_key: "FAKE"
+#      ssh_key: |
+#        -----BEGIN OPENSSH PRIVATE KEY-----
+#        FAKE
+#        -----END OPENSSH PRIVATE KEY-----
+#
+    scheduling:
+      require:
+        - pulsar2
+  tpv_pulsar2_conda:
+    inherits: tpv_pulsar_conda
+    runner: pulsar_tpv_runner
+    scheduling:
+      require:
+        - pulsar2
+  tpv_pulsar2_repex:
+    inherits: tpv_pulsar_repex
+    runner: pulsar_tpv_runner
+    scheduling:
+      require:
+        - pulsar2