diff --git a/galaxy.yml b/galaxy.yml index 6470be5..ced13ba 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -308,7 +308,7 @@ name: "tmpreaper in galaxy temp dir" minute: "7" hour: "*" - job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_mutable_data_dir }}/tmp >> {{ galaxy_mutable_data_dir }}/cleaning_galaxy_temp_dir.log" + job: "/usr/sbin/tmpreaper --verbose=0 --showdeleted --all --mtime --mtime-dir --runtime=600 24h {{ galaxy_mutable_data_dir }}/tmp/ >> {{ galaxy_mutable_data_dir }}/cleaning_galaxy_temp_dir.log" become: yes become_user: "{{ galaxy_user_name }}" # - name: Creates entry in crontab "tmpreaper call in galaxy jobs dir" diff --git a/group_vars/galaxyservers.yml b/group_vars/galaxyservers.yml index c245e65..c2e7f3c 100644 --- a/group_vars/galaxyservers.yml +++ b/group_vars/galaxyservers.yml @@ -87,6 +87,16 @@ galaxy_job_config: amqp_publish_retry: true amqp_publish_retry_max_retries: 60 manager: _default_ + pulsar2_tpv_runner: + load: galaxy.jobs.runners.pulsar:PulsarMQJobRunner + galaxy_url: "https://{{ rabbitmq_hostname }}{{ csnt_galaxy_url_prefix }}" + amqp_url: "pyamqp://{{ pulsar.user_name }}:{{ rabbitmq_users_password.pulsar }}@{{ rabbitmq_hostname }}:5671/pulsar2?ssl=1" + amqp_acknowledge: true + amqp_ack_republish_time: 1200 + amqp_consumer_timeout: 2 + amqp_publish_retry: true + amqp_publish_retry_max_retries: 60 + manager: _default_ handling: assign: ['db-skip-locked'] execution: @@ -453,7 +463,7 @@ rabbitmq_users: # TUS galaxy_tusd_port: 1080 -galaxy_tus_upload_store: "{{ galaxy_mutable_data_dir }}/tus" # /mnt/data/tus +galaxy_tus_upload_store: "{{ galaxy_mutable_data_dir }}/tus/" # /mnt/data/tus #Redis galaxy_additional_venv_packages: diff --git a/host_vars/repeatexplorer-elixir.cerit-sc.cz/vars.yml b/host_vars/repeatexplorer-elixir.cerit-sc.cz/vars.yml index 273ddd6..ee9adf7 100644 --- a/host_vars/repeatexplorer-elixir.cerit-sc.cz/vars.yml +++ b/host_vars/repeatexplorer-elixir.cerit-sc.cz/vars.yml @@ -119,6 +119,11 @@ galaxy_config_templates: dest: "{{ galaxy_config.galaxy.auth_config_file }}" - src: "{{ lookup('first_found', ['templates/'+inventory_hostname+'/config/file_source_templates.yml.j2', 'templates/galaxy/config/file_source_templates.yml.j2']) }}" dest: "{{ galaxy_config_dir }}/file_source_templates.yml" + - src: templates/galaxy/config/local_data_table_conf.xml.j2 + dest: "{{ galaxy_config_dir }}/local_data_table_conf.xml" +# - src: templates/galaxy/config/llm_hub_config.yaml +# dest: "{{ galaxy_config_dir }}/llm_hub_config.yaml" + galaxy_extra_dirs: - "{{ galaxy_mutable_data_dir }}" @@ -126,6 +131,49 @@ galaxy_extra_dirs: - "{{ galaxy_config_dir }}/plugins/activities" - "{{ galaxy_server_dir }}/static/images/umbr_programs_icons" +rabbitmq_vhosts: + - name: pulsar + - name: pulsar2 + - name: galaxy_internal + +rabbitmq_users: + - user: guest + state: absent + - user: debian + password: "{{ rabbitmq_users_password.mqadmin }}" + tags: administrator + permissions: + - vhost: / + configure_priv: .* + read_priv: .* + write_priv: .* + - user: "{{ pulsar.user_name }}" + password: "{{ rabbitmq_users_password.pulsar }}" + permissions: + - vhost: pulsar + configure_priv: .* + read_priv: .* + write_priv: .* + - vhost: pulsar2 + configure_priv: .* + read_priv: .* + write_priv: .* + - user: galaxy + password: "{{ vault_rabbitmq_password_galaxy }}" + permissions: + - vhost: galaxy_internal + configure_priv: .* + read_priv: .* + write_priv: .* + - user: flower + password: "{{ vault_rabbitmq_password_flower }}" + tags: administrator + permissions: + - vhost: galaxy_internal + configure_priv: .* + read_priv: .* + write_priv: .* + #tusd_instances: # - name: uploads # # user that tusd will run as diff --git a/requirements.yml b/requirements.yml index 058b99c..610e34d 100644 --- a/requirements.yml +++ b/requirements.yml @@ -7,7 +7,7 @@ - src: galaxyproject.postgresql_objects version: 1.2.0 - src: usegalaxy_eu.certbot - version: 0.1.11 + version: 0.1.13 - src: galaxyproject.tusd version: 0.0.1 - src: galaxyproject.cvmfs diff --git a/templates/galaxy/config/tpv_rules_meta.yml.j2 b/templates/galaxy/config/tpv_rules_meta.yml.j2 index 4aa07bf..90529df 100644 --- a/templates/galaxy/config/tpv_rules_meta.yml.j2 +++ b/templates/galaxy/config/tpv_rules_meta.yml.j2 @@ -284,6 +284,10 @@ roles: scheduling: require: - highmem + pulsar2: + scheduling: + require: + - pulsar2 destinations: tpv_local: @@ -317,7 +321,7 @@ destinations: # DEMON: this is the common pulsar configuration tpv_pulsar_common: abstract: true - runner: pulsar_tpv_runner + runner: pulsar2_tpv_runner # DEMON: sum of cores in one VM (i.e., elmo5-*) max_accepted_cores: 32 # DEMON: sum of memory in one VM (i.e., elmo5-*) @@ -367,7 +371,6 @@ destinations: # DEMON: this one uses singularity exclusively (consider it as a default configuration) tpv_pulsar: inherits: tpv_pulsar_common - runner: pulsar_tpv_runner params: singularity_enabled: true # DEMON: We need to enable /tmp for writing and it could be possible either by using the tmp_dir: treu or by adding explicit volume like $SCRATCHDIR:/tmp:rw (which should be more benefitial for us) @@ -387,14 +390,12 @@ destinations: # DEMON: Pulsar for TIAAS. It could schedule to a private small queue to ensure the participants of some course will not have to wait tpv_pulsar_training: inherits: tpv_pulsar - runner: pulsar_tpv_runner scheduling: require: - training # DEMON: For tools/users that need more memory for their jobs. Current approach is to add the user to highmem role for some time. Better approach would be to implement rescheduling for PBS over Pulsar runner tpv_pulsar_highmem: inherits: tpv_pulsar - runner: pulsar_tpv_runner max_accepted_mem: 512 max_mem: 512 scheduling: @@ -403,7 +404,6 @@ destinations: # DEMON: For jobs that need whole computing node exclusively (typically RepeatExplorer, DANTE, etc.) tpv_pulsar_excl_jobs: inherits: tpv_pulsar - runner: pulsar_tpv_runner params: submit_native_specification: >- -l place=exclhost @@ -417,14 +417,12 @@ destinations: # DEMON: For testing purposes, set whatever you want here when needed tpv_pulsar_test: inherits: tpv_pulsar - runner: pulsar_tpv_runner scheduling: require: - test # DEMON: Some JAVA tools just ignore the traditional way of limiting the allowed memory tpv_pulsar_nasty_java: inherits: tpv_pulsar - runner: pulsar_tpv_runner params: singularity_run_extra_arguments: '--env _JAVA_OPTIONS="-Xmx{int(mem)}g -Djava.io.tmpdir=$SCRATCHDIR"' scheduling: @@ -432,7 +430,6 @@ destinations: - nasty-java tpv_pulsar_rosettafold: inherits: tpv_pulsar - runner: pulsar_tpv_runner params: singularity_enabled: false submit_native_specification: >- @@ -461,7 +458,6 @@ destinations: inherits: tpv_pulsar max_accepted_gpus: 1 max_gpus: 1 - runner: pulsar_tpv_runner params: singularity_run_extra_arguments: '--nv' singularity_volumes: '$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw,$SCRATCHDIR,$ALPHAFOLD_DB:/data/2.3:ro' @@ -484,3 +480,29 @@ destinations: scheduling: require: - repex + + tpv_pulsar2: + inherits: tpv_pulsar + runner: pulsar_tpv_runner + params: + ssh_key: "FAKE" +# ssh_key: | +# -----BEGIN OPENSSH PRIVATE KEY----- +# FAKE +# -----END OPENSSH PRIVATE KEY----- +# + scheduling: + require: + - pulsar2 + tpv_pulsar2_conda: + inherits: tpv_pulsar_conda + runner: pulsar_tpv_runner + scheduling: + require: + - pulsar2 + tpv_pulsar2_repex: + inherits: tpv_pulsar_repex + runner: pulsar_tpv_runner + scheduling: + require: + - pulsar2