diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 0000000..2637590 --- /dev/null +++ b/docs/about.md @@ -0,0 +1,132 @@ +--- +type: Reference +title: "About this documentation" +description: "What CyVerse is, who this documentation is for, and where each audience should start." +tags: [about, orientation] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: okf-spec + resource: https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md + title: Open Knowledge Format v0.2 specification + author: team:google-cloud-platform + last_modified: 2026-07-29 +--- + +![](assets/cyverse_logo_2022.svg) + +# What CyVerse is + +CyVerse is a computational infrastructure for data-intensive science, and the +people who operate it. It is fully open source and funded by the +[United States National Science Foundation](https://www.nsf.gov/){target=_blank}. + +It is both a Software as a Service platform and the Infrastructure as Code needed +to run one: the same stack that serves the public US deployment can be deployed by +another institution on its own hardware or in the cloud. That is what this +documentation is for. + +
+ ![layercake](assets/layerCake.svg){width=800} +
Hardware at the bottom, services in the middle, products on top
+
+ +# Who this is for + +## Deploying CyVerse + +Start with [prerequisites](deployment/planning/prerequisites.md), then read +[deploying from scratch](deployment/from-scratch.md) end to end before running +anything. Work the phases in [deployment](deployment/) in order, and check each one +against [verification](deployment/07-post-install/verification.md). + +Before provisioning: [component inventory and +sizing](architecture/component-inventory.md) and [network +requirements](architecture/network-requirements.md). + +## Operating a deployment + +[operations/](operations/) covers day-to-day administration: users and VICE access +in [DE administration](operations/discovery-environment.md), data and curation in +[Data Store administration](operations/data-store.md), accounts in [User Portal +administration](operations/user-portal.md), and the recurring questions in the +[FAQ](operations/faq.md). + +## Integrating with the APIs + +[Terrain](api/terrain.md) is the API behind every CyVerse product. The +[endpoint index](api/endpoint-index.md) lists everything documented here, and the +live [Swagger reference](https://de.cyverse.org/terrain/docs/){target=_blank} is +the most current source. Authentication is [OAuth 2.0 through +Keycloak](platform/authentication.md). + +## Contributing code + +[development/](development/) covers the development environment and contribution +workflow. Source lives in the +[CyVerse](https://github.com/cyverse){target=_blank} and +[CyVerse DE](https://github.com/cyverse-de){target=_blank} GitHub organizations. + +# What CyVerse offers its users + +| Product | What it does | +|---------|--------------| +| [Discovery Environment](platform/discovery-environment.md) | Web-based data science workbench with hundreds of integrated tools | +| [Data Store](platform/data-store.md) | Multi-petabyte iRODS storage with HTTPS, WebDAV, SFTP, and API access | +| [Data Commons](platform/data-commons.md) | Publishing curated and community-released datasets, with DataCite DOIs | +| VICE | Interactive computing — JupyterLab, RStudio, Shiny — inside the DE | +| [Cloud services (CACAO)](platform/cloud.md) | Infrastructure as code for multi-cloud deployments | +| [BisQue](platform/bisque.md) | Bio-image semantic query and analysis | +| [DNA Subway](platform/dna-subway.md) | Educational genomics workflows | + +# How this documentation is organized + +This bundle follows the [Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) +v0.2.[^okf-spec] In practice that means three things you can rely on: + +* **Every document declares itself.** Frontmatter carries its `type`, a one-line + `description`, `tags`, and a `status` of `draft`, `stable`, or `deprecated`. A + `draft` document is incomplete and says so rather than pretending otherwise. +* **Every directory has an index.** `index.md` lists what is in a directory with a + line of description each, so you can see what exists before opening anything. +* **Derived documents cite their sources.** Where a document was written from + something else, `sources` in its frontmatter says what, including material + mirrored under [references/](references/). + +Changes to the bundle are recorded in [the log](log.md). + +# Links + +* :material-web: [CyVerse website](https://cyverse.org){target=_blank} +* :material-frequently-asked-questions: [FAQ](operations/faq.md) +* :simple-github: [GitHub organization](https://github.com/cyverse-de){target=_blank} +* :material-api: [Live Terrain API](https://de.cyverse.org/terrain/docs/){target=_blank} +* :simple-docker: [Harbor registry](https://harbor.cyverse.org/){target=_blank} +* :material-school: [User-facing learning materials](https://learning.cyverse.org/){target=_blank} + +# Funding + +[![nsf](assets/NSF.svg){width=100}](https://www.nsf.gov/){target=_blank} + +CyVerse has been funded by the National Science Foundation from 2008 to the +present. + +[![NSF-0735191](https://img.shields.io/badge/NSF-0735191-blue.svg)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=0735191) [![NSF-1265383](https://img.shields.io/badge/NSF-1265383-blue.svg)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1265383) [![NSF-1743442](https://img.shields.io/badge/NSF-1743442-blue.svg)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1743442) + +!!! Info ":fontawesome-brands-creative-commons-by: SOFTWARE LICENSE" + + Copyright (c) 2010-2026, The Arizona Board of Regents on behalf of The University of Arizona + + All rights reserved. + + Developed by: CyVerse as a collaboration between participants at BIO5 at The University of Arizona (the primary hosting institution), Cold Spring Harbor Laboratory, The University of Texas at Austin, and individual contributors. Find out more at http://www.cyverse.org/. + + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of CyVerse, BIO5, The University of Arizona, Cold Spring Harbor Laboratory, The University of Texas at Austin, nor the names of other contributors may be used to endorse or promote products derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +[^okf-spec]: Open Knowledge Format v0.2 specification diff --git a/docs/api/endpoint-index.md b/docs/api/endpoint-index.md new file mode 100644 index 0000000..9dcee94 --- /dev/null +++ b/docs/api/endpoint-index.md @@ -0,0 +1,302 @@ +--- +type: Reference +title: "Endpoint index" +description: "Alphabetical index of every documented Terrain endpoint." +tags: [api, terrain, index] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- +**Jump to:** + +[`/admin`](#admin) + +[`/apps`](#apps) + +[`/coge`](#coge) + +[`/favorites`](#favorites) + +[`/filesystem`](#filesystem) + +[`/permanent-id-requests`](#permanent-id-requests) + +[`/secured`](#secured) + +[`/send-notification`](#send-notification) + +[`/uuid`](#uuid) + +## get + +[`GET /`](./endpoints/misc.md#verifying-that-terrain-is-running) + +## admin + +[`GET /admin/apps/categories`](./endpoints/app-metadata.md#listing-app-categories) + +[`GET /admin/apps/categories/search`](./endpoints/app-metadata.md#searching-for-categories-by-name) + +[`POST /admin/apps/categories/{system-id}`](./endpoints/app-metadata.md#adding-categories) + +[`DELETE /admin/apps/categories/{system-id}/{category-id}`](./endpoints/app-metadata.md#deleting-a-category-by-id) + +[`PATCH /admin/apps/categories/{system-id}/{category-id}`](./endpoints/app-metadata.md#updating-an-app-category) + +[`DELETE /admin/apps/{app-id}/comments/{comment-id}`](./endpoints/comments.md#administratively-deleting-a-comment) + +[`PATCH /admin/apps/{app-id}/comments/{comment-id}`](./endpoints/comments.md#retractingreadmitting-a-comment) + +[`GET /admin/apps/{app-id}/metadata`](./endpoints/app-metadata.md#managing-app-avu-metadata) + +[`POST /admin/apps/{app-id}/metadata`](./endpoints/app-metadata.md#managing-app-avu-metadata) + +[`PUT /admin/apps/{app-id}/metadata`](./endpoints/app-metadata.md#managing-app-avu-metadata) + +[`DELETE /admin/filesystem/entry/{entry-id}/comments/{comment-id}`](./endpoints/comments.md#administratively-deleting-a-comment) + +[`PATCH /admin/filesystem/entry/{entry-id}/comments/{comment-id}`](./endpoints/comments.md#retractingreadmitting-a-comment) + +[`GET /admin/filesystem/metadata/templates`](./endpoints/filesystem/metadata.md#listing-metadata-templates) + +[`POST /admin/filesystem/metadata/templates`](./endpoints/filesystem/metadata.md#adding-metadata-templates) + +[`DELETE /admin/filesystem/metadata/templates/{template-id}`](./endpoints/filesystem/metadata.md#marking-a-metadata-template-as-deleted) + +[`POST /admin/filesystem/metadata/templates/{template-id}`](./endpoints/filesystem/metadata.md#updating-metadata-templates) + +[`GET /admin/notifications/system`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`PUT /admin/notifications/system`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /admin/notifications/system-types`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`DELETE /admin/notifications/system/:uuid`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /admin/notifications/system/:uuid`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`POST /admin/notifications/system/:uuid`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /admin/ontologies`](./endpoints/app-ontologies.md#listing-saved-ontology-details) + +[`POST /admin/ontologies`](./endpoints/app-ontologies.md#save-an-ontology-xml-document) + +[`DELETE /admin/ontologies/{ontology-version}`](./endpoints/app-ontologies.md#logically-deleting-an-ontology) + +[`GET /admin/ontologies/{ontology-version}`](./endpoints/app-ontologies.md#listing-hierarchies-for-any-ontology) + +[`POST /admin/ontologies/{ontology-version}`](./endpoints/app-ontologies.md#set-active-ontology-version) + +[`DELETE /admin/ontologies/{ontology-version}/{root-iri}`](./endpoints/app-ontologies.md#deleting-an-ontology-hierarchy) + +[`GET /admin/ontologies/{ontology-version}/{root-iri}`](./endpoints/app-ontologies.md#listing-filtered-hierarchies-for-any-ontology) + +[`PUT /admin/ontologies/{ontology-version}/{root-iri}`](./endpoints/app-ontologies.md#save-an-ontology-hierarchy) + +[`GET /admin/ontologies/{ontology-version}/{root-iri}/apps`](./endpoints/app-ontologies.md#listing-apps-in-hierarchies-for-any-ontology) + +[`GET /admin/ontologies/{ontology-version}/{root-iri}/unclassified`](./endpoints/app-ontologies.md#listing-unclassified-apps-for-any-ontology) + +[`GET /admin/permanent-id-requests`](./endpoints/permanent-id-requests.md) + +[`GET /admin/permanent-id-requests/{request-id}`](./endpoints/permanent-id-requests.md) + +[`POST /admin/permanent-id-requests/{request-id}/ezid`](./endpoints/permanent-id-requests.md) + +[`POST /admin/permanent-id-requests/{request-id}/status`](./endpoints/permanent-id-requests.md) + +[`DELETE /admin/workspaces`](./endpoints/app-metadata.md#deleting-workspaces) + +[`GET /admin/workspaces`](./endpoints/app-metadata.md#listing-workspaces) + +## apps + +[`GET /apps/{app-id}/comments`](./endpoints/comments.md#listing-comments) + +[`POST /apps/{app-id}/comments`](./endpoints/comments.md#creating-a-comment) + +[`PATCH /apps/{app-id}/comments/{comment-id}`](./endpoints/comments.md#retractingreadmitting-a-comment) + +[`PATCH /apps/{app-id}/comments/{comment-id}`](./endpoints/comments.md#retractingreadmitting-a-comment) + +## coge + +[`GET /coge/genomes`](./endpoints/filesystem/coge.md#searching-for-genomes-in-coge) + +[`POST /coge/genomes/load`](./endpoints/filesystem/coge.md#viewing-a-genome-file-in-coge) + +[`POST /coge/genomes/{genome-id}/export-fasta`](./endpoints/filesystem/coge.md#exporting-coge-genome-data-to-irods) + +## favorites + +[`GET /favorites/filesystem`](./endpoints/favorites.md#listing-stat-info-for-favorite-data) + +## filesystem + +[`PATCH /filesystem/entry/{entry-id}/comments/{comment-id}`](./endpoints/comments.md#retractingreadmitting-a-comment) + +## permanent-id-requests + +[`GET /permanent-id-requests`](./endpoints/permanent-id-requests.md) + +[`POST /permanent-id-requests`](./endpoints/permanent-id-requests.md) + +[`GET /permanent-id-requests/status-codes`](./endpoints/permanent-id-requests.md) + +[`GET /permanent-id-requests/types`](./endpoints/permanent-id-requests.md) + +[`GET /permanent-id-requests/{request-id}`](./endpoints/permanent-id-requests.md) + +## secured + +[`GET /secured/favorites/filesystem`](./endpoints/favorites.md#listing-stat-info-for-favorite-data) + +[`DELETE /secured/favorites/filesystem/{favorite}`](./endpoints/favorites.md#removing-a-data-resource-from-being-a-favorite) + +[`PUT /secured/favorites/filesystem/{favorite}`](./endpoints/favorites.md#marking-a-data-resource-as-favorite) + +[`POST /secured/favorites/filter`](./endpoints/favorites.md#filter-a-set-of-resources-for-favorites) + +[`GET /secured/fileio/download`](./endpoints/fileio.md#downloading) + +[`POST /secured/fileio/save`](./endpoints/fileio.md#save) + +[`POST /secured/fileio/saveas`](./endpoints/fileio.md#save-as) + +[`POST /secured/fileio/upload`](./endpoints/fileio.md#uploading) + +[`POST /secured/fileio/urlupload`](./endpoints/fileio.md#url-uploads) + +[`POST /secured/filesystem/delete`](./endpoints/filesystem/delete.md#deleting-files-andor-directories) + +[`POST /secured/filesystem/delete-contents`](./endpoints/filesystem/delete.md#deleting-all-items-in-a-directory) + +[`POST /secured/filesystem/delete-tickets`](./endpoints/filesystem/tickets.md#deleting-tickets) + +[`POST /secured/filesystem/directories`](./endpoints/filesystem/directory-create.md#batch-directory-creation) + +[`GET /secured/filesystem/directory`](./endpoints/filesystem/directory-listing.md#directory-list-non-recursive) + +[`POST /secured/filesystem/directory/create`](./endpoints/filesystem/directory-create.md#directory-creation) + +[`GET /secured/filesystem/display-download`](./endpoints/fileio.md#downloading) + +[`GET /secured/filesystem/entry/{entry-id}/comments`](./endpoints/comments.md#listing-comments) + +[`POST /secured/filesystem/entry/{entry-id}/comments`](./endpoints/comments.md#creating-a-comment) + +[`PATCH /secured/filesystem/entry/{entry-id}/comments/{comment-id}`](./endpoints/comments.md#retractingreadmitting-a-comment) + +[`POST /secured/filesystem/exists`](./endpoints/filesystem/existence.md#filedirectory-existence) + +[`GET /secured/filesystem/file/manifest`](./endpoints/filesystem/manifest.md#file-manifest) + +[`GET /secured/filesystem/index`](./endpoints/filesystem/search.md#endpoints) + +[`POST /secured/filesystem/list-tickets`](./endpoints/filesystem/tickets.md#listing-tickets) + +[`POST /secured/filesystem/metadata/csv-parser`](./endpoints/filesystem/metadata.md#adding-batch-metadata-to-multiple-paths-from-a-csv-file) + +[`GET /secured/filesystem/metadata/template/attr/{attribute-id}`](./endpoints/filesystem/metadata.md#viewing-a-metadata-attribute) + +[`GET /secured/filesystem/metadata/template/{template-id}`](./endpoints/filesystem/metadata.md#viewing-a-metadata-template) + +[`GET /secured/filesystem/metadata/template/{template-id}/blank-csv`](./endpoints/filesystem/metadata.md#downloading-a-blank-template) + +[`GET /secured/filesystem/metadata/template/{template-id}/guide-csv`](./endpoints/filesystem/metadata.md#downloading-a-template-guide) + +[`GET /secured/filesystem/metadata/templates`](./endpoints/filesystem/metadata.md#listing-metadata-templates) + +[`POST /secured/filesystem/move`](./endpoints/filesystem/move.md#moving-files-andor-directories) + +[`POST /secured/filesystem/move-contents`](./endpoints/filesystem/move.md#moving-all-items-in-a-directory) + +[`GET /secured/filesystem/paged-directory`](./endpoints/filesystem/directory-listing.md#paged-directory-listing) + +[`POST /secured/filesystem/path-list-creator`](./endpoints/filesystem/path-lists.md#ht-path-list-creator) + +[`POST /secured/filesystem/read-chunk`](./endpoints/filesystem/read-chunk.md#reading-a-chunk-of-a-file) + +[`POST /secured/filesystem/read-csv-chunk`](./endpoints/filesystem/csv-tsv-parsing.md#csvtsv-parsing) + +[`POST /secured/filesystem/rename`](./endpoints/filesystem/rename.md#renaming-a-file-or-directory) + +[`POST /secured/filesystem/restore`](./endpoints/filesystem/restore.md#restoring-a-file-or-directory-from-a-users-trash) + +[`POST /secured/filesystem/restore-all`](./endpoints/filesystem/restore.md#restoring-all-items-in-a-users-trash) + +[`GET /secured/filesystem/root`](./endpoints/filesystem/root-listing.md#top-level-root-listing) + +[`POST /secured/filesystem/stat`](./endpoints/filesystem/stat.md#file-and-directory-status-information) + +[`POST /secured/filesystem/tickets`](./endpoints/filesystem/tickets.md#creating-tickets) + +[`DELETE /secured/filesystem/trash`](./endpoints/filesystem/empty-trash.md#emptying-a-users-trash-directory) + +[`POST /secured/filesystem/user-permissions`](./endpoints/filesystem/permissions.md#listing-user-permissions) + +[`GET /secured/filesystem/{data-id}/metadata`](./endpoints/filesystem/metadata.md#getting-metadata) + +[`POST /secured/filesystem/{data-id}/metadata`](./endpoints/filesystem/metadata.md#setting-metadata) + +[`POST /secured/filesystem/{data-id}/metadata/copy`](./endpoints/filesystem/metadata.md#copying-all-metadata-from-a-filefolder) + +[`POST /secured/filesystem/{data-id}/metadata/save`](./endpoints/filesystem/metadata.md#exporting-metadata-to-a-file) + +[`POST /secured/filesystem/{data-id}/ore/save`](./endpoints/filesystem/ore.md#generating-oai-ore-files-for-a-data-set) + +[`GET /secured/notifications/count-messages`](./endpoints/notifications.md#obtaining-notification-counts) + +[`POST /secured/notifications/delete`](./endpoints/notifications.md#marking-notifications-as-deleted) + +[`DELETE /secured/notifications/delete-all`](./endpoints/notifications.md#marking-all-notifications-as-deleted) + +[`GET /secured/notifications/last-ten-messages`](./endpoints/notifications.md#obtaining-the-ten-most-recent-notifications) + +[`POST /secured/notifications/mark-all-seen`](./endpoints/notifications.md#marking-all-notifications-as-seen) + +[`GET /secured/notifications/messages`](./endpoints/notifications.md#obtaining-notifications) + +[`POST /secured/notifications/seen`](./endpoints/notifications.md#marking-notifications-as-seen) + +[`POST /secured/notifications/system/delete`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`DELETE /secured/notifications/system/delete-all`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`POST /secured/notifications/system/mark-all-received`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`POST /secured/notifications/system/mark-all-seen`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /secured/notifications/system/messages`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /secured/notifications/system/new-messages`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`POST /secured/notifications/system/received`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`POST /secured/notifications/system/seen`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /secured/notifications/system/unseen-messages`](./endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) + +[`GET /secured/notifications/unseen-messages`](./endpoints/notifications.md#obtaining-unseen-notifications) + +[`GET /secured/oauth/access-code/{api-name}`](./endpoints/callbacks.md#obtaining-oauth-authorization-codes) + +[`DELETE /secured/preferences`](./endpoints/misc.md#removing-user-preferences) + +[`GET /secured/preferences`](./endpoints/misc.md#retrieving-user-preferences) + +[`POST /secured/preferences`](./endpoints/misc.md#saving-user-preferences) + +[`DELETE /secured/sessions`](./endpoints/misc.md#removing-user-session-data) + +[`GET /secured/sessions`](./endpoints/misc.md#retrieving-user-session-data) + +[`POST /secured/sessions`](./endpoints/misc.md#saving-user-session-data) + +## send-notification + +[`POST /send-notification.`](./endpoints/notifications.md#sending-an-arbitrary-notification) + +## uuid + +[`GET /uuid`](./endpoints/misc.md#obtaining-identifiers) + diff --git a/docs/services/api/endpoints/app-metadata.md b/docs/api/endpoints/app-metadata.md similarity index 91% rename from docs/services/api/endpoints/app-metadata.md rename to docs/api/endpoints/app-metadata.md index 54afecf..03df5df 100644 --- a/docs/services/api/endpoints/app-metadata.md +++ b/docs/api/endpoints/app-metadata.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "App metadata" +description: "Endpoints for reading and writing metadata attached to apps." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- **Jump to:** * [Application Metadata Endpoints](#application-metadata-endpoints) @@ -21,7 +29,7 @@ # Application Metadata Endpoints Note that secured endpoints in Terrain and apps are a little different from each other. -Please see [Terrain Vs. Apps](terrain-v-apps.md) for more information. +Please see [Terrain Vs. Apps](./terrain-v-apps.md) for more information. ## Adding Categories diff --git a/docs/services/api/endpoints/app-ontologies.md b/docs/api/endpoints/app-ontologies.md similarity index 95% rename from docs/services/api/endpoints/app-ontologies.md rename to docs/api/endpoints/app-ontologies.md index 2919148..9bec6aa 100644 --- a/docs/services/api/endpoints/app-ontologies.md +++ b/docs/api/endpoints/app-ontologies.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "App ontologies" +description: "Endpoints for managing ontology hierarchies used to classify apps." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Table of Contents * [Categorizing Apps with Ontology Hierarchies](#categorizing-apps-with-ontology-hierarchies) @@ -44,7 +52,7 @@ * [Listing Unclassified Apps for the Active Ontology](https://de.cyverse.org/terrain/docs/index.html#!/app45hierarchies/get_terrain_apps_hierarchies_root_iri_unclassified) 5. Admins or users with app "write" permissions may categorize those apps under ontology classes by attaching the class IRIs as metadata. - * [Managing App AVU Metadata](app-metadata.md#managing-app-avu-metadata) + * [Managing App AVU Metadata](./app-metadata.md#managing-app-avu-metadata) 6. Apps with an ontology class attached as metadata, or any class under the hierarchy of that root class, may be listed with the following endpoint: * [Listing Apps in Hierarchies for the Active Ontology](https://de.cyverse.org/terrain/docs/index.html#!/app45hierarchies/get_terrain_apps_hierarchies_root_iri_apps) diff --git a/docs/services/api/endpoints/callbacks.md b/docs/api/endpoints/callbacks.md similarity index 97% rename from docs/services/api/endpoints/callbacks.md rename to docs/api/endpoints/callbacks.md index 4a20f18..5be37aa 100644 --- a/docs/services/api/endpoints/callbacks.md +++ b/docs/api/endpoints/callbacks.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Callbacks" +description: "Endpoints other services call to report analysis and job status back to the DE." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Callback Endpoints These endpoints listed in this document accept callbacks from other services indicating that some event has occurred. diff --git a/docs/services/api/endpoints/comments.md b/docs/api/endpoints/comments.md similarity index 96% rename from docs/services/api/endpoints/comments.md rename to docs/api/endpoints/comments.md index 5e3ad6d..e5f5587 100644 --- a/docs/services/api/endpoints/comments.md +++ b/docs/api/endpoints/comments.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Comments" +description: "Endpoints for creating, listing, and retracting comments on data and apps." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- This document describes the comments resource. A _comment_ is something that a user records about a given app, file, or folder. diff --git a/docs/api/endpoints/endpoints.md b/docs/api/endpoints/endpoints.md new file mode 100644 index 0000000..5421fa6 --- /dev/null +++ b/docs/api/endpoints/endpoints.md @@ -0,0 +1,37 @@ +--- +type: API Endpoint +title: "Endpoints overview" +description: "How the Terrain endpoint documentation is organized and which endpoints are documented where." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- +# Terrain Endpoints + +All URLs referenced in endpoint documentation are listed as relative URLs with value names enclosed in braces. +For example, the service to get a list of workflow elements is accessed using the URL, `/get-workflow-elements/{element-type}`. +Where `{element-type}` refers to the type of workflow element that is being retrieved. +For example, to get a list of known property types, you can access the URL, `/get-workflow-elements/property-types`. +On the other hand, all examples use fully qualified URLs. + +Request and response bodies are in JSON format unless otherwise noted. + +* [App Metadata](./app-metadata.md) + +* [Callback](./callbacks.md) + +* [Categorizing Apps with Ontology Hierarchies](./app-ontologies.md) + +* [Comments](./comments.md) + +* [Data Search](./filesystem/search.md) + +* [Favorites](./favorites.md) + +* [Notification](./notifications.md) + +* [Permanent ID Requests](./permanent-id-requests.md) + +* [Quick Launches](./quick-launches.md) + +* [Miscellaneous](./misc.md) diff --git a/docs/services/api/endpoints/favorites.md b/docs/api/endpoints/favorites.md similarity index 97% rename from docs/services/api/endpoints/favorites.md rename to docs/api/endpoints/favorites.md index fe47121..0d619ff 100644 --- a/docs/services/api/endpoints/favorites.md +++ b/docs/api/endpoints/favorites.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Favorites" +description: "Endpoints for managing a user's favorite data items and apps." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- This document describes the favorites resource. A _favorite_ is something that a user has decided is important enough that it should be more readily accessible than other resources of the same time. diff --git a/docs/services/api/endpoints/fileio.md b/docs/api/endpoints/fileio.md similarity index 96% rename from docs/services/api/endpoints/fileio.md rename to docs/api/endpoints/fileio.md index 097b9a5..9cb900d 100644 --- a/docs/services/api/endpoints/fileio.md +++ b/docs/api/endpoints/fileio.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "File I/O" +description: "Endpoints for uploading, downloading, and streaming file contents." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # File IO Provides a REST-like API for uploading and downloading files to and from iRODS. diff --git a/docs/services/api/endpoints/filesystem/coge.md b/docs/api/endpoints/filesystem/coge.md similarity index 92% rename from docs/services/api/endpoints/filesystem/coge.md rename to docs/api/endpoints/filesystem/coge.md index 105091f..c4f1d85 100644 --- a/docs/services/api/endpoints/filesystem/coge.md +++ b/docs/api/endpoints/filesystem/coge.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "CoGe" +description: "Endpoints for exposing genome files in the Data Store to CoGe." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Viewing a Genome File in CoGe ----------------------------- diff --git a/docs/services/api/endpoints/filesystem/csv-tsv-parsing.md b/docs/api/endpoints/filesystem/csv-tsv-parsing.md similarity index 91% rename from docs/services/api/endpoints/filesystem/csv-tsv-parsing.md rename to docs/api/endpoints/filesystem/csv-tsv-parsing.md index 0a99ae7..0e364e9 100644 --- a/docs/services/api/endpoints/filesystem/csv-tsv-parsing.md +++ b/docs/api/endpoints/filesystem/csv-tsv-parsing.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "CSV/TSV parsing" +description: "Endpoints that parse delimited files in the Data Store into structured responses." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- CSV/TSV Parsing ------------------------- diff --git a/docs/services/api/endpoints/filesystem/delete.md b/docs/api/endpoints/filesystem/delete.md similarity index 87% rename from docs/services/api/endpoints/filesystem/delete.md rename to docs/api/endpoints/filesystem/delete.md index c5fe2ca..84d8fb2 100644 --- a/docs/services/api/endpoints/filesystem/delete.md +++ b/docs/api/endpoints/filesystem/delete.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Delete" +description: "Endpoints for moving files and directories to the trash or deleting them outright." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Deleting Files and/or Directories --------------------------------- __URL Path__: /secured/filesystem/delete diff --git a/docs/services/api/endpoints/filesystem/directory-create.md b/docs/api/endpoints/filesystem/directory-create.md similarity index 77% rename from docs/services/api/endpoints/filesystem/directory-create.md rename to docs/api/endpoints/filesystem/directory-create.md index 733d8f6..59c2544 100644 --- a/docs/services/api/endpoints/filesystem/directory-create.md +++ b/docs/api/endpoints/filesystem/directory-create.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Directory create" +description: "Endpoint for creating one or many directories in a single request." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Batch Directory Creation ------------------------ @@ -24,7 +32,7 @@ __Request Body__: __Response Body__: -This endpoint uses a similar response as the [/secured/filesystem/stat](stat.md#file-and-directory-status-information) endpoint. For example: +This endpoint uses a similar response as the [/secured/filesystem/stat](./stat.md#file-and-directory-status-information) endpoint. For example: ```json { diff --git a/docs/services/api/endpoints/filesystem/directory-listing.md b/docs/api/endpoints/filesystem/directory-listing.md similarity index 96% rename from docs/services/api/endpoints/filesystem/directory-listing.md rename to docs/api/endpoints/filesystem/directory-listing.md index a426dc6..eaf7bdf 100644 --- a/docs/services/api/endpoints/filesystem/directory-listing.md +++ b/docs/api/endpoints/filesystem/directory-listing.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Directory listing" +description: "Endpoints for non-recursive directory listings with paging and filtering." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Directory List (Non-Recursive) ------------------------------ diff --git a/docs/services/api/endpoints/filesystem/empty-trash.md b/docs/api/endpoints/filesystem/empty-trash.md similarity index 64% rename from docs/services/api/endpoints/filesystem/empty-trash.md rename to docs/api/endpoints/filesystem/empty-trash.md index af3f712..82adad5 100644 --- a/docs/services/api/endpoints/filesystem/empty-trash.md +++ b/docs/api/endpoints/filesystem/empty-trash.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Empty trash" +description: "Endpoint for permanently emptying a user's trash directory." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Emptying a User's Trash Directory --------------------------------- __URL Path__: /secured/filesystem/trash diff --git a/docs/services/api/endpoints/filesystem/errors.md b/docs/api/endpoints/filesystem/errors.md similarity index 52% rename from docs/services/api/endpoints/filesystem/errors.md rename to docs/api/endpoints/filesystem/errors.md index fad294f..3688ceb 100644 --- a/docs/services/api/endpoints/filesystem/errors.md +++ b/docs/api/endpoints/filesystem/errors.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Filesystem errors" +description: "Error codes returned by the filesystem endpoints." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Error Codes ----------- diff --git a/docs/services/api/endpoints/filesystem/existence.md b/docs/api/endpoints/filesystem/existence.md similarity index 76% rename from docs/services/api/endpoints/filesystem/existence.md rename to docs/api/endpoints/filesystem/existence.md index a3267f7..3f78cba 100644 --- a/docs/services/api/endpoints/filesystem/existence.md +++ b/docs/api/endpoints/filesystem/existence.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Existence" +description: "Endpoint for testing whether paths exist and are visible to the caller." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- File/Directory existence ------------------------ The /exists endpoint allows the caller to check for the existence of a set of files. The following is an example call to the exists endpoint: diff --git a/docs/api/endpoints/filesystem/index.md b/docs/api/endpoints/filesystem/index.md new file mode 100644 index 0000000..a2e4212 --- /dev/null +++ b/docs/api/endpoints/filesystem/index.md @@ -0,0 +1,47 @@ +# Filesystem endpoints + +Terrain's view of the [Data Store](../../../platform/data-store.md). Paths are +iRODS paths, and every operation is subject to iRODS permissions. + +# Browsing + +* [Root listing](root-listing.md) - top-level directories visible to the caller +* [Directory listing](directory-listing.md) - non-recursive listings with paging +* [Stat](stat.md) - status information for files and directories +* [Existence](existence.md) - whether paths exist and are visible +* [Manifest](manifest.md) - file manifest, including preview and infoType + +# Reading content + +* [Read chunk](read-chunk.md) - byte ranges and pages without downloading +* [CSV/TSV parsing](csv-tsv-parsing.md) - delimited files parsed into structured responses + +# Modifying + +* [Directory create](directory-create.md) - create one or many directories +* [Move](move.md) - move files and directories, individually or in bulk +* [Rename](rename.md) - rename in place +* [Delete](delete.md) - move to trash or delete outright +* [Restore](restore.md) - restore from a user's trash +* [Empty trash](empty-trash.md) - permanently empty the trash + +# Metadata and search + +* [Metadata](metadata.md) - read, set, and copy AVU metadata +* [Search](search.md) - search by name, metadata, and permissions + +# Sharing + +* [Permissions](permissions.md) - list and update user permissions +* [Sharing](sharing.md) - share and unshare with other users +* [Tickets](tickets.md) - time- or use-limited anonymous access + +# Integrations and formats + +* [CoGe](coge.md) - expose genome files to CoGe +* [OAI-ORE](ore.md) - generate OAI-ORE descriptions of a dataset +* [Path lists](path-lists.md) - build HT path list files + +# Errors + +* [Filesystem errors](errors.md) - error codes returned by these endpoints diff --git a/docs/services/api/endpoints/filesystem/manifest.md b/docs/api/endpoints/filesystem/manifest.md similarity index 78% rename from docs/services/api/endpoints/filesystem/manifest.md rename to docs/api/endpoints/filesystem/manifest.md index 2f55ead..9f53ace 100644 --- a/docs/services/api/endpoints/filesystem/manifest.md +++ b/docs/api/endpoints/filesystem/manifest.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Manifest" +description: "Endpoint returning the manifest of a file, including preview and infoType details." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- File manifest ------------- diff --git a/docs/services/api/endpoints/filesystem/metadata.md b/docs/api/endpoints/filesystem/metadata.md similarity index 94% rename from docs/services/api/endpoints/filesystem/metadata.md rename to docs/api/endpoints/filesystem/metadata.md index b752f2e..9d5ebec 100644 --- a/docs/services/api/endpoints/filesystem/metadata.md +++ b/docs/api/endpoints/filesystem/metadata.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Metadata" +description: "Endpoints for reading, setting, and copying AVU metadata on data objects and collections." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Metadata --------------------------- diff --git a/docs/services/api/endpoints/filesystem/move.md b/docs/api/endpoints/filesystem/move.md similarity index 91% rename from docs/services/api/endpoints/filesystem/move.md rename to docs/api/endpoints/filesystem/move.md index 57712b7..b3bf7b7 100644 --- a/docs/services/api/endpoints/filesystem/move.md +++ b/docs/api/endpoints/filesystem/move.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Move" +description: "Endpoints for moving files and directories, individually or in bulk." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Moving Files and/or Directories -------------- __URL Path__: /secured/filesystem/move diff --git a/docs/services/api/endpoints/filesystem/ore.md b/docs/api/endpoints/filesystem/ore.md similarity index 54% rename from docs/services/api/endpoints/filesystem/ore.md rename to docs/api/endpoints/filesystem/ore.md index 8ecaf98..4ba1d9f 100644 --- a/docs/services/api/endpoints/filesystem/ore.md +++ b/docs/api/endpoints/filesystem/ore.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "OAI-ORE" +description: "Endpoint for generating OAI-ORE descriptions of a dataset." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Generating OAI-ORE files for a data set. Secured Endpoint: POST /secured/filesystem/{data-id}/ore/save diff --git a/docs/services/api/endpoints/filesystem/path-lists.md b/docs/api/endpoints/filesystem/path-lists.md similarity index 52% rename from docs/services/api/endpoints/filesystem/path-lists.md rename to docs/api/endpoints/filesystem/path-lists.md index 68e94cd..6a272f5 100644 --- a/docs/services/api/endpoints/filesystem/path-lists.md +++ b/docs/api/endpoints/filesystem/path-lists.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Path lists" +description: "Endpoints for creating HT path list files from a set of Data Store paths." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- HT Path List Creator -------------------- diff --git a/docs/services/api/endpoints/filesystem/permissions.md b/docs/api/endpoints/filesystem/permissions.md similarity index 84% rename from docs/services/api/endpoints/filesystem/permissions.md rename to docs/api/endpoints/filesystem/permissions.md index 3e1768b..5098158 100644 --- a/docs/services/api/endpoints/filesystem/permissions.md +++ b/docs/api/endpoints/filesystem/permissions.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Permissions" +description: "Endpoints for listing and updating user permissions on data." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Listing User Permissions ------------------------ diff --git a/docs/services/api/endpoints/filesystem/read-chunk.md b/docs/api/endpoints/filesystem/read-chunk.md similarity index 83% rename from docs/services/api/endpoints/filesystem/read-chunk.md rename to docs/api/endpoints/filesystem/read-chunk.md index f43a306..277829a 100644 --- a/docs/services/api/endpoints/filesystem/read-chunk.md +++ b/docs/api/endpoints/filesystem/read-chunk.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Read chunk" +description: "Endpoints for reading a byte range or page of a file without downloading it." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Reading a chunk of a file ------------------------- diff --git a/docs/services/api/endpoints/filesystem/rename.md b/docs/api/endpoints/filesystem/rename.md similarity index 80% rename from docs/services/api/endpoints/filesystem/rename.md rename to docs/api/endpoints/filesystem/rename.md index cd423b6..ef590ab 100644 --- a/docs/services/api/endpoints/filesystem/rename.md +++ b/docs/api/endpoints/filesystem/rename.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Rename" +description: "Endpoint for renaming a file or directory in place." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Renaming a File or Directory ---------------------------- __URL Path__: /secured/filesystem/rename diff --git a/docs/services/api/endpoints/filesystem/restore.md b/docs/api/endpoints/filesystem/restore.md similarity index 93% rename from docs/services/api/endpoints/filesystem/restore.md rename to docs/api/endpoints/filesystem/restore.md index b332b84..bab3401 100644 --- a/docs/services/api/endpoints/filesystem/restore.md +++ b/docs/api/endpoints/filesystem/restore.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Restore" +description: "Endpoints for restoring files and directories from a user's trash." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Restoring a file or directory from a user's trash ------------------------------------------------- diff --git a/docs/services/api/endpoints/filesystem/root-listing.md b/docs/api/endpoints/filesystem/root-listing.md similarity index 81% rename from docs/services/api/endpoints/filesystem/root-listing.md rename to docs/api/endpoints/filesystem/root-listing.md index 59f22c4..21a6ff3 100644 --- a/docs/services/api/endpoints/filesystem/root-listing.md +++ b/docs/api/endpoints/filesystem/root-listing.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Root listing" +description: "Endpoint returning the top-level directories visible to the caller." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Top Level Root Listing ---------------------- diff --git a/docs/services/api/endpoints/filesystem/search.md b/docs/api/endpoints/filesystem/search.md similarity index 96% rename from docs/services/api/endpoints/filesystem/search.md rename to docs/api/endpoints/filesystem/search.md index 4e50b02..2fde83e 100644 --- a/docs/services/api/endpoints/filesystem/search.md +++ b/docs/api/endpoints/filesystem/search.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Search" +description: "Endpoints for searching data by name, metadata, and permissions." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- This document describes the endpoints used to performing searches of user data. # Table of Contents diff --git a/docs/services/api/endpoints/filesystem/sharing.md b/docs/api/endpoints/filesystem/sharing.md similarity index 76% rename from docs/services/api/endpoints/filesystem/sharing.md rename to docs/api/endpoints/filesystem/sharing.md index bb510c3..e512a39 100644 --- a/docs/services/api/endpoints/filesystem/sharing.md +++ b/docs/api/endpoints/filesystem/sharing.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Sharing" +description: "Endpoints for sharing and unsharing data with other users." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Sharing & Unsharing ------------------- diff --git a/docs/services/api/endpoints/filesystem/stat.md b/docs/api/endpoints/filesystem/stat.md similarity index 87% rename from docs/services/api/endpoints/filesystem/stat.md rename to docs/api/endpoints/filesystem/stat.md index 788172c..df90dfe 100644 --- a/docs/services/api/endpoints/filesystem/stat.md +++ b/docs/api/endpoints/filesystem/stat.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Stat" +description: "Endpoints returning status information for files and directories." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- File and Directory Status Information ------------------------------------- diff --git a/docs/services/api/endpoints/filesystem/tickets.md b/docs/api/endpoints/filesystem/tickets.md similarity index 91% rename from docs/services/api/endpoints/filesystem/tickets.md rename to docs/api/endpoints/filesystem/tickets.md index 56c1c66..bda30fe 100644 --- a/docs/services/api/endpoints/filesystem/tickets.md +++ b/docs/api/endpoints/filesystem/tickets.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Tickets" +description: "Endpoints for creating, listing, and deleting iRODS tickets for anonymous access." +tags: [api, terrain, endpoints, filesystem] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Creating Tickets ---------------- __URL Path__: /secured/filesystem/tickets diff --git a/docs/api/endpoints/index.md b/docs/api/endpoints/index.md new file mode 100644 index 0000000..c76e018 --- /dev/null +++ b/docs/api/endpoints/index.md @@ -0,0 +1,35 @@ +# Terrain endpoints + +Endpoint documentation by resource. The generated Swagger reference at +[de.cyverse.org/terrain/docs](https://de.cyverse.org/terrain/docs){target=_blank} +is the most current source; these documents cover endpoints that are not yet in it, +and add detail where the generated reference is thin. + +* [Endpoints overview](endpoints.md) - how this documentation is organized + +# Apps and analyses + +* [App metadata](app-metadata.md) - metadata attached to apps +* [App ontologies](app-ontologies.md) - ontology hierarchies used to classify apps +* [Quick launches](quick-launches.md) - saved launch configurations and their sharing +* [Callbacks](callbacks.md) - status reported back to the DE by other services +* [Terrain and apps services](terrain-v-apps.md) - how responsibilities are split + +# Data + +* [File I/O](fileio.md) - upload, download, and streaming +* [filesystem/](filesystem/) - listings, metadata, permissions, sharing, and trash + +# Users + +* [Notifications](notifications.md) - user notifications and system messages +* [Comments](comments.md) - comments on data and apps +* [Favorites](favorites.md) - favorite data items and apps + +# Publishing + +* [Permanent ID requests](permanent-id-requests.md) - the DOI and ARK request workflow + +# Other + +* [Miscellaneous](misc.md) - endpoints outside the groups above diff --git a/docs/services/api/endpoints/misc.md b/docs/api/endpoints/misc.md similarity index 94% rename from docs/services/api/endpoints/misc.md rename to docs/api/endpoints/misc.md index 447c621..efe5d92 100644 --- a/docs/services/api/endpoints/misc.md +++ b/docs/api/endpoints/misc.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Miscellaneous" +description: "Endpoints that do not belong to a larger resource group." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- **Jump to:** * [Miscellaneous Terrain Endpoints](#miscellaneous-terrain-endpoints) @@ -20,7 +28,7 @@ # Miscellaneous Terrain Endpoints -Note that secured endpoints in Terrain and apps are a little different from each other. Please see [Terrain Vs. Apps](terrain-v-apps.md) for more information. +Note that secured endpoints in Terrain and apps are a little different from each other. Please see [Terrain Vs. Apps](./terrain-v-apps.md) for more information. ## Verifying that Terrain is Running diff --git a/docs/services/api/endpoints/notifications.md b/docs/api/endpoints/notifications.md similarity index 98% rename from docs/services/api/endpoints/notifications.md rename to docs/api/endpoints/notifications.md index 945c1e5..22cfa26 100644 --- a/docs/services/api/endpoints/notifications.md +++ b/docs/api/endpoints/notifications.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Notifications" +description: "Endpoints for listing, marking, and deleting user notifications and system messages." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- **Jump to:** * [Notification Endpoints](#notification-endpoints) @@ -20,7 +28,7 @@ * [Sending an Arbitrary Notification](#sending-an-arbitrary-notification) -* [Endpoints for System Messages (a.k.a. System Notifications)](#endpoints-for-system-messages-(a.k.a.-system-notifications)) +* [Endpoints for System Messages (a.k.a. System Notifications)](#endpoints-for-system-messages-aka-system-notifications) # Notification Endpoints diff --git a/docs/services/api/endpoints/permanent-id-requests.md b/docs/api/endpoints/permanent-id-requests.md similarity index 95% rename from docs/services/api/endpoints/permanent-id-requests.md rename to docs/api/endpoints/permanent-id-requests.md index 9df2ced..1d498d0 100644 --- a/docs/services/api/endpoints/permanent-id-requests.md +++ b/docs/api/endpoints/permanent-id-requests.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Permanent ID requests" +description: "Endpoints supporting the DOI and ARK request workflow for published data." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Permanent ID Requests These endpoints create and manage user requests for persistent identifiers diff --git a/docs/services/api/endpoints/quick-launches.md b/docs/api/endpoints/quick-launches.md similarity index 91% rename from docs/services/api/endpoints/quick-launches.md rename to docs/api/endpoints/quick-launches.md index 0b447cc..ba85824 100644 --- a/docs/services/api/endpoints/quick-launches.md +++ b/docs/api/endpoints/quick-launches.md @@ -1,3 +1,11 @@ +--- +type: API Endpoint +title: "Quick launches" +description: "Endpoints for saved app launch configurations and their sharing." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Quick Launches Quick Launches provide a way to set default parameter values for an analysis, diff --git a/docs/services/api/endpoints/terrain-v-apps.md b/docs/api/endpoints/terrain-v-apps.md similarity index 85% rename from docs/services/api/endpoints/terrain-v-apps.md rename to docs/api/endpoints/terrain-v-apps.md index 681ea7a..df2b7fd 100644 --- a/docs/services/api/endpoints/terrain-v-apps.md +++ b/docs/api/endpoints/terrain-v-apps.md @@ -1,10 +1,18 @@ +--- +type: API Endpoint +title: "Terrain and apps services" +description: "How responsibilities are split between the Terrain facade and the apps service." +tags: [api, terrain, endpoints] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Overview The "secured" endpoints in apps behave a little bit differently from the secured endpoints in Terrain. Specifically, the secured endpoints in Terrain actually require user authentication whereas those in apps merely require information about the user making the request. ## Calling Secured Terrain Endpoints -All secured endpoints in Terrain require an HTTP header containing a JSON Web Token (JWT). The specific header to use depends on the format of the JWT, but in most cases it will be `X-Iplant-De-Jwt`. Please see the [root API documentation page](../../api_overview.md) for details. +All secured endpoints in Terrain require an HTTP header containing a JSON Web Token (JWT). The specific header to use depends on the format of the JWT, but in most cases it will be `X-Iplant-De-Jwt`. Please see the [root API documentation page](../terrain.md) for details. ## Calling "Secured" Apps Endpoints diff --git a/docs/services/api/errors.md b/docs/api/errors.md similarity index 74% rename from docs/services/api/errors.md rename to docs/api/errors.md index 68b2c23..fa9bd79 100644 --- a/docs/services/api/errors.md +++ b/docs/api/errors.md @@ -1,3 +1,11 @@ +--- +type: Reference +title: "API errors" +description: "Error response format and status codes returned by Terrain." +tags: [api, errors] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Errors If a service call causes an exception that is not caught by the service itself then Terrain will respond with a standardized error message: diff --git a/docs/api/index.md b/docs/api/index.md new file mode 100644 index 0000000..636fd43 --- /dev/null +++ b/docs/api/index.md @@ -0,0 +1,16 @@ +# API + +* [Terrain](terrain.md) - what the API fronts, how to authenticate, and where the generated reference lives +* [Endpoint index](endpoint-index.md) - alphabetical index of every documented endpoint +* [API errors](errors.md) - error response format and status codes +* [Tapis v2 to v3 migration](tapis-v2-v3-migration.md) - migrating legacy Agave apps + +# Endpoints + +* [endpoints/](endpoints/) - endpoint documentation by resource +* [endpoints/filesystem/](endpoints/filesystem/) - Data Store operations + +# Related + +* [Authentication](../platform/authentication.md) +* [Discovery Environment](../platform/discovery-environment.md) diff --git a/docs/services/api/tapis-v2-v3-migration.md b/docs/api/tapis-v2-v3-migration.md similarity index 98% rename from docs/services/api/tapis-v2-v3-migration.md rename to docs/api/tapis-v2-v3-migration.md index bb62986..b418b7a 100644 --- a/docs/services/api/tapis-v2-v3-migration.md +++ b/docs/api/tapis-v2-v3-migration.md @@ -1,3 +1,11 @@ +--- +type: Guide +title: "Tapis v2 to v3 migration" +description: "Migrating Tapis v2 (Agave) apps to Tapis v3 for the Discovery Environment." +tags: [api, tapis, migration] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Tapis v2 (Agave) App migration to v3 for the Discovery Environment The [Tapis v3 App docs](https://tapis.readthedocs.io/en/latest/technical/apps.html) diff --git a/docs/services/api_overview.md b/docs/api/terrain.md similarity index 76% rename from docs/services/api_overview.md rename to docs/api/terrain.md index 7b4bbfc..2c4edec 100644 --- a/docs/services/api_overview.md +++ b/docs/api/terrain.md @@ -1,3 +1,11 @@ +--- +type: API Overview +title: "Terrain" +description: "The Terrain API: what it fronts, how to authenticate, and where the generated reference lives." +tags: [api, terrain] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # :material-terrain: Terrain Terrain provides the primary point of communication between the Discovery Environment (DE) UI and backend services. @@ -17,11 +25,11 @@ and those endpoints will be listed under the Swagger category. The documentation for those endpoints may still be found below. -* [Endpoints](./api/endpoints/endpoints.md) +* [Endpoints](./endpoints/endpoints.md) -* [Endpoints Index](./api/endpoint-index.md) +* [Endpoints Index](./endpoint-index.md) -* [Errors](./api/errors.md) +* [Errors](./errors.md) ## Authentication @@ -32,4 +40,4 @@ allowing clients to obtain OAuth or OIDC tokens for accessing secured API endpoi See the [/terrain/token/keycloak](https://de.cyverse.org/terrain/docs/index.html#!/token/get_terrain_token_keycloak) endpoint docs for details on obtaining tokens. -## [Tapis v2 App migration to v3 for the Discovery Environment](./api/tapis-v2-v3-migration.md) +## [Tapis v2 App migration to v3 for the Discovery Environment](./tapis-v2-v3-migration.md) diff --git a/docs/architecture/component-inventory.md b/docs/architecture/component-inventory.md new file mode 100644 index 0000000..45f33e2 --- /dev/null +++ b/docs/architecture/component-inventory.md @@ -0,0 +1,125 @@ +--- +type: Reference +title: "Component inventory and sizing" +description: "The components a CyVerse deployment is built from, what each one needs, and what depends on what." +tags: [architecture, sizing, capacity, dependencies] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# Components + +A minimal CyVerse deployment is five moving parts plus the DE service set: + +| Component | Role | +|-----------|------| +| HAProxy | Single public entry point; terminates HTTPS and forwards to cluster node ports | +| PostgreSQL | Catalog (iCAT) database plus one database per DE service | +| RabbitMQ | AMQP message bus between iRODS, the DE, and indexing services | +| iRODS 4.3.3 | Data Store catalog provider and storage vault | +| Discovery Environment | The DE service set, split into non-analysis services and analyses (VICE) | + +# Resource requirements + +Per-component requirements, as sized for the two-node pilot: + +| Component | Cores | Memory | Storage | +|-----------|------:|-------:|--------:| +| HAProxy | 4 | 8 GB | — | +| PostgreSQL | 22 | 56 GB | 510 GB | +| RabbitMQ | 1 | 2 GB | 20 GB | +| iRODS | 20 | 120 GB | 110 TB | +| DE without analyses | 24 | 56 GB | 1.4 TB | +| DE analyses (VICE) | 48 | 192 GB | 3 TB | + +!!! note "Storage figure for iRODS" + + The pilot record quotes iRODS storage as both 100 TB and 110 TB. 110 TB is + the figure its own totals are computed from, so that is what this table + carries. Treat it as the data vault target for a pilot, not a fixed + requirement — the vault is sized to the science, and is the one number that + grows without bound. + +## Two-node allocation + +The pilot placed everything except analyses on the first node: + +| First node (`core-1`) | Cores | Memory | Storage | +|-----------------------|------:|-------:|--------:| +| Physical capacity | 192 | 2.3 TB | 610 TB | +| Required by HAProxy, PostgreSQL, RabbitMQ | 27 | 66 GB | 530 GB | +| Required by iRODS | 20 | 120 GB | 110 TB | +| Required by DE (no analyses) | 24 | 56 GB | 1.4 TB | +| **Total required** | **71** | **242 GB** | **112 TB** | +| Over-allocation ratio (capacity ÷ required) | 2.7× | 9.5× | 5.4× | + +The totals are the exact sums of the rows above them; the pilot record rounded +them to 250 GB and 120 TB, which shifts the storage ratio it quotes from 5.4× +to 5.0×. Nothing downstream depends on the rounding. + +Headroom is not left idle. The pilot reserved the surplus by workload so that +one component cannot starve another: + +| Reservation | Cores | Memory | Storage | +|-------------|------:|-------:|--------:| +| Scaled iRODS reservation | 54 | 1.2 TB | 550 TB | +| Scaled DE reservation | 65 | 540 GB | 7 TB | +| Unreserved (HAProxy, PostgreSQL, RabbitMQ) | 73 | 560 GB | 53 TB | + +The second node carries only DE analyses, and is sized from the DE analyses row +above plus whatever concurrency target the site sets for VICE. + +!!! tip "Deriving PostgreSQL settings from this table" + + Several `postgresql.conf` values are functions of the memory and cores + reserved for the database rather than of total machine capacity — see + [PostgreSQL tuning](../deployment/01-foundation/postgresql.md#tuning). + Decide the reservation first, then compute. + +# Dependencies + +```mermaid +graph TD + PG[PostgreSQL] + MQ[RabbitMQ] + HA[HAProxy] + IRODS[iRODS] + DE[Discovery Environment] + + IRODS --> PG + IRODS --> MQ + DE --> HA + DE --> PG + DE --> MQ + DE --> IRODS +``` + +Read as "depends on": + +* **iRODS** → PostgreSQL, RabbitMQ +* **Discovery Environment** → HAProxy, PostgreSQL, RabbitMQ, iRODS + +That graph is why the deployment order in +[deployment](../deployment/index.md) is what it is: nothing in a later phase +can come up before its dependencies in an earlier one. + +# Beyond the pilot + +The two-node shape is the smallest useful deployment, not the production +target. Scaling is additive rather than a redesign: + +* More VICE capacity means more workers in the `k8s_vice_workers` inventory + group. +* More data capacity means more iRODS resources, which may live on separate + hardware in the same zone. +* Separating PostgreSQL onto its own host is the first split most sites make, + because the DE and the catalog compete for the same buffer cache. + +For the shape of the production US CyVerse deployment, see +[system overview](./system-overview.md). diff --git a/docs/architecture/index.md b/docs/architecture/index.md new file mode 100644 index 0000000..d694fd0 --- /dev/null +++ b/docs/architecture/index.md @@ -0,0 +1,11 @@ +# Architecture + +* [System overview](system-overview.md) - the CyVerse stack end to end: interfaces, APIs, compute, storage +* [Component inventory and sizing](component-inventory.md) - what a deployment is made of, what each part needs, and what depends on what +* [Network requirements](network-requirements.md) - every port that has to be open, and to whom +* [Kubernetes namespaces](namespaces.md) - how workloads are grouped in the cluster + +# Related + +* [platform/](../platform/) - the products and services these components deliver +* [deployment/](../deployment/) - building it, in dependency order diff --git a/docs/architecture/namespaces.md b/docs/architecture/namespaces.md new file mode 100644 index 0000000..7aa14ac --- /dev/null +++ b/docs/architecture/namespaces.md @@ -0,0 +1,75 @@ +--- +type: Reference +title: "Kubernetes namespaces" +description: "Namespaces used by a CyVerse Kubernetes deployment and what runs in each one." +tags: [architecture, kubernetes, namespaces] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# How namespaces are used + +CyVerse groups workloads into namespaces along two lines: what has to be isolated +for security (user containers), and what has its own lifecycle (add-ons that are +installed and upgraded independently of the DE service set). + +The DE service namespace is conventionally `prod`. A site running more than one +environment in a cluster names its namespaces after the environments; where a +document in this bundle says ``, that is the choice it refers to. + +![Namespaces](../assets/namespaces.png) + +# The DE service namespace + +`prod` in a standard deployment. It holds the DE service set and the supporting +services the DE talks to directly: + +| Workload | Document | +|----------|----------| +| DE services (Terrain, apps, analyses, metadata, notifications, search, UI) | [Discovery Environment](../deployment/06-applications/discovery-environment.md) | +| `de-nginx` front end | [Discovery Environment](../deployment/06-applications/discovery-environment.md) | +| Redis and Redis HAProxy | [Redis HA](../deployment/05-core-services/redis-ha.md) | +| Search cluster | [OpenSearch](../deployment/05-core-services/opensearch.md) | +| Grouper loader and web services | [Grouper](../deployment/05-core-services/grouper.md) | +| Unleash | [Unleash](../deployment/05-core-services/unleash.md) | +| NATS | [NATS](../deployment/05-core-services/nats.md) | +| User Portal (or its own `user-portal` namespace) | [User Portal](../deployment/06-applications/user-portal.md) | + +# Dedicated namespaces + +| Namespace | Contents | Why it is separate | +|-----------|----------|--------------------| +| `vice-apps` | Interactive analyses, `app-exposer`, the VICE operator | User-supplied containers need their own network policy and service accounts — see [VICE](../deployment/06-applications/vice.md) | +| `keycloak` | Keycloak | Authentication is upgraded on its own schedule — see [Keycloak](../deployment/05-core-services/keycloak.md) | +| `openldap` | OpenLDAP | System of record for accounts — see [OpenLDAP](../deployment/05-core-services/openldap.md) | +| `irods-csi-driver` | The iRODS CSI driver | Node-level storage plugin with its own upgrade procedure — see [iRODS CSI driver](../deployment/05-core-services/irods-csi-driver.md) | +| `longhorn-system` | Longhorn | Cluster storage — see [storage](../deployment/04-kubernetes/storage.md) | +| `openebs` | OpenEBS (legacy) | Older cluster storage — see [storage](../deployment/04-kubernetes/storage.md) | +| `ingress-nginx` | ingress-nginx | VICE ingresses; being retired — see [ingress](../deployment/04-kubernetes/ingress.md) | +| `cert-manager` | cert-manager and cluster issuers | TLS issuance — see [cert-manager](../deployment/04-kubernetes/cert-manager.md) | +| `argo` | Argo Workflows | Batch analyses — see [Argo](../deployment/04-kubernetes/argo.md) | +| `harbor` | Harbor registry | Registry lifecycle — see [Harbor](../deployment/04-kubernetes/harbor.md) | +| `mail` | exim4 smarthost | Optional; see [mail](../deployment/05-core-services/mail.md) | +| `jaeger` | Jaeger collector, query, rollover cron | Optional tracing — see [Jaeger](../deployment/05-core-services/jaeger.md) | + +Which of these exist depends on what you deployed: Longhorn or OpenEBS, OpenSearch +or Elasticsearch, mail and tracing only if installed. + +# Practical notes + +* **Namespaced manifests.** Several manifests in + [cluster resources](../deployment/04-kubernetes/resources.md) carry a namespace + in a kustomization or in an argument. Deploying into a namespace other than the + default means changing them; each document flags where. +* **Cross-namespace addresses.** In-cluster names are + `.` — `ldap://openldap.openldap` for the directory, + `http://vice-operator.vice-apps:10000` for the VICE operator. +* **`kubectl` scope.** Most troubleshooting starts with + `kubectl get pods -A`; per-namespace commands in this bundle use `` + wherever the value is a site choice. + +# Related + +* [System overview](./system-overview.md) +* [Deployment](../deployment/index.md) +* [Network requirements](./network-requirements.md) diff --git a/docs/architecture/network-requirements.md b/docs/architecture/network-requirements.md new file mode 100644 index 0000000..4ba967e --- /dev/null +++ b/docs/architecture/network-requirements.md @@ -0,0 +1,103 @@ +--- +type: Reference +title: "Network requirements" +description: "Every port a CyVerse deployment needs open, who needs to reach it, and why." +tags: [architecture, networking, firewall, ports] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# How to read this + +Open these before installing anything. A closed port in this table shows up +later as a timeout in an Ansible task or a pod stuck in `CrashLoopBackOff`, +several phases away from the cause. + +Audience columns use three scopes: + +* **Public** — reachable from anywhere users or clients connect from. +* **Internal** — between deployment nodes only. +* **Admin** — from administrator subnets only, never public. + +# Public + +| Port | Protocol | Service | Notes | +|------|----------|---------|-------| +| 80 | TCP | HAProxy → DE | Redirects to HTTPS | +| 443 | TCP | HAProxy → DE and VICE | Must be reachable from everywhere the DE and VICE are used | + +# Data transfer + +| Port | Protocol | Service | Reachable from | +|------|----------|---------|----------------| +| 1247 | TCP | iRODS provider | Analysis nodes and any host transferring data in or out | +| 20000–20199 | TCP | iRODS parallel transfer range | Same as above | +| 20000–20199 | UDP | iRODS parallel transfer range | Same as above | + +The port range is a configuration choice made at install time; if you narrow it, +narrow it in `server_config.json` and in the firewall together, or transfers +stall after the control connection succeeds. + +# Backing services + +| Port | Protocol | Service | Reachable from | +|------|----------|---------|----------------| +| 5432 | TCP | PostgreSQL | Both deployment nodes, the pod network, and admin hosts | +| 5672 | TCP | RabbitMQ (AMQP) | Analysis nodes and in-cluster services | +| 15672 | TCP | RabbitMQ management UI | Admin only | + +The pod network is not knowable until the cluster exists; see +[phase 4.5](../deployment/from-scratch.md#45-let-the-pods-reach-postgresql) for +adding the pod CIDR to `pg_hba.conf` after the fact. + +# Kubernetes + +Control-plane and worker ports for a k0s cluster: + +| Port | Protocol | Purpose | Scope | +|------|----------|---------|-------| +| 6443 | TCP | Kubernetes API server | Internal + admin | +| 9443 | TCP | k0s join API | Internal + admin | +| 2380 | TCP | etcd peer traffic | Internal | +| 10250 | TCP | Kubelet metrics | Internal + admin | +| 8132 | TCP | Konnectivity | Internal | +| 179 | TCP | BGP (Calico) | Internal | +| 4789 | UDP | VXLAN (Calico) | Internal | +| — | IP protocol 112 | VRRP (keepalived) | Internal | +| 31343 | TCP | Traefik HTTP node port | Internal, from HAProxy | +| 31344 | TCP | Traefik HTTPS node port | Internal, from HAProxy | + +!!! note "Corrections to watch for" + + * **VRRP is not a TCP port.** keepalived uses IP protocol 112, so the + firewall rule is a protocol rule, not a port rule. Deployments with a + single control-plane node do not run keepalived at all and can drop it. + * **BGP and VXLAN depend on the CNI configuration.** A single-node control + plane with VXLAN encapsulation may not need `179/tcp`. Confirm against + your own Calico configuration rather than opening it by default. + * **The Traefik node ports are configurable.** If you change them, change + the HAProxy back end to match. + +# Outbound + +| Destination | Protocol | Purpose | +|-------------|----------|---------| +| DNS provider API | HTTPS | Let's Encrypt DNS-01 challenge for cert-manager | +| Container registries | HTTPS | Pulling service and VICE images | +| Package repositories | HTTPS | OS, iRODS, and Helm chart installation | + +Outbound HTTPS to the DNS API is optional in the sense that the deployment +comes up without it, and strongly recommended in the sense that certificate +renewal is otherwise a recurring manual chore. + +# Related + +* [Component inventory and sizing](./component-inventory.md) +* [Ingress deployment](../deployment/04-kubernetes/ingress.md) +* [Deploying from scratch](../deployment/from-scratch.md) diff --git a/docs/services/system_overview.md b/docs/architecture/system-overview.md similarity index 76% rename from docs/services/system_overview.md rename to docs/architecture/system-overview.md index 28489e6..4679850 100644 --- a/docs/services/system_overview.md +++ b/docs/architecture/system-overview.md @@ -1,4 +1,11 @@ - +--- +type: Architecture Overview +title: "System overview" +description: "The CyVerse stack end to end: interfaces, APIs, compute, storage, and the resources behind them." +tags: [architecture, overview] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- [de]: ../assets/de/deIcon.svg [data]: ../assets/de/dataIcon.svg [cacao]: ../assets/de/cacao-04.png @@ -10,9 +17,9 @@ The US public CyVerse primarily runs on hardware located at The University of Ar The full CyVerse SaaS stack can be deployed either on-premises consumer hardware or on cloud resources. -Data storage is managed by an iRODS [![data]{width=25} Data Store](ds.md). +Data storage is managed by an iRODS [![data]{width=25} Data Store](../platform/data-store.md). -Computing can be done in either the [![de]{width=25} Discovery Environment (DE)](de.md) data science workbench or with the [![cacao]{width=25} CACAO IaC](cloud.md) which leverages both public research computing and commercial cloud. +Computing can be done in either the [![de]{width=25} Discovery Environment (DE)](../platform/discovery-environment.md) data science workbench or with the [![cacao]{width=25} CACAO IaC](../platform/cloud.md) which leverages both public research computing and commercial cloud. Event-based triggers are accomplished through the DataWatch API. @@ -22,6 +29,13 @@ Event-based triggers are accomplished through the DataWatch API. +!!! tip "Deploying your own" + + This document describes the production US CyVerse deployment. For the smallest + useful deployment and its sizing, see + [component inventory](./component-inventory.md), and for the ordered build see + [deployment](../deployment/index.md). + ## :material-api: Application Programming Interfaces (APIs) All CyVerse APIs are [:simple-openapiinitiative: OpenAPI](https://www.openapis.org/) compliant. @@ -39,7 +53,7 @@ CyVerse public-facing APIs are frequently leveraged by "[Powered-by-CyVerse](htt ## :octicons-cloud-24: Cloud Services -[![][cacao]{width=25}](https://cyverse.org/cacao){target=_blank} [Continous Automation / Continuous Analysis & Orchestration (CACAO)](https://cyverse.org/cacao){target=_blank} - Infrastructure as Code for multi-cloud deployments +[![][cacao]{width=25}](https://cyverse.org/cacao){target=_blank} [Continuous Automation / Continuous Analysis & Orchestration (CACAO)](https://cyverse.org/cacao){target=_blank} - Infrastructure as Code for multi-cloud deployments * [:simple-terraform: CACAO Terraform Templates](https://gitlab.com/cyverse/cacao-tf-os-ops/){target=_blank} * [:octicons-stopwatch-24: DataWatch](https://gitlab.com/cyverse/datawatch){target=_blank} - a notification system for reporting data events @@ -48,48 +62,48 @@ CyVerse public-facing APIs are frequently leveraged by "[Powered-by-CyVerse](htt The DE runs on-premises hardware located at University of Arizona (UArizona) in the UITS colocation space at the high performance computing center. The data store is mirrored nightly at TACC. -CyVerse staff maintain over XXX servers at UArizona and 1 server at TACC. +CyVerse staff maintain several hundred servers at UArizona and a mirror at TACC. Exact counts change as hardware is added, replaced, or retired. -Hardware is added, replaced, or upgraded every few months. Table values below may not be up-to-date. +Hardware is added, replaced, or upgraded every few months. The tables below record the shape of the deployment rather than a current inventory; a dash means the aggregate is not tracked in this document. **Primary Hardware Specifications** -Compute Nodes (XXX nodes) +Compute nodes | System Configuration | Aggregate information | Per Node (Compute Node) | |----------------------|-----------------------|-------------------------| -| Machine types | Dell, SuperMicro, XXX | | -| Operating Systems | Centos, Rocky | Centos, Rocky | -| Processor cores | XX,XXX | average XX | +| Machine types | Dell, SuperMicro | | +| Operating systems | Rocky, CentOS | Rocky, CentOS | +| Processor cores | — | varies by generation | | CPUs | 128, 64, 40, 32, 16 | 1, 2 | -| RAM | XXX TiB | 256, 128, 64, 32 GiB | +| RAM | — | 256, 128, 64, 32 GiB | | Network | 100 Gbps to Internet2 | 10 Gpbs to switch | -| Storage | X PB | X TB | +| Storage | — | varies by node | -GPU Nodes (XXX nodes) +GPU nodes | System Configuration | Aggregate information | Per Node (Compute Node) | |----------------------|-----------------------|-------------------------| -| Machine types | Dell, SuperMicro, XXX | | -| Operating Systems | Centos, Rocky | Centos, Rocky | +| Machine types | Dell, SuperMicro | | +| Operating systems | Rocky, CentOS | Rocky, CentOS | | Processor cores | | 256 | | CPUs | | 2 | | RAM | | 1 TB, 512 GB | | GPUs | NVIDIA (A100 80GB), (Tesla T4 16GB) | 4 | | Network | 100 Gbps to Internet2 | 10 Gpbs to switch | -| Storage | XXX TB | 28 TB SSD, 21 TB NVMe| +| Storage | — | 28 TB SSD, 21 TB NVMe | -Storage Resource Nodes (44 nodes) +Storage resource nodes (44 nodes) | System Configuration | Aggregate information | Per Node (Compute Node) | |----------------------|-----------------------|-------------------------| -| Machine types | Dell, SuperMicro, XXX | | -| Operating Systems | Centos, Ubuntu | Centos, Ubuntu | -| Processor cores | 1506 | average XX | +| Machine types | Dell, SuperMicro | | +| Operating systems | Rocky, CentOS, Ubuntu | Rocky, CentOS, Ubuntu | +| Processor cores | 1,506 | varies by generation | | CPUs | 128, 64, 40, 32, 16 | 1, 2 | | RAM | 11 TiB | 256, 128, 64, 32 GiB | | Network | 100 Gbps to Internet2 | 10 Gpbs to switch | -| Storage | 18 PB | X TB | +| Storage | 18 PB | varies by node | ### :simple-kubernetes: Federated Kubernetes Clusters @@ -126,7 +140,7 @@ CyVerse is partnered with [Texas Advanced Computing Center (TACC)](https://www.t The CyVerse Data Store manages over 6 PB data via [iRODS (integrated Rule Oriented Data System)](https://irods.org){target=_blank} within the `iplant` zone. The zone name is related to the original project name. It is retained to preserve access to data through URLs published during the time period of the original project. -Data storage is organized into resources. The main resource is named `CyVerseRes`, and it holds all user data and most project data. Data on `CyVerseRes` are stored ath the University of Arizona. This resource is mirrored with a second resource named `taccRes` which is backed by storage located at the Texas Advanced Computing Center (TACC). There are also special purpose resources dedicated to certain projects. The data in these resources are stored on hardware owned by these projects. +Data storage is organized into resources. The main resource is named `CyVerseRes`, and it holds all user data and most project data. Data on `CyVerseRes` are stored at the University of Arizona. This resource is mirrored with a second resource named `taccRes` which is backed by storage located at the Texas Advanced Computing Center (TACC). There are also special purpose resources dedicated to certain projects. The data in these resources are stored on hardware owned by these projects. ![datastore](../assets/datastore.svg){width=500} diff --git a/docs/database/de-db.md b/docs/database/de-db.md deleted file mode 100644 index 1b210a3..0000000 --- a/docs/database/de-db.md +++ /dev/null @@ -1,103 +0,0 @@ -# DE Database - -This Database is used for CyVerse discovery environment. - -## Initialize database - -### Access Database - -```bash -ssh root@DB_HOST.com -psql -U postgres -``` - -### Create required Database and User - -```bash -# create de user -create user de with password '********'; - -# create de database -create database de with owner de; -``` - -### Add required extensions - -```bash -# psql -U postgres -\c de -create extension "uuid-ossp"; -create extension "moddatetime"; -create extension "btree_gist"; -``` - -### Populate Database -#### Clone de-database repository - -```bash -git clone https://github.com/cyverse-de/de-database.git -cd de-database -``` -#### install [golang-migrate](https://github.com/golang-migrate/migrate) -These steps are used on **Ubuntu** debian based OS. -```bash -curl -s https://packagecloud.io/install/repositories/golang-migrate/migrate/script.deb.sh | sudo bash -apt-get update -apt-get install -y migrate - -# check -migrate -help -``` - -#### Run to populate de database -**Note**: we are running the `/migration` directory from [de-database](https://github.com/cyverse-de/de-database.git) -```bash -migrate -database postgres://USER:PASSWORD@DB_HOST.com/de?sslmode=disable -path migrations up -``` - -#### Additional database queries - -```bash -# checkout to de user -psql -U de - - -SET search_path = public, pg_catalog; - -# create table version -CREATE TABLE version ( -version character varying(20) NOT NULL, -applied timestamp DEFAULT now() -); - -# populate the Version table from a file -# location of 999_version.sql: -# https://github.com/cyverse-de/de-database/edit/master/old-databases/de-db/src/main/data/999_version.sql -psql -U postgres -h localhost -d de -f 999_version.sql -``` - -### Migrate Database -Once a while upon updating the k8s services, we would require to migrate the **de-database**, to add the latest database changes. - -#### Clone latest de-database - -```bash -git clone https://github.com/cyverse-de/de-database.git -cd de-database -``` -#### install [golang-migrate](https://github.com/golang-migrate/migrate) -These steps are used on **Ubuntu** debian based OS. -```bash -curl -s https://packagecloud.io/install/repositories/golang-migrate/migrate/script.deb.sh | sudo bash -apt-get update -apt-get install -y migrate - -# check -migrate -help -``` - -#### Run to migrate de database -**Note**: we are running the `/migration` directory from [de-database](https://github.com/cyverse-de/de-database.git) -```bash -migrate -database postgres://USER:PASSWORD@DB_HOST.com/de?sslmode=disable -path migrations up -``` diff --git a/docs/database/grouper-db.md b/docs/database/grouper-db.md deleted file mode 100644 index 86f1ab7..0000000 --- a/docs/database/grouper-db.md +++ /dev/null @@ -1,40 +0,0 @@ -# Grouper Database - -This Database is used for CyVerse Grouper service. - -## Initialize database - -### Access Database - -```bash -ssh root@DB_HOST.com -psql -U postgres -``` - -### Create required Database and User - -```bash -## create grouper user -create user grouper with password '********'; - -## create grouper database -create database grouper with owner grouper; -``` - -### Add required extensions - -```bash -# psql -U postgres -\c grouper -create extension "uuid-ossp"; -create extension "moddatetime"; -create extension "btree_gist"; -``` - -### Populate Database - -**TODO** - -### Migrate Database - -**TODO** \ No newline at end of file diff --git a/docs/database/keycloak-db.md b/docs/database/keycloak-db.md deleted file mode 100644 index 7314036..0000000 --- a/docs/database/keycloak-db.md +++ /dev/null @@ -1,26 +0,0 @@ -# Keycloak Database - -This Database is used for Keycloak. - -## Initialize database - -### Access Database - -```bash -ssh root@DB_HOST.com -psql -U postgres -``` - -### Create required Database and User - -```bash -# create keycloak user -create user keycloak with password '********'; - -# create keycloak database -create database keycloak with owner keycloak; -``` - -### Migrate Database - -**TODO** \ No newline at end of file diff --git a/docs/database/main.md b/docs/database/main.md deleted file mode 100644 index 050cfaf..0000000 --- a/docs/database/main.md +++ /dev/null @@ -1,106 +0,0 @@ -# Databases - -CyVerse uses [:simple-postgresql: PostgreSQL](https://www.postgresql.org/) as its primary database platform. - -Each database is maintained in its own GitHub Repository in the core [:simple-github: CyVerse Organization](https://github.com/cyverse){target=_blank} or [:simple-github: CyVerse Discovery Environment Organization](https://github.com/cyverse-de){target=_blank} - -## :octicons-database-24: Provisioning - -[:octicons-database-24: iCAT]() - the iRODS iCAT metadata database - -[:octicons-database-24: DE](../database/de-db.md) - the Discovery Environment database - -[:octicons-database-24: Metadata](../database/metadata-db.md) - the CyVerse Metadata Service - -[:octicons-database-24: Keycloak](../database/keycloak-db.md) - the database used by Keycloak - -[:octicons-database-24: Notifications](../database/notifications-db.md) - the database used for notifications - -[:octicons-database-24: Unleash](../database/unleash-db.md) - the database used for Unleash service - -[:octicons-database-24: Grouper](../database/grouper-db.md) - the database for Grouper service - -[:octicons-database-24: QMS](../database/qms-db.md) - the database for QMS service - -[:octicons-database-24: Portal](../database/portal-db.md) - the User Portal database - -## :material-map-search-outline: User Guides - -[:fontawesome-solid-gears: DevOps](../guides/devops.md) - instructions for setting up a remote DevOps environment to manage a CyVerse deployment - -[Discovery Environment](../guides/de.md) - designed for data scientists, students, and researchers who want to use CyVerse for research. - -[Data Store](../guides/ds.md) - instructions for using the iRODS Data Store. - -**This database dedicated to the *discovery environment* of Cyverse, which is used for multiple services such as:** - -* [de](de-db.md) -* [notifications](notifications-db.md) -* [metadata](metadata-db.md) -* [unleash](unleash-db.md) -* [grouper](grouper-db.md) -* [qms](qms-db.md) -* [portal](portal-db.md) -* [keycloak](keycloak-db.md) - -**NOTE: permissions database has been merged with DE database.** - -## Install -The installation of this database is manully done on the host `DB_HOST.com'`. -See documentation on [how to install postgresql?](https://www.postgresguide.com/setup/install/) - -**Installing postgresql 12 on Centos7** - -```bash -sudo yum install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm - -sudo yum install -y postgresql12-server -sudo /usr/pgsql-12/bin/postgresql-12-setup initdb -sudo systemctl enable postgresql-12 -sudo systemctl start postgresql-12 -sudo yum install postgresql12-contrib -``` - -## Setup -On this paragraph we will cover first and necessary steps to configure the database. - - -**~postgres/12/data/pg_hba.conf** - -**IPv4 local connections:** - -Add IP or IP range of kubernetes worker node, that requires connection to this database. - -| TYPE | DATABASE | USER | ADDRESS | METHOD | -|------|----------|------|---------|--------| -|host |all | all | *******/32 | md5 | -|host |all | all | *******/32 | md5 | -|host |all | all | *******/32 | md5 | -|host |all | all | *******/32 | md5 | -|host |all | all | *******/32 | md5 | -|host |all | all | *******/32 | md5 | - -**~postgres/12/data/postgresql.conf** - -```bash -# vi ~postgres/12/data/postgresql.conf -listen_addresses = '*' # what IP address(es) to listen on; -``` - -** .pgpass** - -**TODO:** - -### Databases and its Users - -| DATABASE | USER | -|----------|------| -| de | de | -| notifications | de | -| metadata | de | -| unleash | unleash_user | -| grouper | grouper | -| portal | portal | -| keycloak | keycloak | -| qms | de | - diff --git a/docs/database/metadata-db.md b/docs/database/metadata-db.md deleted file mode 100644 index 4978bad..0000000 --- a/docs/database/metadata-db.md +++ /dev/null @@ -1,37 +0,0 @@ -# Metadata Database - -This Database is used for CyVerse Metadata service. - -## Initialize database - -### Access Database - -```bash -ssh root@DB_HOST.com -psql -U postgres -``` - -### Create required Database - -```bash -# create metadata database with de user -create database metadata with owner de; -``` - -### Add required extensions - -```bash -# psql -U postgres -\c metadata -create extension "uuid-ossp"; -create extension "moddatetime"; -create extension "btree_gist"; -``` - -### Populate Database - -**TODO** - -### Migrate Database - -**TODO** \ No newline at end of file diff --git a/docs/database/notifications-db.md b/docs/database/notifications-db.md deleted file mode 100644 index a9e24ca..0000000 --- a/docs/database/notifications-db.md +++ /dev/null @@ -1,38 +0,0 @@ -# Notifications Database - -This Database is used for CyVerse notifications service. - -## Initialize database - -### Access Database - -```bash -ssh root@DB_HOST.com -psql -U postgres -``` - -### Create required Database - -```bash -# create notifications database with de owner. -create database notifications with owner de; -``` - -### Add required extensions - -```bash -# psql -U postgres -\c notifications -create extension "uuid-ossp"; -create extension "moddatetime"; -create extension "btree_gist"; -``` - -### Populate Database - -**TODO** - -### Migrate Database - -**TODO** - diff --git a/docs/database/portal-db.md b/docs/database/portal-db.md deleted file mode 100644 index e107010..0000000 --- a/docs/database/portal-db.md +++ /dev/null @@ -1,113 +0,0 @@ -# Portal Database - -This Database is used for CyVerse User portal Service. - -## Initialize database - -### Access Database - -```bash -ssh root@DB_HOST.com -psql -U postgres -``` - -### Create required Database and User - -See also [portal2/setup-database](https://gitlab.com/cyverse/portal2#setup-database) - - -```bash -# create user -create user portal_db_reader with password '********'; - -# create portal database with the owner portal_db_reader -create database portal with owner portal_db_reader; - -## Grant user to member of postgres -# psql -U postgres -GRANT postgres TO portal_db_reader; -``` - -### Restore from dump -For this we have to download [portal.sql](https://gitlab.com/cyverse/portal2/-/blob/master/portal.sql) file. - -```bash -# restore to user portal_db_reader and database portal -psql -U portal_db_reader -d portal -f portal.sql -``` - -### Populate Database - -#### make sure session Table exist - -```bash -CREATE TABLE public.session ( - sid character varying NOT NULL, - sess json NOT NULL, - expire timestamp(6) without time zone NOT NULL -); -ALTER TABLE public.session OWNER TO portal; -CREATE INDEX "IDX_session_expire" ON public.session USING btree (expire); -``` - -#### Import GRID institutions - -##### download required grid file - -[Official website](https://digitalscience.figshare.com/articles/dataset/GRID_release_2021-09-16/16685428?backTo=/collections/GRID/3812929) - -```bash -# download grid -wget https://digitalscience.figshare.com/ndownloader/files/30895309 - -# unzip -unzip 30895309 -``` - -##### import to the database -For imorting this grid file we will use the script from [portal2/scripts/import_grid_institutions.py](https://gitlab.com/cyverse/portal2/-/blob/master/src/scripts/import_grid_institutions.py). - -```bash -./import_grid_institutions.py --host root@DB_HOST.com --user portal_db_reader --database portal grid.csv -``` - -#### Populate these Tables -**These sql files can be found [here](https://github.com/cyverse-austria/portal2-db)**. - -```bash -psql -U portal_db_reader -d portal -f ./account_country.sql -psql -U portal_db_reader -d portal -f ./account_region.sql -psql -U portal_db_reader -d portal -f ./account_gender.sql -psql -U portal_db_reader -d portal -f ./account_occupation.sql -psql -U portal_db_reader -d portal -f ./account_ethnicity.sql -psql -U portal_db_reader -d portal -f ./account_fundingagency.sql -psql -U portal_db_reader -d portal -f ./account_awarechannel.sql -psql -U portal_db_reader -d portal -f ./account_researcharea.sql -``` - -## Extra - -### Give Admin privilege to a user - -```sql ---update is_superuser -UPDATE account_user SET is_superuser = true WHERE username='USERNAME'; - ----update is_staff -UPDATE account_user SET is_staff = true WHERE username='USERNAME'; -``` - -### Verify User email - -```sql --- check if its verified -select has_verified_email from account_user where username='USERNAME'; - ---Verify email -UPDATE account_user SET has_verified_email = true WHERE username='USERNAME'; -``` - - -## Migrate Database - -**TODO** diff --git a/docs/database/qms-db.md b/docs/database/qms-db.md deleted file mode 100644 index 384d5dd..0000000 --- a/docs/database/qms-db.md +++ /dev/null @@ -1,66 +0,0 @@ -# QMS Database - -This Database is used for CyVerse QMS service. - -## Initialize database - -### Access vm - -```bash -ssh root@DB_HOST.com -``` - -### Create required Database - -```bash -# create qms database with de owner -create database qms with owner de; -``` - -### Add required extensions - -```bash -# psql -U postgres -\c de -create extension "uuid-ossp"; -create extension "moddatetime"; -create extension "btree_gist"; -create extension "insert_username"; -``` - -### Populate Database - -**TODO** - - -### Migrate Database - -Once a while upon updating the k8s services, we would require to migrate the **qms-database**, to add the latest database changes. - -#### Clone QMS repo - -```bash -git clone https://github.com/cyverse/QMS.git -git fetch && git checkout prod -cd QMS -``` - -#### install [golang-migrate](https://github.com/golang-migrate/migrate) -These steps are used on **Ubuntu** debian based OS. - -```bash -curl -s https://packagecloud.io/install/repositories/golang-migrate/migrate/script.deb.sh | sudo bash -apt-get update -apt-get install -y migrate - -# check -migrate -help -``` - -#### Run to migrate QMS database - -**Note**: we are running the `/migration` directory from [QMS](https://github.com/cyverse/QMS.git) - -```bash -migrate -database postgres://USER:PASSWORD@DB_HOST.com/qms?sslmode=disable -path migrations up -``` diff --git a/docs/database/unleash-db.md b/docs/database/unleash-db.md deleted file mode 100644 index 62d5c4c..0000000 --- a/docs/database/unleash-db.md +++ /dev/null @@ -1,39 +0,0 @@ -# Unleash Database - -This Database is used for CyVerse Unleash service. - -## Initialize database - -### Access vm - -```bash -ssh root@DB_HOST.com -``` - -### Create required Database and User - -```bash -# create unleash user -create user unleash with password '********'; - -# create unleash databased -create database unleash with owner unleash; -``` - -### Add required extensions - -```bash -# psql -U postgres -\c unleash -create extension "uuid-ossp"; -create extension "moddatetime"; -create extension "btree_gist"; -``` - -### Populate Database - -**TODO** - -### Migrate Database - -**TODO** \ No newline at end of file diff --git a/docs/deployment/01-foundation/haproxy.md b/docs/deployment/01-foundation/haproxy.md new file mode 100644 index 0000000..fb48872 --- /dev/null +++ b/docs/deployment/01-foundation/haproxy.md @@ -0,0 +1,74 @@ +--- +type: Deployment Procedure +title: "HAProxy" +description: "The public entry point that terminates HTTPS and forwards to the cluster's Traefik node ports." +tags: [deployment, foundation, haproxy, networking] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: haproxy-docs + resource: https://docs.haproxy.org/ + title: HAProxy documentation + author: team:haproxy +--- + +# Role in the deployment + +HAProxy is the single public entry point. It listens on `80/tcp` and `443/tcp` +and forwards to the node ports Traefik exposes inside the Kubernetes cluster. +Everything a user or a VICE client reaches — the DE, the User Portal, Keycloak, +interactive app URLs — arrives through it. + +In the production US deployment, a second HAProxy fronts the Data Store's +access services (`data.cyverse.org`) as described in +[Data Store](../../platform/data-store.md); a pilot deployment usually runs +only the DE-facing instance. + +# Order of operations + +HAProxy is installed early, in phase 1, because it is a plain package install +with no dependencies. It is **configured** later, in phase 4, because its back +ends are the Traefik node ports, which do not exist until the cluster does. + +1. **Phase 1** — install the `haproxy` package on the node in the + `04_haproxy` inventory group. +2. **Phase 4** — after the cluster is up and Traefik is installed, apply the + configuration: + + ```bash + ansible-playbook -i /path/to/inventory --tags haproxy kubernetes.yml + ``` + +Sizing: 4 cores, 8 GB memory, no dedicated storage. See +[component inventory](../../architecture/component-inventory.md). + +# What the configuration has to line up with + +| HAProxy front end | Back end | +|-------------------|----------| +| `80/tcp` | Redirect to HTTPS | +| `443/tcp` | Traefik HTTPS node port, `31344/tcp` by default | +| (optional) plain HTTP passthrough | Traefik HTTP node port, `31343/tcp` by default | + +Both node ports are configurable. If you change them in the Traefik values, the +HAProxy back end has to change with them — a mismatch here produces a +connection refused on the public address while every pod looks healthy. + +# TLS + +Certificates are issued in-cluster by cert-manager, and Traefik terminates TLS +for in-cluster routes. Where HAProxy terminates TLS itself, its certificate has +to be renewed alongside the cluster issuer, and its trust store has to contain +the CA that signed the back-end certificates — a private CA in a pilot means +adding that CA to the system bundle on the HAProxy host. + +# Related + +* [Network requirements](../../architecture/network-requirements.md) +* [Ingress](../04-kubernetes/ingress.md) +* [Deploying from scratch](../from-scratch.md#11-haproxy) diff --git a/docs/deployment/01-foundation/index.md b/docs/deployment/01-foundation/index.md new file mode 100644 index 0000000..87fbc1a --- /dev/null +++ b/docs/deployment/01-foundation/index.md @@ -0,0 +1,12 @@ +# Phase 1: foundation + +Host services installed before anything else. Everything in later phases depends on +at least one of them. + +* [HAProxy](haproxy.md) - public entry point; installed now, configured in phase 4 +* [PostgreSQL](postgresql.md) - the instance that backs the iRODS catalog and every service database +* [RabbitMQ](rabbitmq.md) - the AMQP bus between iRODS and the DE + +# Next + +* [Phase 2: databases](../02-databases/) diff --git a/docs/deployment/01-foundation/postgresql.md b/docs/deployment/01-foundation/postgresql.md new file mode 100644 index 0000000..52aaeaa --- /dev/null +++ b/docs/deployment/01-foundation/postgresql.md @@ -0,0 +1,181 @@ +--- +type: Deployment Procedure +title: "PostgreSQL" +description: "Installing and tuning the PostgreSQL server that backs the iRODS catalog and every CyVerse service database." +tags: [deployment, foundation, postgresql, database] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: irods-install + resource: https://docs.irods.org/4.3.3/getting_started/installation/ + title: iRODS 4.3.3 installation guide + author: team:irods-consortium + - id: postgres-docs + resource: https://www.postgresql.org/docs/ + title: PostgreSQL documentation + author: team:postgresql +--- + +# Role in the deployment + +One PostgreSQL instance carries two very different workloads: + +* the **iRODS catalog (iCAT)**, which is latency-sensitive and touched by every + data operation, and +* the **DE service databases**, which are created and migrated by Ansible. + +Because both live in the same instance in a pilot, tuning is sized against the +memory and cores you deliberately reserve for the database — not against the +machine's total capacity. Decide that reservation first, from +[component inventory](../../architecture/component-inventory.md) (the pilot +reserved 22 cores and 56 GB), then compute the settings below from it. + +Separating the catalog onto its own instance is the first split most sites make +as they grow, because the DE and the catalog otherwise compete for the same +buffer cache. + +# Host preparation + +## Transparent huge pages + +PostgreSQL uses explicit huge pages; transparent huge pages cause latency +spikes and have to be off. + +1. Append `transparent_hugepage=never` to `GRUB_CMDLINE_LINUX_DEFAULT` in + `/etc/default/grub`. +2. Run `update-grub` to write the boot loader configuration. +3. Reboot for the change to take effect. + +## Kernel parameters + +Persist these (`/etc/sysctl.d/`), do not just `sysctl -w` them: + +| Parameter | Value | +|-----------|-------| +| `vm.nr_hugepages` | `0.6 × (memory_available_to_postgres_in_kiB ÷ 2048)` | +| `vm.swappiness` | `5` | + +The huge pages figure reserves roughly 60% of the database's memory allocation +as 2 MiB pages, which is what `shared_buffers` plus overhead needs when +`huge_pages = on`. + +## Packages + +Install `postgresql`, `postgresql-client`, and `python3-psycopg2` — the last one +so Ansible's PostgreSQL modules can talk to the instance. The pilot ran +PostgreSQL 16 from the distribution packages, which puts configuration in +`/etc/postgresql/16/main/`. + +# Tuning + +Substitute your own reservation for `M` (memory available to PostgreSQL, in +kiB) and `C` (cores available to PostgreSQL): + +| Setting | Value | Why | +|---------|-------|-----| +| `max_connections` | `300` | Estimated by the deployment team for the DE service set plus iRODS agents | +| `shared_buffers` | `M ÷ 4` kB | Conventional quarter-of-memory starting point | +| `huge_pages` | `on` | Pairs with `vm.nr_hugepages` above | +| `work_mem` | `150MB` | Starting point only; tune against real query plans | +| `maintenance_work_mem` | `2GB` | Keeps index builds and vacuums off disk | +| `effective_io_concurrency` | `200` | Assumes SSD or NVMe storage | +| `max_worker_processes` | `C` | | +| `max_parallel_maintenance_workers` | `C` | | +| `max_parallel_workers_per_gather` | `C` | | +| `max_parallel_workers` | `C` | | +| `checkpoint_timeout` | `15min` | Fewer, larger checkpoints | +| `max_wal_size` | `4GB` | | +| `min_wal_size` | `1GB` | | +| `random_page_cost` | `1.1` | Assumes SSD or NVMe storage | +| `effective_cache_size` | `M ÷ 2` kB | What the planner assumes the OS caches | +| `shared_preload_libraries` | `'pg_stat_statements'` | Query-level visibility | +| `pg_stat_statements.max` | `10000` | | +| `pg_stat_statements.track` | `all` | | +| `standard_conforming_strings` | `off` | Required by the DE database | +| `listen_addresses` | `'*'` | Needed for cluster pods and the other node | + +!!! warning "`work_mem` is per sort, not per connection" + + With `max_connections = 300`, a plan with several concurrent sorts can + multiply `work_mem` well past what you expect. Treat `150MB` as a starting + point and lower it if the instance starts swapping. + +!!! warning "`standard_conforming_strings = off`" + + The DE database expects this off. Leaving it on causes some migrations to be + skipped; re-running the `update-databases` tag after fixing it picks them + up. See [troubleshooting](../07-post-install/troubleshooting.md). + +Restart PostgreSQL after editing `postgresql.conf`. + +# Prepare for iRODS + +Follow the iRODS project's instructions for preparing PostgreSQL for +iRODS,[^irods-install] which cover creating the catalog database, the `irods` +role, and its ODBC connectivity. Then create a role for the DE with a generated +password and `SELECT` on all tables in the catalog database — the DE reads the +catalog directly for some listings and never writes to it. + +# Access control + +`pg_hba.conf` is edited twice, in two different phases: + +1. **Phase 2**, before the cluster exists: allow your admin host or hosts and + both deployment nodes, with method `scram-sha-256`. +2. **Phase 4**, once the cluster exists: add the Kubernetes pod CIDR or CIDRs, + because the DE services connect from pods: + + ```bash + kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' && echo + ``` + + For each distinct CIDR add: + + ``` + host all all scram-sha-256 + ``` + +Restart PostgreSQL after each change. + +!!! danger "Do not use `trust`, and prefer `scram-sha-256` over `md5`" + + Older CyVerse deployment notes used `md5`. New deployments should use + `scram-sha-256` throughout: it is the default from PostgreSQL 14 onward, and + `md5` is deprecated upstream. + +# Databases and their owners + +Created in [phase 2](../02-databases/index.md), listed here because they all +live in this instance: + +| Database | Owner role | +|----------|------------| +| iCAT (name chosen at iRODS install time) | `irods` | +| `de` | `de` | +| `notifications` | `de` | +| `metadata` | `de` | +| `qms` | `de` | +| `unleash` | `unleash` | +| `grouper` | `grouper` | +| `portal` | `portal_db_reader` | +| `keycloak` | `keycloak` | + +!!! note "Merged databases" + + The former `permissions` database has been merged into the DE database. A + deployment created from current playbooks will not have a separate + `permissions` database, and older configuration that refers to one is stale. + +# Related + +* [Service databases](../02-databases/index.md) +* [Database migrations](../02-databases/migrations.md) +* [iRODS provider](../03-data-store/irods-provider.md) +* [Deploying from scratch](../from-scratch.md#12-postgresql) + +[^irods-install]: iRODS 4.3.3 installation guide diff --git a/docs/deployment/01-foundation/rabbitmq.md b/docs/deployment/01-foundation/rabbitmq.md new file mode 100644 index 0000000..8f29a76 --- /dev/null +++ b/docs/deployment/01-foundation/rabbitmq.md @@ -0,0 +1,131 @@ +--- +type: Deployment Procedure +title: "RabbitMQ" +description: "Installing the AMQP broker, creating the Data Store vhost and accounts, and reindexing search from the message bus." +tags: [deployment, foundation, rabbitmq, amqp] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: rabbitmq-docs + resource: https://www.rabbitmq.com/docs + title: RabbitMQ documentation + author: team:rabbitmq +--- + +# Role in the deployment + +RabbitMQ is the message bus between iRODS and the DE. iRODS publishes data +events to it; DE services consume them to keep search indexes and notifications +current. It is also how a full search reindex is triggered. + +It is a host service, not a Kubernetes workload — it is installed on the node in +phase 1, before anything that depends on it. Sizing is modest: 1 core, 2 GB +memory, 20 GB storage. + +# Install and configure + +Run on the foundation node (`core-1`): + +1. Install `rabbitmq-server` with the OS package manager. +2. Enable the management plugin and restart the service: + + ```bash + rabbitmq-plugins enable rabbitmq_management + systemctl restart rabbitmq-server + ``` + +3. Create an administrator account with a generated password, grant it full + configure, write, and read permissions, and tag it `administrator`. +4. **Delete the default `guest` account.** It ships with well-known credentials. +5. Create the vhost `/data-store` and grant the administrator full configure, + write, and read permissions on it. +6. Create a separate account for iRODS with its own generated password and full + configure, write, and read permissions on `/data-store` — iRODS should not + authenticate as the administrator. +7. Create a `topic` exchange named `irods` on `/data-store`. + +The iRODS account's credentials go into `IRODS_AMQP_URI` in +`/etc/irods/server_config.json`; see +[iRODS provider](../03-data-store/irods-provider.md). + +The DE's own vhost, exchange, and accounts are created later, in phase 6, by +the deployment repository's `rabbitmq_configure.yml` playbook: + +```bash +ansible-playbook -i /path/to/inventory rabbitmq_configure.yml +``` + +The existing Data Store playbooks (`playbooks/amqp.yml`, +`playbooks/amqp_exchange.yml` in the +[Data Store collection](https://github.com/cyverse/ds-collection)) automate +steps 1 through 7 and are worth reusing rather than doing this by hand. + +# Ports + +| Port | Purpose | Scope | +|------|---------|-------| +| 5672 | AMQP | Analysis nodes and in-cluster services | +| 15672 | Management UI and `rabbitmqadmin` download | Administrators only | + +Do not expose `15672` publicly. See +[network requirements](../../architecture/network-requirements.md). + +# Operations: reindex search + +A full reindex is requested by publishing an empty message with the routing key +`index.all` to the DE exchange, then restarting the indexer. + +## One-time: get `rabbitmqadmin` + +`rabbitmqadmin` is served by the management plugin, so fetch it from the broker +host itself: + +```bash +ssh +mkdir -p ~/adm && cd ~/adm +wget http://localhost:15672/cli/rabbitmqadmin +chmod +x rabbitmqadmin +``` + +## Trigger the reindex + +Replace ``, ``, and `` with the values for the +environment you are working in — the DE vhost and user come from the deployment +group variables, and the namespace is where the DE services run (`prod` in a +standard deployment). + +```bash +# check the broker is healthy +systemctl status rabbitmq-server.service -l + +# read the password into the environment instead of putting it in shell history +read -rs PASSWORD && export PASSWORD + +# confirm you are pointed at the right vhost +./rabbitmqadmin -V list exchanges -u -p "$PASSWORD" + +# request a full reindex +./rabbitmqadmin publish -V -u -p "$PASSWORD" \ + exchange=de routing_key=index.all payload="" + +# restart the indexer so it picks the request up +kubectl rollout restart deployment infosquito2 -n +``` + +If `infosquito2` is not deployed at all, deploy it rather than restarting it — +see [cluster resources](../04-kubernetes/resources.md). + +A reindex reads the entire catalog. On a large zone it takes hours and adds load +to both PostgreSQL and the search cluster, so run it deliberately. + +# Related + +* [OpenSearch](../05-core-services/opensearch.md) +* [Elasticsearch (legacy)](../05-core-services/elasticsearch.md) +* [iRODS provider](../03-data-store/irods-provider.md) +* [Deploying from scratch](../from-scratch.md#13-rabbitmq) diff --git a/docs/deployment/02-databases/de.md b/docs/deployment/02-databases/de.md new file mode 100644 index 0000000..6b7a7dd --- /dev/null +++ b/docs/deployment/02-databases/de.md @@ -0,0 +1,65 @@ +--- +type: Database +title: "DE database" +description: "Creating and migrating the Discovery Environment database." +tags: [deployment, databases, discovery-environment] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +The core Discovery Environment database. It holds apps, tools, analyses, and — since the merge of the former `permissions` database — permissions as well. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser +create user de with password ''; +create database de with owner de; +``` + +# Extensions + +```sql +\c de +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +``` + +# Populate and migrate + +Schema and data come from [de-database](https://github.com/cyverse-de/de-database), applied with the shared procedure in +[database migrations](./migrations.md). In a normal deployment the +`setup-databases` and `update-databases` Ansible tags do this for you. + +# Legacy version table + +Deployments created before the move to golang-migrate carry a `version` table that +newer migrations do not create: + +```sql +SET search_path = public, pg_catalog; + +CREATE TABLE version ( + version character varying(20) NOT NULL, + applied timestamp DEFAULT now() +); +``` + +It was seeded from `old-databases/de-db/src/main/data/999_version.sql` in the +de-database repository. A new deployment does not need it; it is documented here so +that finding it in an existing database is not a surprise. + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/02-databases/grouper.md b/docs/deployment/02-databases/grouper.md new file mode 100644 index 0000000..09cec8c --- /dev/null +++ b/docs/deployment/02-databases/grouper.md @@ -0,0 +1,48 @@ +--- +type: Database +title: "Grouper database" +description: "Creating the database behind the Internet2 Grouper group-management service." +tags: [deployment, databases, grouper, groups] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +Holds Grouper's groups, folders, and memberships — the data the DE authorizes against. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser +create user grouper with password ''; +create database grouper with owner grouper; +``` + +# Extensions + +```sql +\c grouper +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +``` + +# Migrations + +Grouper manages its own schema through its installer (`gsh` / the Grouper +installer image) rather than through golang-migrate. Run the schema step from the +Grouper distribution before starting `grouper-loader`; see +[Grouper deployment](../05-core-services/grouper.md). + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/02-databases/icat.md b/docs/deployment/02-databases/icat.md new file mode 100644 index 0000000..7e10027 --- /dev/null +++ b/docs/deployment/02-databases/icat.md @@ -0,0 +1,61 @@ +--- +type: Database +title: "iCAT database" +description: "The iRODS catalog database: what creates it, what may read it, and what must never write to it." +tags: [deployment, databases, irods, icat] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: irods-install + resource: https://docs.irods.org/4.3.3/getting_started/installation/ + title: iRODS 4.3.3 installation guide + author: team:irods-consortium +--- + +# What it is + +The iCAT is iRODS's catalog: every collection, data object, replica, AVU, ticket, +and permission in the zone. It is the one CyVerse database that is not managed by +CyVerse migrations — its schema belongs to iRODS. + +# Who creates it + +The iRODS installer, during +[iRODS provider setup](../03-data-store/irods-provider.md). You do not create it +by hand. What you do beforehand is prepare PostgreSQL for iRODS as the iRODS +project describes,[^irods-install] which creates the database, the `irods` role, +and its ODBC connectivity — see +[PostgreSQL](../01-foundation/postgresql.md#prepare-for-irods). + +The database name is chosen at install time and recorded in +`/etc/irods/server_config.json`. + +# Who may read it + +The DE reads the catalog directly for some listings, through a PostgreSQL role +with `SELECT` on all iCAT tables and nothing more. That role is separate from the +`de-irods` iRODS account. + +!!! danger "Nothing but iRODS writes to the iCAT" + + Writing to the catalog outside iRODS bypasses policy: no rules fire, no AVUs + are maintained, no messages are published, and the vault and catalog can + disagree. Every write goes through the iRODS protocol, including + administrative ones. + +# Backup + +The iCAT and the vault have to be backed up as a pair. A catalog restored to a +different point in time than the vault leaves data objects registered that do not +exist, and files on disk that nothing can reach. + +Include both in the operational readiness checks in +[verification](../07-post-install/verification.md). + +# Related + +* [iRODS catalog provider](../03-data-store/irods-provider.md) +* [PostgreSQL](../01-foundation/postgresql.md) +* [Data Store](../../platform/data-store.md) + +[^irods-install]: iRODS 4.3.3 installation guide diff --git a/docs/deployment/02-databases/index.md b/docs/deployment/02-databases/index.md new file mode 100644 index 0000000..2632d38 --- /dev/null +++ b/docs/deployment/02-databases/index.md @@ -0,0 +1,31 @@ +# Phase 2: databases + +One database per service, all in the PostgreSQL instance from +[phase 1](../01-foundation/postgresql.md). In a normal deployment the +`setup-databases` and `update-databases` Ansible tags create and migrate the DE's +databases for you; these documents describe what those tags produce and how to do +it by hand. + +* [Database migrations](migrations.md) - the shared golang-migrate procedure and shared extensions + +# Data Store + +* [iCAT database](icat.md) - the iRODS catalog, created by the iRODS installer + +# Discovery Environment + +* [DE database](de.md) - apps, tools, analyses, and permissions +* [Metadata database](metadata.md) - metadata templates and AVUs +* [Notifications database](notifications.md) - user notifications and system messages +* [QMS database](qms.md) - quotas, plans, and recorded usage + +# Platform services + +* [Keycloak database](keycloak.md) - realms, clients, and sessions +* [Grouper database](grouper.md) - groups, folders, and memberships +* [Unleash database](unleash.md) - feature flags +* [Portal database](portal.md) - accounts, requests, workshops, and form data + +# Next + +* [Phase 3: Data Store](../03-data-store/) diff --git a/docs/deployment/02-databases/keycloak.md b/docs/deployment/02-databases/keycloak.md new file mode 100644 index 0000000..5456b69 --- /dev/null +++ b/docs/deployment/02-databases/keycloak.md @@ -0,0 +1,40 @@ +--- +type: Database +title: "Keycloak database" +description: "Creating the database Keycloak uses for realms, clients, and sessions." +tags: [deployment, databases, keycloak] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +Holds Keycloak's realms, clients, roles, and sessions. Losing it means reconfiguring every realm and client by hand, so it is one of the databases most worth backing up. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser +create user keycloak with password ''; +create database keycloak with owner keycloak; +``` + +# Migrations + +Keycloak applies its own schema migrations at startup, so there is nothing to run +here. The first start after a Keycloak version upgrade takes noticeably longer for +that reason. + +No extensions are required. + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/02-databases/metadata.md b/docs/deployment/02-databases/metadata.md new file mode 100644 index 0000000..0deaaa0 --- /dev/null +++ b/docs/deployment/02-databases/metadata.md @@ -0,0 +1,46 @@ +--- +type: Database +title: "Metadata database" +description: "Creating the database behind the CyVerse metadata service." +tags: [deployment, databases, metadata] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +Holds user-defined metadata templates and the AVU metadata the metadata service manages on behalf of the DE. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser; owned by the de role, not a role of its own +create database metadata with owner de; +``` + +# Extensions + +```sql +\c metadata +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +``` + +# Populate and migrate + +Schema and data come from [de-database](https://github.com/cyverse-de/de-database), applied with the shared procedure in +[database migrations](./migrations.md). In a normal deployment the +`setup-databases` and `update-databases` Ansible tags do this for you. + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/02-databases/migrations.md b/docs/deployment/02-databases/migrations.md new file mode 100644 index 0000000..536bb8a --- /dev/null +++ b/docs/deployment/02-databases/migrations.md @@ -0,0 +1,105 @@ +--- +type: Deployment Procedure +title: "Database migrations" +description: "The shared golang-migrate procedure used to create and update every CyVerse service schema." +tags: [deployment, databases, migrations, postgresql] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: golang-migrate + resource: https://github.com/golang-migrate/migrate + title: golang-migrate + author: team:golang-migrate +--- + +# Two ways to migrate + +Every CyVerse schema is versioned as a directory of `migrations` applied with +[golang-migrate](https://github.com/golang-migrate/migrate).[^golang-migrate] You +will use one of two paths: + +| Path | When | +|------|------| +| Ansible tags | Normal deployments and upgrades — the playbooks migrate every DE database in one pass | +| `migrate` by hand | Bootstrapping a single database, or debugging a failed migration | + +## With Ansible + +```bash +# create the databases +ansible-playbook -i /path/to/inventory --tags setup-databases kubernetes.yml + +# apply outstanding migrations +ansible-playbook -i /path/to/inventory --tags update-databases kubernetes.yml +``` + +Run `update-databases` after every service upgrade. It is idempotent: with nothing +outstanding it reports no changes. + +## By hand + +Install `migrate` once. On Debian and Ubuntu: + +```bash +curl -s https://packagecloud.io/install/repositories/golang-migrate/migrate/script.deb.sh | sudo bash +apt-get update +apt-get install -y migrate +migrate -help +``` + +Then, from a checkout of the repository that owns the schema: + +```bash +migrate -database "postgres://:@/?sslmode=disable" \ + -path migrations up +``` + +!!! warning "`sslmode=disable` and credentials on the command line" + + Both appear in existing CyVerse notes and both are compromises. Prefer + `sslmode=require` where the server supports it, and put the URL in an + environment variable rather than in the command, so the password stays out of + shell history and process listings. + +# Where the migrations live + +| Database | Repository | +|----------|------------| +| `de`, `notifications`, `metadata` | [de-database](https://github.com/cyverse-de/de-database) | +| `qms` | [QMS](https://github.com/cyverse/QMS) (`prod` branch) | +| `portal` | [portal2](https://gitlab.com/cyverse/portal2) | +| `unleash` | Applied by Unleash itself at startup | +| `keycloak` | Applied by Keycloak itself at startup | +| `grouper` | Applied by the Grouper installer | + +# Shared extensions + +Most CyVerse databases need the same extensions, created as a superuser in the +target database before migrating: + +```sql +\c +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +``` + +The QMS database also needs `insert_username`. A migration that fails on a missing +function almost always means an extension was not created first. + +# Troubleshooting + +**Migrations reported as applied but the schema is wrong.** Check +`standard_conforming_strings`; when it is on, some DE migrations are skipped. See +[troubleshooting](../07-post-install/troubleshooting.md). + +**A migration failed halfway.** `migrate` records a dirty version. Inspect +`schema_migrations`, fix the cause, then force the version back to the last good +one before re-running — do not delete the table. + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Service databases](./index.md) + +[^golang-migrate]: golang-migrate diff --git a/docs/deployment/02-databases/notifications.md b/docs/deployment/02-databases/notifications.md new file mode 100644 index 0000000..29c6018 --- /dev/null +++ b/docs/deployment/02-databases/notifications.md @@ -0,0 +1,46 @@ +--- +type: Database +title: "Notifications database" +description: "Creating the database behind the user notification service." +tags: [deployment, databases, notifications] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +Holds user notifications and system messages, including the read and seen state the DE displays. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser; owned by the de role +create database notifications with owner de; +``` + +# Extensions + +```sql +\c notifications +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +``` + +# Populate and migrate + +Schema and data come from [de-database](https://github.com/cyverse-de/de-database), applied with the shared procedure in +[database migrations](./migrations.md). In a normal deployment the +`setup-databases` and `update-databases` Ansible tags do this for you. + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/02-databases/portal.md b/docs/deployment/02-databases/portal.md new file mode 100644 index 0000000..86fae5b --- /dev/null +++ b/docs/deployment/02-databases/portal.md @@ -0,0 +1,147 @@ +--- +type: Database +title: "Portal database" +description: "Creating, restoring, and seeding the User Portal database." +tags: [deployment, databases, user-portal] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: portal2 + resource: https://gitlab.com/cyverse/portal2 + title: CyVerse User Portal (portal2) + author: team:cyverse + - id: grid + resource: https://www.grid.ac/ + title: GRID institution identifiers +--- + +Backs the [User Portal](../06-applications/user-portal.md): accounts, access +requests, workshops, form submissions, and the reference tables the signup forms +are built from. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +create user portal_db_reader with password ''; +create database portal with owner portal_db_reader; +``` + +The portal's own setup notes also grant the role membership in `postgres`: + +```sql +GRANT postgres TO portal_db_reader; +``` + +!!! warning "That grant is broader than it looks" + + Membership in `postgres` gives the portal role superuser-equivalent reach over + every database in the instance, including the iCAT. Grant it only if a portal + migration actually needs it, revoke it afterwards, and prefer granting the + specific privileges the portal needs. See + [portal2](https://gitlab.com/cyverse/portal2)[^portal2] for what the + application itself requires. + +# Restore the base schema + +The portal ships a SQL dump rather than an incremental migration history for the +initial load: + +```bash +psql -U portal_db_reader -d portal -f portal.sql +``` + +`portal.sql` comes from the [portal2](https://gitlab.com/cyverse/portal2) +repository. + +## Session table + +The portal stores sessions in the database. Confirm the table exists after the +restore: + +```sql +CREATE TABLE public.session ( + sid character varying NOT NULL, + sess json NOT NULL, + expire timestamp(6) without time zone NOT NULL +); +ALTER TABLE public.session OWNER TO portal; +CREATE INDEX "IDX_session_expire" ON public.session USING btree (expire); +``` + +!!! note "Two roles appear in the portal's own SQL" + + The dump creates the database owned by `portal_db_reader` but assigns the + session table to `portal`. Reconcile these against the roles your deployment + actually uses before the portal starts, or session writes fail with a + permission error at first sign-in. + +# Seed reference data + +## Institutions + +Institution autocomplete is seeded from the GRID dataset.[^grid] Download a +release, unzip it, and import with the script from the portal repository: + +```bash +./import_grid_institutions.py \ + --host --user portal_db_reader --database portal grid.csv +``` + +The script lives at `src/scripts/import_grid_institutions.py` in +[portal2](https://gitlab.com/cyverse/portal2). + +## Form reference tables + +The signup and profile forms read from a set of lookup tables. Load each one: + +```bash +for table in country region gender occupation ethnicity \ + fundingagency awarechannel researcharea; do + psql -U portal_db_reader -d portal -f "./account_${table}.sql" +done +``` + +The SQL files are published at +[portal2-db](https://github.com/cyverse-austria/portal2-db). + +# Administrative queries + +Promote an existing account to portal administrator: + +```sql +UPDATE account_user SET is_superuser = true WHERE username = ''; +UPDATE account_user SET is_staff = true WHERE username = ''; +``` + +Check or force email verification: + +```sql +SELECT has_verified_email FROM account_user WHERE username = ''; +UPDATE account_user SET has_verified_email = true WHERE username = ''; +``` + +Prefer the [admin panel](../../operations/user-portal.md) for routine work. These +queries exist for bootstrapping the first administrator and for repairing accounts +the UI cannot reach — the normal path is +[bootstrap](../07-post-install/bootstrap.md). + +# Migrations + +Ongoing schema changes come from the portal application itself. See +[database migrations](./migrations.md) for the shared procedure. + +# Related + +* [User Portal deployment](../06-applications/user-portal.md) +* [User Portal administration](../../operations/user-portal.md) +* [PostgreSQL](../01-foundation/postgresql.md) + +[^portal2]: CyVerse User Portal (portal2) +[^grid]: GRID institution identifiers diff --git a/docs/deployment/02-databases/qms.md b/docs/deployment/02-databases/qms.md new file mode 100644 index 0000000..0a2d4a7 --- /dev/null +++ b/docs/deployment/02-databases/qms.md @@ -0,0 +1,53 @@ +--- +type: Database +title: "QMS database" +description: "Creating and migrating the quota management service database." +tags: [deployment, databases, qms, quotas] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +Backs the quota management service: subscription plans, resource quotas, and recorded usage. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser; owned by the de role +create database qms with owner de; +``` + +# Extensions + +```sql +\c qms +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +create extension "insert_username"; +``` + +# Populate and migrate + +Schema and data come from [QMS](https://github.com/cyverse/QMS) (`prod` branch), applied with the shared procedure in +[database migrations](./migrations.md). In a normal deployment the +`setup-databases` and `update-databases` Ansible tags do this for you. + +!!! warning "Connect to the right database first" + + Older notes run `\c de` before creating these extensions, which installs them + into the DE database and leaves `qms` without them. The subsequent migration + then fails on a missing function. Connect to `qms`, as above. + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/02-databases/unleash.md b/docs/deployment/02-databases/unleash.md new file mode 100644 index 0000000..0aa5efd --- /dev/null +++ b/docs/deployment/02-databases/unleash.md @@ -0,0 +1,46 @@ +--- +type: Database +title: "Unleash database" +description: "Creating the database behind the Unleash feature-flag service." +tags: [deployment, databases, unleash, feature-flags] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +Holds the DE's feature toggles, including the maintenance flag. + +# Create + +Connect as a superuser on the host running +[PostgreSQL](../01-foundation/postgresql.md): + +```bash +psql -h -U postgres +``` + +```sql +-- as a superuser +create user unleash with password ''; +create database unleash with owner unleash; +``` + +# Extensions + +```sql +\c unleash +create extension "uuid-ossp"; +create extension "moddatetime"; +create extension "btree_gist"; +``` + +# Migrations + +Unleash applies its own schema migrations at startup; there is nothing to run here. +See [Unleash deployment](../05-core-services/unleash.md). + + +# Related + +* [PostgreSQL](../01-foundation/postgresql.md) +* [Database migrations](./migrations.md) +* [Service databases](./index.md) diff --git a/docs/deployment/03-data-store/de-integration.md b/docs/deployment/03-data-store/de-integration.md new file mode 100644 index 0000000..5b9c6f8 --- /dev/null +++ b/docs/deployment/03-data-store/de-integration.md @@ -0,0 +1,98 @@ +--- +type: Deployment Procedure +title: "iRODS integration for the DE" +description: "The specific queries, service account, and messaging wiring the Discovery Environment needs from an iRODS zone." +tags: [deployment, data-store, irods, discovery-environment] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: ds-collection + resource: https://github.com/cyverse/ds-collection + title: CyVerse Data Store collection (playbooks and iRODS policy) + author: team:cyverse-devops +--- + +# Prerequisites + +A working [iRODS catalog provider](./irods-provider.md) with the zone +initialized. + +# Specific queries + +The DE relies on iRODS *specific queries* — named SQL registered in the catalog +— for listings that the general query interface cannot express efficiently, such +as counting collections beneath a path. + +The queries live in `playbooks/files/irods/specific-queries` in the +[Data Store collection](https://github.com/cyverse/ds-collection).[^ds-collection] +Each file name is the query alias and the file contents are the query, so +installation is mechanical: + +```bash +iadmin asq "$(cat IPCCountCollectionsUnderPath.sql)" IPCCountCollectionsUnderPath +``` + +Repeat for every file in that directory. Missing a query does not break iRODS — +it breaks a specific DE listing later, with an error that does not obviously +point back here, so install them all in one pass and verify with `iadmin lsq`. + +# DE service account + +The DE authenticates to iRODS as a dedicated `rodsadmin` account rather than as +`rods`: + +```bash +iadmin mkuser de-irods rodsadmin +iadmin moduser de-irods password '' +iadmin atg rodsadmin de-irods +``` + +The same password goes into the DE's group variables (the `IRODS` section of the +deployment configuration; see +[cluster resources](../04-kubernetes/resources.md)) and into the +`porklock-config` secret used by analysis data transfers (see +[VICE](../06-applications/vice.md)). + +!!! warning "One account, several consumers" + + `de-irods` credentials appear in the DE configuration, in the VICE + `porklock-config` secret, and in the iRODS CSI driver values. Rotating the + password means updating all three and restarting the services that read + them. + +# Catalog read access + +The DE also reads the catalog database directly for some queries, using the +PostgreSQL role with `SELECT` on all iCAT tables created in +[PostgreSQL](../01-foundation/postgresql.md#prepare-for-irods). That role is +separate from `de-irods` and has no write access to the catalog — anything that +writes goes through the iRODS protocol so that policy applies. + +# Event flow + +```mermaid +graph LR + IRODS[iRODS provider] -->|publishes to exchange irods| MQ[RabbitMQ /data-store] + MQ --> IDX[infosquito2] + IDX --> SEARCH[OpenSearch] + MQ --> DE[DE services] +``` + +Data events published by iRODS policy drive search indexing and DE +notifications. If search results go stale, the usual causes are in that chain: +the AMQP URI in `server_config.json`, the exchange, or a stopped indexer. See +[RabbitMQ operations](../01-foundation/rabbitmq.md#operations-reindex-search) +for triggering a full reindex. + +# Related + +* [iRODS CSI driver](../05-core-services/irods-csi-driver.md) +* [Discovery Environment deployment](../06-applications/discovery-environment.md) +* [Deploying from scratch](../from-scratch.md#35-prepare-irods-for-the-de) + +[^ds-collection]: CyVerse Data Store collection (playbooks and iRODS policy) diff --git a/docs/deployment/03-data-store/index.md b/docs/deployment/03-data-store/index.md new file mode 100644 index 0000000..42c5022 --- /dev/null +++ b/docs/deployment/03-data-store/index.md @@ -0,0 +1,18 @@ +# Phase 3: Data Store + +The iRODS zone that holds user data. It needs PostgreSQL and RabbitMQ from +[phase 1](../01-foundation/), and everything in later phases reads or writes +through it. + +* [iRODS catalog provider](irods-provider.md) - install iRODS 4.3.3, apply CyVerse policy, initialize the zone +* [iRODS integration for the DE](de-integration.md) - specific queries, the DE service account, and the event flow + +# Related + +* [iCAT database](../02-databases/icat.md) +* [iRODS CSI driver](../05-core-services/irods-csi-driver.md) - mounting collections into pods +* [Data Store](../../platform/data-store.md) - the access services on top of the zone + +# Next + +* [Phase 4: Kubernetes](../04-kubernetes/) diff --git a/docs/deployment/03-data-store/irods-provider.md b/docs/deployment/03-data-store/irods-provider.md new file mode 100644 index 0000000..87b1fac --- /dev/null +++ b/docs/deployment/03-data-store/irods-provider.md @@ -0,0 +1,209 @@ +--- +type: Deployment Procedure +title: "iRODS catalog provider" +description: "Installing iRODS 4.3.3 as a catalog provider, applying CyVerse policy, and initializing the zone." +tags: [deployment, data-store, irods, storage] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: irods-install + resource: https://docs.irods.org/4.3.3/getting_started/installation/ + title: iRODS 4.3.3 installation guide + author: team:irods-consortium + - id: irods-packages + resource: https://packages.irods.org/ + title: iRODS package repository setup + author: team:irods-consortium + - id: ds-collection + resource: https://github.com/cyverse/ds-collection + title: CyVerse Data Store collection (playbooks and iRODS policy) + author: team:cyverse-devops +--- + +# Prerequisites + +* [PostgreSQL](../01-foundation/postgresql.md) installed, tuned, and prepared + for iRODS. +* [RabbitMQ](../01-foundation/rabbitmq.md) running, with the `/data-store` + vhost, the `irods` topic exchange, and an iRODS account. +* The vault filesystem mounted, with enough capacity for the data the zone will + hold. + +The Data Store collection's `playbooks/irods_catalog_provider.yml` automates +most of what follows. The manual steps are documented because the playbooks +target older iRODS releases and need review against 4.3.3. + +# Configure logging first + +iRODS logs through syslog, so set this up **before** the first server start or +the initial run is lost. + +`/etc/rsyslog.d/00-irods.conf`: + +``` +$FileCreateMode 0644 +$DirCreateMode 0755 +$Umask 0000 +$template irods_format,"%msg%\n" +:programname,startswith,"irodsServer" /var/log/irods/irods.log;irods_format +& stop +:programname,startswith,"irodsDelayServer" /var/log/irods/irods.log;irods_format +& stop +:programname,startswith,"irodsAgent" /var/log/irods/irods.log;irods_format +& stop +``` + +!!! note "Path correction" + + Earlier deployment notes give this path as `/etc/rsyslog/00-irods.conf`. + rsyslog includes `/etc/rsyslog.d/*.conf`, so a file placed in + `/etc/rsyslog/` is silently ignored and iRODS logging appears not to work. + +`/etc/logrotate.d/irods`: + +``` +/var/log/irods/irods.log { + weekly + rotate 26 + copytruncate + delaycompress + compress + dateext + notifempty + missingok + su root root +} +``` + +Twenty-six weekly rotations keeps six months of history. `copytruncate` is used +because the iRODS server holds the log open. + +# Install + +1. Set TCP keepalive to 120 seconds with `sysctl`, so long-lived agent + connections are not dropped by intermediate firewalls. +2. Install `python-is-python3` and `python3-pika` — the CyVerse policy scripts + are Python 3 and publish to AMQP. +3. Add the iRODS apt repository.[^irods-packages] +4. **Pin `irods-*` to `4.3.3`.** Without a pin, an unattended upgrade can move + the catalog provider to a release the policy has not been tested against. +5. Run the iRODS setup script.[^irods-install] + +## Setup answers + +| Prompt | Value | +|--------|-------| +| iRODS user / group | `irods` / `irods` | +| Server role | `provider` | +| ODBC driver for PostgreSQL | `PostgreSQL Unicode` | +| Catalog host / port | `localhost` / `5432` | +| Catalog database name | your iCAT database name | +| Catalog database user | `irods` | +| Password salt | a generated alphanumeric string — see the warning below | +| Local storage on this server | `yes` | +| Default resource name | anything except `demoResc` | +| Vault directory | root of the filesystem that will hold the data | +| Zone name | ``, something meaningful to the project or institution | +| Server port | `1247` | +| Port range | `20000` to `20199` | +| Control plane port | `1248` | +| Schema validation base URI | `file:///var/lib/irods/configuration_schemas` | +| Administrator username | `rods` | +| Zone key | generated alphanumeric string, fewer than 40 characters | +| Negotiation key | generated alphanumeric string | + +!!! danger "Never accept an empty password salt" + + An empty salt makes the passwords stored in the catalog recoverable. + Generate one per install (`openssl rand -hex 16`) and keep it with the rest + of the deployment secrets in your private inventory. + +!!! warning "The zone name is effectively permanent" + + Zone names appear in every published data URL. The production US zone is + still named `iplant` after a project that was renamed years ago, precisely + because renaming would break every published link. Choose a name you can + live with. + +# Install CyVerse policy + +From the [Data Store collection](https://github.com/cyverse/ds-collection) +branch that matches your site,[^ds-collection] as the `irods` service account: + +1. Copy `playbooks/files/irods/var/lib/irods/msiExecCmd_bin/*` into + `/var/lib/irods/msiExecCmd_bin/` and make them executable. +2. Render `playbooks/templates/irods/etc/irods/cyverse-env.re.j2` to + `/etc/irods/cyverse-env.re`: + * `cyverse_RE_HOST` — the FQDN of the catalog provider host. + * `cyverse_ZONE` — ``. +3. Copy `playbooks/files/etc/irods/*` into `/etc/irods/`. + +Then set in `/etc/irods/server_config.json`: + +| Key | Value | +|-----|-------| +| `advanced_settings.number_of_concurrent_delay_rule_executors` | `12` | +| `environment_variables.IRODS_AMQP_URI` | `amqp://:@localhost:5672/%2Fdata-store` | +| `plugin_configuration.rule_engines[0].re_rulebase_set` | `["cve", "cyverse_core", "core"]` | + +The rule base order is significant: `cve` overrides `cyverse_core`, which +overrides `core`. Reordering them silently changes policy. + +`%2F` in the AMQP URI is the URL-encoded leading slash of the `/data-store` +vhost — it is not a typo. + +Enable the service so it starts at boot, then start it. + +# Initialize the zone + +As the `irods` service account, substituting `` throughout. + +## Administrative group + +1. Create the `rodsadmin` group and add `rods` to it. +2. Delete the collections group creation leaves behind: + `//home/rodsadmin` and `//trash/home/rodsadmin`. +3. Delete `//trash/home/public`. + +## UUIDs on predefined collections + +CyVerse policy expects every collection to carry a time-based (version 1) UUID. +The collections created by the installer predate the policy, so check and add +them for: + +`/`, `//home`, `//home/public`, +`//home/rods`, `//trash`, `//trash/home`, +`//trash/home/rods`. + +## Permissions + +| Group or user | Permission | Collections | +|---------------|------------|-------------| +| `rodsadmin` | `write` | `/`, `//home`, `//trash`, `//trash/home` | +| `rodsadmin` | `own` | `//home/rods`, `//trash/home/rods` | +| `anonymous` | `read` | `/`, `//home` | + +## Anonymous access + +Create the `anonymous` `rodsuser` **without a password**, then grant it the +`read` permissions above. This account is what makes public data public — WebDAV +and the Data Commons read as `anonymous`. A password on this account breaks +anonymous access rather than securing it. + +# Next + +* [DE integration](./de-integration.md) — the specific queries and service + account the Discovery Environment needs. +* [iRODS CSI driver](../05-core-services/irods-csi-driver.md) — mounting Data + Store paths into VICE pods. +* [Data Store](../../platform/data-store.md) — the access services layered on + top of the zone. + +[^irods-install]: iRODS 4.3.3 installation guide +[^irods-packages]: iRODS package repository setup +[^ds-collection]: CyVerse Data Store collection (playbooks and iRODS policy) diff --git a/docs/deployment/04-kubernetes/argo.md b/docs/deployment/04-kubernetes/argo.md new file mode 100644 index 0000000..8061826 --- /dev/null +++ b/docs/deployment/04-kubernetes/argo.md @@ -0,0 +1,53 @@ +--- +type: Deployment Procedure +title: "Argo Workflows" +description: "Installing Argo Workflows and the workflow resources CyVerse analyses depend on." +tags: [deployment, kubernetes, argo, workflows] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: argo-docs + resource: https://argo-workflows.readthedocs.io/ + title: Argo Workflows documentation + author: team:argoproj +--- + +# Role in the deployment + +Argo Workflows runs the containerized, non-interactive side of DE analyses inside +Kubernetes — the batch counterpart to VICE. Deployments that predate it dispatch +those jobs to HTCondor instead; the pilot leaves the `01_condor` inventory group +empty and uses Argo. + +# Install + +Two steps, in order: + +```bash +ansible-playbook -i /path/to/inventory --tags argo kubernetes.yml +ansible-playbook -i /path/to/inventory argo_resources.yml +``` + +The first installs the controller and CRDs. The second creates the workflow +resources — service accounts, roles, and templates — that DE analyses submit +against. The resources playbook fails if the CRDs are not in place yet, so let +the first finish before starting the second. + +# Verify + +```bash +kubectl -n argo get pods +kubectl get crd | grep argoproj +kubectl -n argo get workflowtemplates +``` + +# Related + +* [Cluster](./cluster.md) +* [Discovery Environment deployment](../06-applications/discovery-environment.md) +* [VICE](../06-applications/vice.md) diff --git a/docs/deployment/04-kubernetes/cert-manager.md b/docs/deployment/04-kubernetes/cert-manager.md new file mode 100644 index 0000000..3b8439b --- /dev/null +++ b/docs/deployment/04-kubernetes/cert-manager.md @@ -0,0 +1,79 @@ +--- +type: Deployment Procedure +title: "cert-manager" +description: "Installing cert-manager and the cluster issuers that mint TLS certificates for every CyVerse route." +tags: [deployment, kubernetes, tls, cert-manager] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: cert-manager-docs + resource: https://cert-manager.io/docs/ + title: cert-manager documentation + author: team:cert-manager +--- + +# Why it comes first + +cert-manager is installed immediately after the cluster and before any ingress, +because ingress routes reference certificates by secret name. Install it the +other way round and every route comes up without TLS and has to be reconciled +later. + +# Install + +```bash +ansible-playbook -i /path/to/inventory --tags cert-manager kubernetes.yml +ansible-playbook -i /path/to/inventory --tags cert-issuers kubernetes.yml +``` + +The two tags are deliberately separate: the first installs the controller and +its CRDs, the second creates the `ClusterIssuer` objects. The issuers cannot be +created until the CRDs exist, so run them in this order and let the first +finish. + +# Issuers + +A CyVerse deployment normally defines two issuers: + +| Issuer | Use | +|--------|-----| +| Let's Encrypt staging | Validating the issuance path without burning rate limits | +| Let's Encrypt production | The certificates users see | + +VICE gives interactive apps hostnames under `*.vice.`, which means +a **wildcard** certificate, which means the **DNS-01** challenge — HTTP-01 +cannot satisfy a wildcard. That is why +[network requirements](../../architecture/network-requirements.md) calls for +outbound HTTPS to your DNS provider's API: the issuer needs credentials for a +DNS provider it can write TXT records through. + +Those credentials are a secret. They go in your private inventory and reach the +cluster as a Kubernetes secret referenced by the issuer, never in a manifest +committed here. + +# Verify + +```bash +kubectl get clusterissuers +kubectl get certificates -A +kubectl describe certificate -n +``` + +A certificate stuck in `False` / `Issuing` is almost always the challenge, not +the certificate: check the `Order` and `Challenge` objects it owns, and confirm +the DNS credentials and outbound access. + +Validate against the staging issuer first. Production Let's Encrypt rate limits +are per registered domain per week, and a misconfigured wildcard can exhaust +them quickly. + +# Related + +* [Ingress](./ingress.md) +* [HAProxy](../01-foundation/haproxy.md) +* [Deploying from scratch](../from-scratch.md#44-cluster-add-ons-in-order) diff --git a/docs/deployment/04-kubernetes/cluster.md b/docs/deployment/04-kubernetes/cluster.md new file mode 100644 index 0000000..82f48e2 --- /dev/null +++ b/docs/deployment/04-kubernetes/cluster.md @@ -0,0 +1,187 @@ +--- +type: Deployment Procedure +title: "Cluster" +description: "Standing up the Kubernetes control plane and workers that run CyVerse services, with k0sctl." +tags: [deployment, kubernetes, k0s, cluster] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: k0sctl + resource: https://docs.k0sproject.io/stable/k0sctl-install/ + title: k0sctl installation and cluster configuration + author: team:k0sproject +--- + +# Prerequisites + +* [Prerequisites](../planning/prerequisites.md) and + [Ansible](../planning/ansible.md) complete. +* [Foundation services](../01-foundation/index.md) installed, and + [iRODS](../03-data-store/irods-provider.md) running. +* Passwordless `ssh` from your workstation to every node, and `sudo` on each. +* The Kubernetes ports in + [network requirements](../../architecture/network-requirements.md) open + between nodes. + +# Inventory + +The cluster's shape comes from the Ansible inventory. For a two-node pilot: + +| Inventory group | Hosts | +|-----------------|-------| +| `k8s_api_proxy` | `core-1` | +| `k8s_controllers` | `core-1` | +| `k8s_de_workers` | `core-1` | +| `k8s_vice_workers` | `analysis-1` | + +A node can appear in several groups; `core-1` is control plane and DE worker at +once. VICE workers stay separate so interactive analyses cannot starve the +service set. + +# Prepare the nodes + +```bash +ansible-playbook -i /path/to/inventory --tags prep-nodes kubernetes.yml +``` + +This installs the packages, kernel modules, and sysctl settings the container +runtime needs, and applies host firewall rules. + +# Create the cluster with k0sctl + +## k0sctl.yaml + +`k0sctl.yaml` lists the hosts and their roles. Keep it in your private +inventory repository — it names hosts and paths and identifies your SSH user. + +```yaml +apiVersion: k0sctl.k0sproject.io/v1beta1 +kind: Cluster +metadata: + name: -cluster +spec: + hosts: + - role: controller+worker + noTaints: true + ssh: + address: core-1. + user: + keyPath: /path/to/private-key + - role: worker + ssh: + address: analysis-1. + user: + keyPath: /path/to/private-key + k0s: + version: + config: + spec: + network: + provider: calico + telemetry: + enabled: false +``` + +Treat this as a skeleton: pin `` to the release you tested, and add +the node labels, taints, and API server SANs your site needs. The authoritative +reference for the fields is the k0sctl documentation.[^k0sctl] + +!!! note "Where the example belongs" + + Earlier notes pointed at an example `k0sctl.yaml` shared in a chat channel. + Keep the example in the deployment repository beside the playbooks instead — + a cluster definition that only exists in chat history is a cluster nobody + can rebuild. + +## Apply + +```bash +export K0S_SSH_USER= +export K0S_SSH_KEY_PATH=/path/to/private-key +export KUBECONFIG="$HOME/.kube/config" +mkdir -p "$(dirname "$KUBECONFIG")" +k0sctl apply --config /path/to/k0sctl.yaml +``` + +!!! warning "`dirname`, not `basename`" + + The directory to create is the *parent* of the kubeconfig path. Earlier + notes used `basename`, which creates a directory named `config` and leaves + `k0sctl` writing to a path that does not exist. + +## Untaint a combined control-plane node + +When the control-plane node is also a DE worker, k0s may have tainted it. On the +control node: + +```bash +k0s kubectl taint node core-1 node-role.kubernetes.io/control-plane:NoSchedule- +``` + +If the taint was never applied, the command says so. That message is the +expected outcome, not a problem to investigate. + +## Verify + +```bash +kubectl get nodes -o wide +kubectl get pods -A +``` + +Every node `Ready`, and no pod outside `Running` or `Completed`, before moving +on. + +# Legacy: Ansible-provisioned clusters + +Deployments predating k0sctl built the cluster with the playbooks in +[cyverse-de/deployments](https://github.com/cyverse-de/deployments/tree/main/ansible/kubernetes) +against CentOS 7 hosts. The inventory looked like this: + +```ini +[k8s:children] +k8s-control-plane +k8s-worker + +[kube-apiserver-haproxy] +k8s-reverse-proxy. + +[k8s-control-plane] +k8s-c1. + +[k8s-storage:children] +k8s-worker + +[k8s-worker] +k8s-w1. +k8s-w2. +vice-w1. + +[vice-workers] +vice-w1. +``` + +with `firewalld-config.yml` and `provision-nodes.yml`, and VICE workers labelled +and tainted by hand: + +```bash +kubectl label nodes vice-w1. vice=true +kubectl taint nodes vice-w1. vice=only:NoSchedule +``` + +CentOS 7 is end of life, so this path is documented for reading existing +clusters rather than for building new ones. New deployments use k0sctl, and node +labelling is handled by the `prep-nodes` tag and node feature discovery. + +# Next + +* [Cluster resources](./resources.md) +* [cert-manager](./cert-manager.md) +* [Storage](./storage.md) +* [Ingress](./ingress.md) + +[^k0sctl]: k0sctl installation and cluster configuration diff --git a/docs/deployment/04-kubernetes/harbor.md b/docs/deployment/04-kubernetes/harbor.md new file mode 100644 index 0000000..4093966 --- /dev/null +++ b/docs/deployment/04-kubernetes/harbor.md @@ -0,0 +1,59 @@ +--- +type: Deployment Procedure +title: "Harbor" +description: "Deploying the Harbor container registry that holds CyVerse service and tool images." +tags: [deployment, kubernetes, harbor, registry] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: harbor-docs + resource: https://goharbor.io/docs/ + title: Harbor documentation + author: team:harbor +--- + +# Role in the deployment + +Harbor is the container registry for CyVerse service images and integrated tool +images. The public CyVerse instance is at +[harbor.cyverse.org](https://harbor.cyverse.org/); a self-contained deployment +runs its own so that image pulls do not depend on another site's registry. + +Install it after [storage](./storage.md) — Harbor is stateful, and its registry +and database volumes need a working storage class before the chart will come up. + +# Install + +```bash +ansible-playbook -i /path/to/inventory --tags harbor kubernetes.yml +``` + +# What consumes it + +| Consumer | How it authenticates | +|----------|---------------------| +| DE services | `harbor-registry-credentials` secret in the DE namespace | +| VICE apps | `vice-image-pull-secret` in the `vice-apps` namespace | +| Image cache | Pre-pulls frequently used VICE images onto workers | +| Tool integration | Users and administrators push tool images | + +Both pull secrets are created as part of +[cluster resources](./resources.md) and +[VICE deployment](../06-applications/vice.md). A missing pull secret shows up as +`ImagePullBackOff` on an image that exists and is readable by hand — check the +secret before the registry. + +The deployment configuration also holds a Harbor robot account (the `Harbor` +section of the group variables: URL, robot name, robot secret) used by automated +image operations. Treat the robot secret like any other deployment secret. + +# Related + +* [Cluster resources](./resources.md) +* [VICE](../06-applications/vice.md) +* [Storage](./storage.md) diff --git a/docs/deployment/04-kubernetes/index.md b/docs/deployment/04-kubernetes/index.md new file mode 100644 index 0000000..2dbc815 --- /dev/null +++ b/docs/deployment/04-kubernetes/index.md @@ -0,0 +1,22 @@ +# Phase 4: Kubernetes + +The cluster and the add-ons every service above it assumes. Install in this order — +cert-manager before ingress so routes can reference their certificates, storage +before anything stateful. + +* [Cluster](cluster.md) - control plane and workers with k0sctl +* [Cluster resources](resources.md) - generated configuration, secrets, and manifests +* [cert-manager](cert-manager.md) - TLS issuance, including the VICE wildcard +* [Ingress](ingress.md) - HAProxy, Traefik, and the legacy ingress-nginx path +* [Storage](storage.md) - persistent volumes with Longhorn or OpenEBS +* [Harbor](harbor.md) - the container registry +* [Argo Workflows](argo.md) - batch analysis execution + +# Related + +* [Kubernetes namespaces](../../architecture/namespaces.md) +* [Network requirements](../../architecture/network-requirements.md) + +# Next + +* [Phase 5: core services](../05-core-services/) diff --git a/docs/deployment/04-kubernetes/ingress.md b/docs/deployment/04-kubernetes/ingress.md new file mode 100644 index 0000000..3767028 --- /dev/null +++ b/docs/deployment/04-kubernetes/ingress.md @@ -0,0 +1,104 @@ +--- +type: Deployment Procedure +title: "Ingress" +description: "Routing external traffic into the cluster with HAProxy, Traefik, and the legacy ingress-nginx path." +tags: [deployment, kubernetes, ingress, networking, traefik] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: traefik-docs + resource: https://doc.traefik.io/traefik/ + title: Traefik documentation + author: team:traefik +--- + +# The path a request takes + +```mermaid +graph LR + U[Client] -->|443/tcp| HAP[HAProxy on core-1] + HAP -->|31344/tcp nodeport| TR[Traefik] + TR --> DE[DE services] + TR --> KC[Keycloak] + TR --> UP[User Portal] + NG[ingress-nginx] --> VICE[VICE app ingresses] + HAP --> NG +``` + +Three pieces, each with a distinct job: + +* **[HAProxy](../01-foundation/haproxy.md)** is the public entry point on the + host, listening on `80` and `443`. +* **Traefik** is the in-cluster ingress controller, reached through node ports + (`31343` HTTP, `31344` HTTPS by default). +* **ingress-nginx** still handles VICE app ingresses and is being retired. + +# Traefik + +```bash +ansible-playbook -i /path/to/inventory --tags traefik kubernetes.yml +``` + +Install it after [cert-manager](./cert-manager.md), so routes can reference the +certificates they need at creation time. + +The node ports are configurable. If you change them, change the HAProxy back end +to match, and reopen the firewall on the new ports — see +[network requirements](../../architecture/network-requirements.md). + +Verify: + +```bash +kubectl get svc -A | grep -i traefik +kubectl get ingressroute -A +``` + +# ingress-nginx (transitional) + +ingress-nginx gives VICE apps their per-analysis ingresses. Traefik is intended +to take this over, and the deployment tag remains only until that migration +completes. + +```bash +ansible-playbook -i /path/to/inventory --tags ingress-nginx kubernetes.yml +``` + +The kustomize manifests are in the +[cluster resources](./resources.md) repository. If you deploy into a namespace +other than `prod`, the default backend argument has to follow: + +```diff +- --default-backend-service=prod/vice-default-backend ++ --default-backend-service=/vice-default-backend +``` + +Applying the overlay creates the `ingress-nginx` namespace. + +!!! warning "Do not build new routes on ingress-nginx" + + New services should get Traefik routes. Anything added to ingress-nginx now + is work that has to be migrated later. + +# DNS and certificates + +| Hostname | Serves | +|----------|--------| +| `de.` | Discovery Environment | +| `keycloak.` | Keycloak | +| `user.` | User Portal | +| `vice.` | VICE landing and operator public base URL | +| `*.vice.` | Individual interactive analyses | + +The wildcard is what forces DNS-01 certificate challenges; see +[cert-manager](./cert-manager.md). + +# Related + +* [HAProxy](../01-foundation/haproxy.md) +* [VICE deployment](../06-applications/vice.md) +* [Namespaces](../../architecture/namespaces.md) diff --git a/docs/deployment/04-kubernetes/resources.md b/docs/deployment/04-kubernetes/resources.md new file mode 100644 index 0000000..8fce153 --- /dev/null +++ b/docs/deployment/04-kubernetes/resources.md @@ -0,0 +1,462 @@ +--- +type: Deployment Procedure +title: "Cluster resources" +description: "Generating and loading the configuration, secrets, and manifests every DE service reads." +tags: [deployment, kubernetes, configuration, secrets] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- +# What this repository holds + +The cluster resources repository carries everything the DE services read at +runtime that is not part of a service image: the generated ConfigMaps and Secrets, +the kustomize bases and overlays, service accounts, roles, network policies, and +add-on manifests. Several other documents in this phase point back here for +manifests. + +```bash +git clone +``` + +Throughout this document, `` is the environment name in `config_values/` +(`prod` in a standard deployment) and `` is the Kubernetes namespace +you are deploying into. They are frequently but not necessarily the same string. + +## Tools + +| Tool | Install | +|------|---------| +| [gomplate](https://docs.gomplate.ca/installing/) | Renders the configuration and secret templates | +| [skaffold](https://skaffold.dev/docs/install/) | Builds and deploys individual services | + +```bash +sudo curl -o /usr/local/bin/gomplate -sSL \ + https://github.com/hairyhenderson/gomplate/releases/download/v3.10.0/gomplate_linux-amd64 +sudo chmod 755 /usr/local/bin/gomplate +gomplate --help + +curl -Lo skaffold \ + https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 +sudo install skaffold /usr/local/bin/ +skaffold --help +``` + +Pin both to a known version for a production deployment rather than tracking +`latest`. + +## Generate and load configuration + +```bash +# render templates from config_values/.yaml +./generate_configs.py -e +./generate_secrets.py -e + +# load them into the cluster +./load_configs.py -e -n +./load_secrets.py -e -n +``` + +Rendered output contains every credential in the deployment. Keep it out of any +public repository, and re-render rather than hand-editing what is loaded. + +## Prerequisites for deploying services + +Registry pull secrets, created where the workloads that need them run: + +| Secret | Namespace | Created by | +|--------|-----------|------------| +| `harbor-registry-credentials` | DE namespace | [Harbor](./harbor.md) | +| `vice-image-pull-secret` | `vice-apps` | [VICE](../06-applications/vice.md) | + +Secrets and files the DE services expect, loaded by `load_secrets.py` from your +private secrets repository: + +* `gpg-keys` +* `ui-nginx-tls` +* `pgpass-files` +* `signing-keys` +* `accepted-keys` +* `ssl-files` + +Also required before the service set will come up: + +* A search cluster — [OpenSearch](../05-core-services/opensearch.md), or + [Elasticsearch](../05-core-services/elasticsearch.md) in older deployments. +* The service accounts applied: + + ```bash + kubectl apply -f resources/serviceaccounts/app-exposer.yml -n + ``` + +## Deploy + +Everything listed in `repos`: + +```bash +./deploy.py -n -BCa +``` + +A single service, for example `search`: + +```bash +./deploy.py -Bn -p search -C +``` + +In a full deployment, Ansible's `deploy-all-services` tag is the supported path; +`deploy.py` is for iterating on one service. See +[Discovery Environment](../06-applications/discovery-environment.md). + +# Creating a new environment + +Each environment is one file in `config_values/`. Copy an existing one and fill it +in for your site: + +```bash +cp config_values/prod.yaml config_values/.yaml +``` + +Every value has to be reviewed — the file is the single inventory of what the DE +services are configured with. The template below is that inventory; sections such +as `Keycloak`, `IRODS`, `Grouper`, `Harbor`, and the per-database blocks are +cross-referenced from the documents that produce their values. + +```yaml +# config_values/.yaml +--- +Environment: + +Agave: + Key: + Secret: + RedirectURI: + StorageSystem: + CallbackBaseURI: + ReadTimeout: + Enabled: + JobsEnabled: + +AMQP: + URI: + +AnonFiles: + BaseURI: + +AppExposer: + BaseURI: + +BaseURLs: + Analyses: + Apps: + AsyncTasks: + DashboardAggregator: + DataInfo: + GrouperWebServices: + IplantEmail: + IplantGroups: + JexAdapter: + Metadata: + Notifications: + Permissions: + Requests: + Search: + Terrain: + UserInfo: + +CAS: + BaseURI: + ServerName: + UIDDomain: + +DashboardAggregator: + PublicGroup: + LogLevel: + +DataOne: + BaseURI: + +DE: + Version: + VersionName: + AMQP: + URI: + Host: + BaseURI: + Legacy: + BaseURI: + Subscriptions: + CheckoutURL: + KeepAlive: + Service: + Target: + ContextMenu: + Enabled: + BaseTrash: + Path: + ProdDeployment: + DefaultOutputFolder: + WSO2: + JWTHeader: + Coge: + BaseURI: + Tools: + Admin: + MaxCpuLimit: + MaxMemoryLimit: + MaxDiskLimit: + +Docker: + TrustedRegistries: + Tag: + +Elasticsearch: + BaseURI: + Username: + Password: + Index: + +Email: + AppDeletion: + Src: + Dest: + AppPublicationRequest: + Src: + Dest: + ToolRequest: + Src: + Dest: + PermIDRequest: + Src: + Dest: + Support: + Src: + Dest: + +Grouper: + Environment: + MorphString: + WebService: + Password: + Password: + DB: + User: + Password: + Host: + Port: + Name: + FolderNamePrefix: + Loader: + URI: + User: + Password: + SubjectSource: + ID: + Name: + SearchBase: + +ICAT: + Host: + Port: + User: + Password: + +Infosquito: + DayNum: + PrefixLength: + +InteractiveApps: + BaseURI: + ServiceSuffix: + +Intercom: + AppID: + CompanyID: + CompanyName: + Intercom: + +IRODS: + AMQP: + URI: + Host: + User: + Zone: + Password: + AdminUsers: + PermsFilter: + ExternalHost: + QuotaRootResources: + +Jobs: + DataTransferImage: + +JobStatusListener: + BaseURI: + +Keycloak: + ServerURI: + Realm: + ClientID: + ClientSecret: + VICE: + ClientID: + ClientSecret: + +Kifshare: + ExternalUri: + +PGP: + KeyPassword: + +PermanentID: + CuratorsGroup: + DataCite: + BaseURI: + User: + Password: + DOIPrefix: + +Redis: + Host: + Port: + HA: + Name: + Password: + DB: + Number: + +TimeZone: + +Vault: + Token: + URL: + IRODS: + MountPath: + ChildToken: + UseLimit: + +VICE: + DB: + User: + Password: + Host: + Port: + Name: + FileTransfers: + Image: + Tag: + JobStatus: + BaseURI: + K8sEnabled: + BackendNamespace: + ImagePullSecret: + ImageCache: + UseCSIDriver: + DefaultImage: + DefaultName: + DefaultCasUrl: + DefaultCasValidate: + ConcurrentJobs: + UseCaseCharsMin: + DefaultBackend: + LoadingPageTemplateString: + +Sonora: + BaseURI: + +Terrain: + CASClientID: + CASClientSecret: + JWT: + SigningKey: + Password: + +Unleash: + BaseUrl: + APIPath: + APIToken: + MaintenanceFlag: + +UserPortal: + BaseURI: + +DEDB: + User: + Password: + Host: + Port: + Name: + +NewNotificationsDB: + User: + Password: + Host: + Port: + Name: + +NotificationsDB: + User: + Password: + Host: + Port: + Name: + +PermissionsDB: + User: + Password: + Host: + Port: + Name: + +QMSDB: + User: + Password: + Host: + Port: + Name: + Reinitialize: + +MetadataDB: + User: + Password: + Host: + Port: + Name: + +UnleashDB: + User: + Password: + Host: + Port: + Name: + +Admin: + Groups: + Attribute: + +FileIdentifier: + HtPathList: + MultiInputPathList: + +Analytics: + Enabled: + Id: + +Harbor: + URL: + ProjectQARobotName: + ProjectQARobotSecret: + +QMS: + Enabled: + Base: + Usage: + +Jaeger: + Endpoint: +``` + +# Related + +* [Harbor](./harbor.md) +* [Discovery Environment](../06-applications/discovery-environment.md) +* [VICE](../06-applications/vice.md) +* [Keycloak](../05-core-services/keycloak.md) diff --git a/docs/deployment/04-kubernetes/storage.md b/docs/deployment/04-kubernetes/storage.md new file mode 100644 index 0000000..10e7aa4 --- /dev/null +++ b/docs/deployment/04-kubernetes/storage.md @@ -0,0 +1,88 @@ +--- +type: Deployment Procedure +title: "Storage" +description: "Providing persistent volumes to the cluster with Longhorn or OpenEBS." +tags: [deployment, kubernetes, storage, longhorn, openebs] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: longhorn-docs + resource: https://longhorn.io/docs/ + title: Longhorn documentation + author: team:longhorn + - id: openebs-docs + resource: https://openebs.io/docs + title: OpenEBS documentation + author: team:openebs +--- + +# What needs persistent volumes + +Cluster storage is for the stateful services that run *inside* Kubernetes — +Keycloak, Redis, the search cluster, Harbor — not for user data. User data lives +in the [Data Store](../../platform/data-store.md) and reaches pods through the +[iRODS CSI driver](../05-core-services/irods-csi-driver.md), which is a separate +concern with a separate lifecycle. + +Two provisioners are in use across CyVerse deployments: + +| Provisioner | Status | Notes | +|-------------|--------|-------| +| Longhorn | Current | Used by the pilot; replicated block storage with a management UI | +| OpenEBS | Legacy | `openebs-hostpath` storage class in older deployments | + +Both can be present in one cluster, and a migration between them is a +volume-by-volume exercise. Pick one for new work. + +# Longhorn + +```bash +ansible-playbook -i /path/to/inventory --tags longhorn kubernetes.yml +``` + +Longhorn replicates each volume across nodes, so on a two-node cluster set the +replica count to what the node count can actually satisfy — asking for three +replicas on two nodes leaves volumes permanently `Degraded`. It also needs +`open-iscsi` on every node; the `prep-nodes` tag installs it. + +Verify: + +```bash +kubectl -n longhorn-system get pods +kubectl get storageclass +``` + +# OpenEBS (legacy) + +Older deployments provision `openebs-hostpath` volumes: + +```bash +kubectl create ns openebs +kubectl -n openebs apply -f https://openebs.github.io/charts/openebs-operator.yaml +``` + +`openebs-hostpath` is node-local: a pod using one of these volumes is pinned to +the node holding the data, and the volume does not survive that node's loss. The +[Redis HA](../05-core-services/redis-ha.md) values in this bundle still name +`openebs-hostpath` as their storage class — change it to your Longhorn class in +a new deployment. + +# Choosing a storage class per service + +| Service | Guidance | +|---------|----------| +| Keycloak, Harbor | Replicated (Longhorn); losing this data means re-registering clients or re-pushing images | +| Redis | Replicated preferred; Redis HA replicates at the application layer too | +| Search cluster | Replicated, sized generously; reindexing is expensive but possible | +| Scratch and caches | Node-local is fine | + +# Related + +* [Cluster](./cluster.md) +* [iRODS CSI driver](../05-core-services/irods-csi-driver.md) +* [Namespaces](../../architecture/namespaces.md) diff --git a/docs/deployments/elasticsearch.md b/docs/deployment/05-core-services/elasticsearch.md similarity index 79% rename from docs/deployments/elasticsearch.md rename to docs/deployment/05-core-services/elasticsearch.md index cb56355..0245e6f 100644 --- a/docs/deployments/elasticsearch.md +++ b/docs/deployment/05-core-services/elasticsearch.md @@ -1,19 +1,31 @@ +--- +type: Deployment Procedure +title: "Elasticsearch (legacy)" +description: "The legacy Elasticsearch stateful set and index mappings, superseded by OpenSearch in new deployments." +tags: [deployment, core-services, elasticsearch, search] +status: deprecated +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Elasticsearch -Deploying the `statefulsets` Elasticsearch cluster on kubernetes. +!!! warning "Superseded by OpenSearch" + New deployments install [OpenSearch](./opensearch.md) instead. This document + is kept for the clusters still running Elasticsearch — the index mappings + below are Elasticsearch 6 era and use `_parent` and multiple mapping types + per index, neither of which exists in current Elasticsearch or OpenSearch. + Do not copy them into a new deployment. -!!! success "Prerequisites" - - * clone the repo - - * see also [k8s-resources](k8s-resources.md) - - * `cd /k8s-resources` +Deploying the Elasticsearch stateful set on Kubernetes. - * In case you have limited resources change these: +!!! success "Prerequisites" - `cd /k8s-resources/resources/addons/elasticsearch/elasticsearch.yml` + * A checkout of the [cluster resources](../04-kubernetes/resources.md) + repository; the paths below are relative to its root. + * A storage class for the stateful set; see + [storage](../04-kubernetes/storage.md). + * On a small cluster, lower the replica count and heap in + `resources/addons/elasticsearch/elasticsearch.yml`: ```yaml replicas: 2 @@ -28,17 +40,15 @@ Deploying the `statefulsets` Elasticsearch cluster on kubernetes. ### Deploy -```bash -# deploy ES on prod env -kubectl apply -n prod -f resources/addons/elasticsearch/elasticsearch.yml +Substitute the namespace the DE services run in: -## deploy ES on discover env -# kubectl apply -n discover -f resources/addons/elasticsearch/elasticsearch.yml +```bash +kubectl apply -n -f resources/addons/elasticsearch/elasticsearch.yml ``` ## Indexing -### Preq +### Prerequisites Save below json to a file `settings.json`, we will use this file to index our elasticsearch. @@ -318,19 +328,11 @@ Save below json to a file `settings.json`, we will use this file to index our el For indexing we will run a container inside your **namespace** where the elasticsearch is running, and copy the `settings.json` file inside this container and run the following commands: ```bash -## run container if es in prod namespace -kubectl run --namespace=prod --rm utils -it --image arunvelsriram/utils bash - -## run container if es in discover namespace -# kubectl run --namespace=discover --rm utils -it --image arunvelsriram/utils bash - -######### PS open a new terminal######### -## copy the settings.json if PROD -kubectl -n prod cp settings.json utils:/home/utils - -## copy the settings.json if Discover -# kubectl -n discover cp settings.json utils:/home/utils +# start a utility pod in the namespace the search cluster runs in +kubectl run --namespace= --rm utils -it --image arunvelsriram/utils bash +# then, from a second terminal, copy the mappings into that pod +kubectl -n cp settings.json utils:/home/utils ``` #### run indexing @@ -358,6 +360,13 @@ curl -sX DELETE "http://elasticsearch:9200/*" #### restart related services ```bash -# kubectl rollout restart statefulset elasticsearch -n prod # not sure kubectl rollout restart deployment infosquito2 search -n ``` + +Restarting the two consumers is what is needed; the stateful set itself does not +have to be restarted for a mapping change on a new index. + +### Related + +* [OpenSearch](./opensearch.md) — what new deployments use instead +* [Reindex search](../01-foundation/rabbitmq.md#operations-reindex-search) diff --git a/docs/deployment/05-core-services/grouper.md b/docs/deployment/05-core-services/grouper.md new file mode 100644 index 0000000..4cb5733 --- /dev/null +++ b/docs/deployment/05-core-services/grouper.md @@ -0,0 +1,63 @@ +--- +type: Deployment Procedure +title: "Grouper" +description: "Deploying the Grouper loader and web services for group management." +tags: [deployment, core-services, grouper, groups] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# Role in the deployment + +[Internet2 Grouper](https://www.internet2.edu/products-services/trust-identity/grouper/) +manages the groups CyVerse authorizes against — the DE reads group membership +through it rather than querying LDAP directly. Two deployments make it up: + +| Deployment | Job | +|------------|-----| +| `grouper-loader` | Syncs subjects and groups from the directory on a schedule | +| `grouper-ws` | Web services the DE queries at request time | + +# Prerequisites + +* [Grouper database](../02-databases/grouper.md) created. +* [OpenLDAP](./openldap.md) running, with the `ou=Groups` branch populated. +* The `Grouper` section of the deployment group variables filled in — loader URI + and credentials, web service password, morph string, folder name prefix, and + subject source configuration. See + [cluster resources](../04-kubernetes/resources.md). + +# Deploy + +Ansible deploys both parts along with the other core services: + +```bash +ansible-playbook -i /path/to/inventory \ + --tags=feature-discovery,image-cache,grouper kubernetes.yml +``` + +To apply the manifests directly instead — from the +[cluster resources](../04-kubernetes/resources.md) checkout, substituting the +namespace the DE runs in: + +```bash +kubectl apply -f resources/deployments/grouper-loader.yml -n +kubectl apply -f resources/deployments/grouper-ws.yml -n +``` + +# Verify + +```bash +kubectl -n get pods -l app=grouper-ws +kubectl -n logs deploy/grouper-loader --tail=100 +``` + +The loader logs each sync. A loader that starts and then idles without syncing +usually cannot reach either the database or the directory; check both before +looking at Grouper's own configuration. + +# Related + +* [Grouper database](../02-databases/grouper.md) +* [OpenLDAP](./openldap.md) +* [Keycloak](./keycloak.md) diff --git a/docs/deployment/05-core-services/index.md b/docs/deployment/05-core-services/index.md new file mode 100644 index 0000000..b6f81d3 --- /dev/null +++ b/docs/deployment/05-core-services/index.md @@ -0,0 +1,32 @@ +# Phase 5: core services + +The services the Discovery Environment authenticates, searches, caches, and +communicates through. Deploy the directory before Keycloak, and Keycloak before +anything that needs its client secrets. + +# Identity + +* [OpenLDAP](openldap.md) - accounts and POSIX groups +* [Keycloak](keycloak.md) - realm, LDAP federation, mappers, roles, and OAuth clients +* [Grouper](grouper.md) - group management the DE authorizes against + +# Search + +* [OpenSearch](opensearch.md) - the data search index used by new deployments +* [Elasticsearch (legacy)](elasticsearch.md) - the superseded search stack + +# Messaging and state + +* [NATS](nats.md) - internal service-to-service messaging +* [Redis HA](redis-ha.md) - caching and session state +* [Unleash](unleash.md) - feature flags + +# Storage and utilities + +* [iRODS CSI driver](irods-csi-driver.md) - mounting Data Store paths into pods +* [Mail](mail.md) - outbound mail for DE and portal notifications +* [Jaeger](jaeger.md) - distributed tracing (optional) + +# Next + +* [Phase 6: applications](../06-applications/) diff --git a/docs/deployment/05-core-services/irods-csi-driver.md b/docs/deployment/05-core-services/irods-csi-driver.md new file mode 100644 index 0000000..02b1a41 --- /dev/null +++ b/docs/deployment/05-core-services/irods-csi-driver.md @@ -0,0 +1,161 @@ +--- +type: Deployment Procedure +title: "iRODS CSI driver" +description: "Installing the iRODS Container Storage Interface driver that mounts Data Store paths into pods." +tags: [deployment, core-services, irods, storage, csi] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- +# Role in the deployment + +The [iRODS CSI driver](https://github.com/cyverse/irods-csi-driver) implements the +Container Storage Interface so that Kubernetes can mount Data Store paths straight +into pods. It is how a VICE analysis sees the user's home collection as a +filesystem instead of having to stage data in and out. + +It is a separate concern from [cluster storage](../04-kubernetes/storage.md): +Longhorn and OpenEBS provide volumes to stateful cluster services, while this +driver provides user data to analyses. + +# Prerequisites + +* A running [iRODS provider](../03-data-store/irods-provider.md) reachable from + the cluster on `1247/tcp`. +* The `de-irods` account from + [DE integration](../03-data-store/de-integration.md), or another iRODS admin + account for the driver to proxy through. + +# Values + +Create a `values.yaml`. It contains an iRODS admin password, so keep it in your +private inventory. + +```yaml +globalConfig: + secret: + stringData: + client: "irodsfuse" + host: + port: "1247" + zone: "" + user: + password: + retainData: "false" + enforceProxyAccess: "true" + mountPathWhitelist: "//home" +nodeService: + irodsPool: + extraArgs: + - --cache_size_max=10737418240 + - --cache_root=/irodsfs_pool_cache + - '--cache_timeout_settings=[{"path":"/","timeout":"-1ns","inherit":false},{"path":"/","timeout":"-1ns","inherit":false},{"path":"//home","timeout":"5m","inherit":false},{"path":"//home/shared","timeout":"5m","inherit":true}]' + +``` + +Two settings deserve attention: + +* **`enforceProxyAccess: "true"`** makes the driver act on behalf of the + requesting user rather than as the admin account. Leave it on — with it off, any + pod that can mount a volume reads the zone with admin rights. +* **`mountPathWhitelist`** bounds what can be mounted at all. Keep it as narrow as + your analyses allow. + +# Deploy + +```bash +# Add the Helm repository. +helm repo add irods-csi-driver-repo https://cyverse.github.io/irods-csi-driver-helm/ + +# Update the local repository caches. +helm repo update + +# create namespace +kubectl create namespace irods-csi-driver + +# install csi-driver +# make sure to edit values.yaml +helm install -n irods-csi-driver irods-csi-driver irods-csi-driver-repo/irods-csi-driver -f ./values.yaml + +# or upgrade +helm upgrade -n irods-csi-driver irods-csi-driver irods-csi-driver-repo/irods-csi-driver -f ./values.yaml + +``` + +# Verify + +```bash +kubectl -n irods-csi-driver get pods +kubectl get csidrivers +``` + +# Upgrading + +An upgrade is disruptive: running interactive analyses hold volumes provisioned by +the current driver, so they have to be stopped and their claims removed first. +Schedule it, and warn users. + +```bash +# update helm repo +helm repo update + +# delete the pvc +kubectl delete pvc -l app-type=interactive -n vice-apps + +# uninstall the irods-csi-driver +helm uninstall irods-csi-driver -n irods-csi-driver + +# delete all the vice-apps deployments +## see below for the content of this file +./nuke-vice-analysis.sh $(kubectl get deployments -n vice-apps -l app-type=interactive -o name) + +# install again +helm install -n irods-csi-driver irods-csi-driver irods-csi-driver-repo/irods-csi-driver -f values.yaml +``` + +## Pinning a version + +```bash +helm install -n irods-csi-driver irods-csi-driver \ + --version irods-csi-driver-repo/irods-csi-driver -f values.yaml +``` + +Pin the chart version in a production deployment. `helm search repo +irods-csi-driver-repo` lists what is available. + +!!! warning "Configuration change after 0.8.7" + + In chart versions above 0.8.7, the `user_config.yaml` handling changed: the + `--cache_root` and `--temp_root` flags must be removed if you were passing + them. Leaving them in place makes the node service fail to start. + +# nuke-vice-analysis.sh + +Used by the upgrade procedure above to tear down interactive analyses along with +the resources `app-exposer` created for them. + +```sh +function delete_resources() { + local external_id="$1" + kubectl -n vice-apps delete deployment "${external_id}" + kubectl -n vice-apps delete service "vice-${external_id}" + kubectl -n vice-apps delete ingress "${external_id}" + kubectl -n vice-apps delete configmap "excludes-file-${external_id}" + kubectl -n vice-apps delete configmap "input-path-list-${external_id}" +} + +function remove_deployment_prefix() { + local external_id="$1" + echo -n "$external_id" | sed 's;^deployment.apps/;;' +} + +# Iterate over all arguments on the command line. +for id in "$@"; do + delete_resources $(remove_deployment_prefix "$id") +done +``` + +# Related + +* [iRODS provider](../03-data-store/irods-provider.md) +* [VICE](../06-applications/vice.md) +* [Cluster storage](../04-kubernetes/storage.md) diff --git a/docs/deployment/05-core-services/jaeger.md b/docs/deployment/05-core-services/jaeger.md new file mode 100644 index 0000000..390dfd8 --- /dev/null +++ b/docs/deployment/05-core-services/jaeger.md @@ -0,0 +1,59 @@ +--- +type: Deployment Procedure +title: "Jaeger" +description: "Deploying Jaeger for end-to-end distributed tracing of DE services." +tags: [deployment, core-services, jaeger, observability, tracing] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# Role in the deployment + +[Jaeger](https://www.jaegertracing.io/) collects distributed traces from DE +services, which is how a slow request is attributed to a specific service rather +than to "the DE". Services send spans to the collector endpoint configured in the +`Jaeger` section of the deployment group variables. + +Jaeger is optional. Nothing user-facing depends on it, and it can be added to a +running deployment later. + +# Prerequisites + +* A search cluster to store spans in — [OpenSearch](./opensearch.md), or + [Elasticsearch](./elasticsearch.md) in older deployments. +* The manifests from [cluster resources](../04-kubernetes/resources.md). + +# Deploy + +```bash +kubectl create ns jaeger + +kubectl apply -f resources/addons/jaeger/rollover-cron.yaml -n jaeger +kubectl apply -f resources/addons/jaeger/query.yaml -n jaeger +kubectl apply -f resources/addons/jaeger/collector.yaml -n jaeger +``` + +Each of those three manifests names the search cluster it talks to. If your +search cluster is not in the `prod` namespace, update the endpoint in all three +before applying: + +```diff +- "http://elasticsearch.prod:9200" ++ "http://.:9200" +``` + +The rollover cron job is what keeps span indices from growing without bound. +Deploy it, not just the collector and query components — a Jaeger install without +rollover fills its storage and then takes the search cluster down with it. + +# Verify + +```bash +kubectl -n jaeger get pods +kubectl -n jaeger logs deploy/jaeger-collector --tail=50 +``` + +# Related + +* [OpenSearch](./opensearch.md) +* [Cluster resources](../04-kubernetes/resources.md) diff --git a/docs/deployment/05-core-services/keycloak.md b/docs/deployment/05-core-services/keycloak.md new file mode 100644 index 0000000..7b6187c --- /dev/null +++ b/docs/deployment/05-core-services/keycloak.md @@ -0,0 +1,314 @@ +--- +type: Deployment Procedure +title: "Keycloak" +description: "Deploying Keycloak and configuring the realm, LDAP federation, mappers, roles, and OAuth clients the DE requires." +tags: [deployment, core-services, keycloak, authentication, oauth] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: keycloak-docs + resource: https://www.keycloak.org/documentation + title: Keycloak documentation + author: team:keycloak +--- + +# Prerequisites + +* [Keycloak database](../02-databases/keycloak.md) created. +* [OpenLDAP](./openldap.md) running, with the reader service account and the + `ou=People` and `ou=Groups` branches ready. +* [cert-manager](../04-kubernetes/cert-manager.md) and + [ingress](../04-kubernetes/ingress.md) in place, so + `keycloak.` can be served over HTTPS. + +# Deploy + +```bash +ansible-playbook -i /path/to/inventory --tags keycloak kubernetes.yml +``` + +Add DNS for `keycloak.` — a `CNAME` to the ingress is typical. An +`/etc/hosts` entry will carry you through the configuration below, but it is not +a deployment; fix DNS before anything else points at Keycloak. + +??? info "Deploying by kustomize instead of the playbook" + + Older deployments applied Keycloak with kustomize, generating its secrets and + configuration inline. The shape of that configuration is still a useful + reference for what Keycloak needs: + + ```yaml + secretGenerator: + - name: dbuser + literals: + - username= + - password= + - name: kcadmin + literals: + - username= + - password= + configMapGenerator: + - name: keycloak-config + literals: + - KEYCLOAK_HOSTNAME=keycloak. + - KEYCLOAK_LOGLEVEL=INFO + - DB_VENDOR=postgres + - DB_ADDR= + - DB_PORT=5432 + - PROXY_ADDRESS_FORWARDING=true + - JDBC_PARAMS=connectTimeout=21600 + namespace: keycloak + resources: + - deployment.yaml + - service.yaml + generatorOptions: + disableNameSuffixHash: true + ``` + + ```bash + kubectl apply -k ./base/ -n keycloak + ``` + + Generate every literal at install time. Never commit a rendered + `kustomization.yaml` containing real secrets. + +# Realm + +Create a realm named for your site (``). It can take a few moments to +appear; confirm it is selected as the current realm before continuing — the rest +of this configuration silently lands in `master` otherwise. + +**Configure → Realm settings**: + +| Setting | Value | +|---------|-------| +| Realm name | `` | +| Display name / HTML display name | Whatever users should see | +| Frontend URL | blank | +| Require SSL | All requests | + +Leave everything else at its default, and save at the bottom of the page. + +# LDAP user federation + +**Configure → User federation → Add LDAP provider**, with the realm selected: + +| Setting | Value | +|---------|-------| +| Vendor | Other | +| Connection URL | `ldap://openldap.openldap` | +| StartTLS | off | +| Use Truststore SPI | Always | +| Connection pooling | on | +| Connection timeout | blank | +| Bind type | simple | +| Bind DN | `uid=ldap_reader,ou=People,` (from `ldap_cn` and `ldap_base_dn`) | +| Bind credentials | the value of `ldap_ldap_reader_pw` | + +Use **Test connection** and **Test authentication** before going further. A toast +in the upper right reports the result; if either fails, nothing below will work. + +LDAP searching and updating: + +| Setting | Value | +|---------|-------| +| Edit mode | `READ_ONLY` | +| Users DN | the bind DN without the `uid` component, e.g. `ou=People,` | +| Relative user creation | blank | +| Username LDAP attribute | `uid` | +| RDN LDAP attribute | `uid` | +| UUID LDAP attribute | `uidNumber` | +| User object classes | `inetOrgPerson, posixAccount` | +| LDAP filter | blank | +| Search scope | One Level | +| Read timeout | blank | +| Pagination | on | +| Referral | blank | + +Synchronization: + +| Setting | Value | +|---------|-------| +| Import users | on | +| Sync registrations | off | +| Batch size | `500` | +| Periodic full sync | on, period `86400` | +| Periodic changed users sync | on, period `3600` | +| Remove invalid users during searches | on | + +Leave Kerberos integration off, cache settings at `DEFAULT`, and advanced +settings off. Save. + +!!! note "`READ_ONLY` and sync registrations off are deliberate" + + Accounts are created in LDAP by the User Portal, not in Keycloak. Letting + Keycloak write back would produce two systems of record for one account. + +# LDAP mappers + +**Configure → User federation → (the LDAP provider) → Mappers → Add mapper.** +Three mappers are needed. + +## entitlement + +| Setting | Value | +|---------|-------| +| Name | `entitlement` | +| Mapper type | `group-ldap-mapper` | +| LDAP Groups DN | `ou=Groups,` | +| Group Object Classes | `posixGroup` | +| Preserve Group Inheritance | off | +| Membership LDAP Attribute | `memberUid` | +| Membership Attribute Type | `UID` | +| Membership User LDAP Attribute | `uid` | +| Mode | `READ_ONLY` | +| User Groups Retrieve Strategy | `LOAD_GROUPS_BY_MEMBER_ATTRIBUTE` | +| Member-Of LDAP Attribute | `memberOf` | +| Drop non-existing groups during sync | on | +| Groups Path | `/` | + +Preserve Group Inheritance must be off because POSIX groups are flat; leaving it +on makes the sync fail on groups that have no parent. + +## name + +| Setting | Value | +|---------|-------| +| Name | `name` | +| Mapper type | `user-attribute-ldap-mapper` | +| User Model Attribute | `name` (typed into the text box, nothing selected from the dropdown) | +| LDAP attribute | `cn` | +| Read Only | on | +| Always Read Value From LDAP | on | +| Is Mandatory in LDAP | on | +| Force a Default Value | on | + +## roles + +| Setting | Value | +|---------|-------| +| Name | `roles` | +| Mapper type | `role-ldap-mapper` | +| LDAP Roles DN | `ou=Groups,` (same as the entitlement mapper) | +| Role Object Classes | `posixGroup` | +| Membership LDAP Attribute | `memberUid` | +| Membership Attribute Type | `UID` | +| Mode | `READ_ONLY` | + +Everything not listed can stay at its default. + +# Client scope mappers + +The DE reads `name` and `entitlement` out of the token, so both have to be added +to the `profile` client scope. **Manage → Client scopes → profile → Mappers → +Add mapper → By configuration**: + +| Mapper | Type | Settings | +|--------|------|----------| +| `name` | User Property | Name, Property, and Token Claim Name all set to `name` | +| `entitlement` | Group Membership | Name and Token Claim Name set to `entitlement`; **Full group path off** | + +Leaving Full group path on prefixes every group with `/`, and the DE's +entitlement checks then match nothing. + +# Authentication + +**Configure → Authentication**. Leave the Flows tab alone. In **Required +actions**, enable exactly these and disable the rest: + +* Terms and Conditions +* Update Password +* Update Profile +* Verify Email +* Delete Credential +* Linking Identity Provider +* Update User Locale + +# Realm roles + +**Manage → Realm roles**, add: + +| Role | Held by | +|------|---------| +| `app-runner` | Accounts permitted to run apps | +| `cyverse-emailer` | The service that sends mail | +| `cyverse-ldap-reader` | The directory reader | +| `cyverse-subscription-updater` | Subscription and quota updates | +| `vice-operator` | The VICE operator service account | + +# Clients + +Eight clients. **Manage → Clients → Create client.** The create wizard presents +fields in a different order than the finished client page, so work from the +tables below rather than from wizard step order, and re-check the client detail +page when you are done. + +Every client secret below is read from the **Credentials** tab on the client +detail page and written into `group_vars/all.yml` in your **private** inventory. + +## Interactive clients + +| Client ID | Root / Home URL | Valid redirect URIs | Web origins | Admin URL | +|-----------|-----------------|---------------------|-------------|-----------| +| `de-` | `https://de.` | `https://de./*` | `https://de.` | `https://de.` | +| `portal-` | `https://user.` | `https://user./*` | `https://user.` | `https://user.` | +| `vice-` | — | `https://*` | `https://*.vice./*` | — | + +All three: client authentication **on**, authorization **off**, valid post logout +redirect URIs `+`. Authentication flow: Standard flow and Direct access grants +checked and nothing else — except `vice-`, which also needs Service account +roles. + +| Client | Client ID variable | Secret variable | +|--------|--------------------|-----------------| +| `de-` | `keycloak_client_id` | `keycloak_client_secret` | +| `vice-` | `keycloak_vice_client_id` | `keycloak_vice_client_secret` | +| `portal-` | `portal_keycloak_client` | `portal_keycloak_secret` | + +## VICE service clients + +| Client ID | Purpose | Configuration | +|-----------|---------|---------------| +| `vice-api` | Lets the app-exposer backend reach the VICE operator | Client authentication on; **only** Service account roles checked. After saving: **Service account roles → Assign role → Realm roles → `vice-operator` → Assign** | +| `vice-users` | VICE authentication callback | Client authentication on; Standard flow and Direct access grants checked; valid redirect URI `https://vice-api.vice./auth/callback`; web origins `+` | +| `vice-swagger` | VICE API documentation UI | Client authentication on, authorization off; Standard flow checked; valid redirect URI `https://vice-api.vice./docs/callback`; web origins `https://vice-api.vice.`; post logout redirect `+` | + +Their client IDs default to the names above, so only the secrets need setting: + +| Client | Secret variable | +|--------|-----------------| +| `vice-api` | `vice_api_keycloak_client_secret` | +| `vice-users` | `vice_operator_keycloak_client_secret` | +| `vice-swagger` | `vice_operator_swagger_client_secret` | + +!!! note "Two variable names do not match their client names" + + `vice-users` and `vice-swagger` write into variables named for the *operator*. + That is what the playbooks read, so copy the values into the variables exactly + as listed rather than "correcting" them. + +## Remaining service clients + +| Client ID | Purpose | Configuration | Variables | +|-----------|---------|---------------|-----------| +| `formation-service-account` | Service-to-service access for formation | Client authentication on; Service account roles checked; everything else default | `formation_keycloak_client_id`, `formation_keycloak_client_secret` | +| `de-admin-api` | DE administrative API | Root and Home URL `https://de.`; redirect `https://de./*`; post logout `+`; web origins `+`; Admin URL `https://de.`; client authentication on; Standard flow **and** Service account roles checked, **Direct access grants unchecked** | `keycloak_admin_client_id`, `keycloak_admin_client_secret` | + +# After Keycloak + +The client secrets you just collected are inputs to the phase that follows, so +finish this document before running further playbooks. Next: + +1. [Generate service signing keys](../from-scratch.md#53-service-signing-keys). +2. Apply configuration, ingress, networking, and NATS. + +# Related + +* [Authentication architecture](../../platform/authentication.md) +* [OpenLDAP](./openldap.md) +* [Keycloak database](../02-databases/keycloak.md) diff --git a/docs/deployment/05-core-services/mail.md b/docs/deployment/05-core-services/mail.md new file mode 100644 index 0000000..94afa75 --- /dev/null +++ b/docs/deployment/05-core-services/mail.md @@ -0,0 +1,101 @@ +--- +type: Deployment Procedure +title: "Mail" +description: "Deploying outbound mail: the exim4 smarthost chart and the in-cluster exim-sender deployment." +tags: [deployment, core-services, mail, exim, smtp] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: exim4-helm + resource: https://github.com/mb-wali/exim4-helm + title: exim4 Helm chart + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# What needs mail + +The DE sends mail for app publication requests, tool requests, permanent ID +requests, app deletion notices, and support messages. The destinations are +configured in the `Email` section of the deployment group variables; see +[cluster resources](../04-kubernetes/resources.md). The User Portal also sends +account verification mail. + +Two deployments exist, and they are alternatives rather than layers. Pick the one +that matches how your site relays mail. + +| Option | Use when | +|--------|----------| +| exim4 smarthost (Helm) | You relay through an institutional or provider SMTP smarthost | +| exim-sender (manifest) | You want a minimal in-cluster sender managed with the other DE manifests | + +Both present an SMTP endpoint inside the cluster that DE services point at +through `SMTP_HOST`. + +# Option 1: exim4 smarthost + +A Helm chart providing exim4 as a mail transfer agent in smarthost +mode.[^exim4-helm] + +```bash +helm repo add exim4 https://mb-wali.github.io/exim4-helm +helm repo update + +helm install exim4 exim4/exim4 \ + --namespace mail --create-namespace --wait \ + --set secrets.EXIM_SMARTHOST='' \ + --set secrets.EXIM_PASSWORD='' \ + --set secrets.EXIM_ALLOWED_SENDERS='' +``` + +!!! warning "Values on the command line are not private" + + `--set` puts the smarthost password into your shell history and into the Helm + release. Prefer a values file kept in the private inventory, or a + pre-created secret that the chart references. + + `EXIM_ALLOWED_SENDERS='*'` appears in older notes. It permits relaying from + any sender; scope it to your own domains instead. + +In-cluster endpoint: + +``` +SMTP_HOST=exim4.mail.svc.cluster.local +``` + +## Verify + +```bash +kubectl -n mail get pods +kubectl -n mail exec -it deploy/exim4 -- bash + +# from inside the pod +echo "This is a test" | mail -s "Test subject" \ + you@ -aFrom:noreply@ +``` + +Then check the exim logs in the pod for the delivery result. A message accepted +locally but never delivered is usually the smarthost rejecting the envelope +sender. + +# Option 2: exim-sender + +Also known as `local-exim`, deployed from the manifests in +[cluster resources](../04-kubernetes/resources.md) alongside the other DE +services: + +```bash +kubectl apply -f resources/deployments/exim-sender.yml -n +``` + +Use the namespace the DE services run in (`prod` in a standard deployment). + +# Related + +* [Cluster resources](../04-kubernetes/resources.md) +* [User Portal](../06-applications/user-portal.md) + +[^exim4-helm]: exim4 Helm chart diff --git a/docs/deployment/05-core-services/nats.md b/docs/deployment/05-core-services/nats.md new file mode 100644 index 0000000..7ac20cd --- /dev/null +++ b/docs/deployment/05-core-services/nats.md @@ -0,0 +1,59 @@ +--- +type: Deployment Procedure +title: "NATS" +description: "Deploying the NATS messaging layer used for internal service-to-service communication." +tags: [deployment, core-services, nats, messaging] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: nats-docs + resource: https://docs.nats.io/ + title: NATS documentation + author: team:nats-io +--- + +# Role in the deployment + +NATS carries in-cluster messages between DE services. It sits alongside +[RabbitMQ](../01-foundation/rabbitmq.md) rather than replacing it: RabbitMQ is +the AMQP bus that iRODS publishes data events to from outside the cluster, while +NATS is the lower-latency internal fabric newer DE services use. + +# Install + +NATS is installed together with the global configuration, ingress, and networking +resources, because those services expect it to be there when they start: + +```bash +ansible-playbook -i /path/to/inventory \ + --tags=configure-services,ingress,networking,nats kubernetes.yml +``` + +# Reinstalling + +Helm keeps a release record even after the resources are gone, so a reinstall +into the same namespace can fail with an "already exists" error. Remove the +release first: + +```bash +helm -n prod uninstall nats +``` + +Then re-run the tag above. + +# Verify + +```bash +kubectl -n prod get pods -l app.kubernetes.io/name=nats +kubectl -n prod logs -l app.kubernetes.io/name=nats --tail=50 +``` + +# Related + +* [Cluster resources](../04-kubernetes/resources.md) +* [RabbitMQ](../01-foundation/rabbitmq.md) diff --git a/docs/deployment/05-core-services/openldap.md b/docs/deployment/05-core-services/openldap.md new file mode 100644 index 0000000..b6fc623 --- /dev/null +++ b/docs/deployment/05-core-services/openldap.md @@ -0,0 +1,122 @@ +--- +type: Deployment Procedure +title: "OpenLDAP" +description: "Deploying the LDAP directory that holds CyVerse accounts and groups, and the service accounts that read it." +tags: [deployment, core-services, ldap, openldap, accounts] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# Role in the deployment + +OpenLDAP is the system of record for CyVerse accounts and POSIX groups. Keycloak +federates it read-only, the User Portal creates entries in it, and the DE reads +group membership from it through Grouper. Deploy it before +[Keycloak](./keycloak.md) — Keycloak's user federation configuration fails +without something to bind to. + +# Install + +```bash +ansible-playbook -i /path/to/inventory --tags de-reqs kubernetes.yml +ansible-playbook -i /path/to/inventory --tags openldap-docker kubernetes.yml +``` + +The `de-reqs` tag installs shared prerequisites and runs first. + +In-cluster, the directory is reachable as `ldap://openldap.openldap` — service +`openldap` in namespace `openldap`. That is the URL Keycloak's federation +provider uses; there is no need to expose LDAP outside the cluster. + +# Directory layout + +| Branch | Holds | +|--------|-------| +| `ou=People,` | User accounts (`inetOrgPerson`, `posixAccount`) | +| `ou=Groups,` | POSIX groups (`posixGroup`), including `de_admins` | + +The group branch is what Keycloak's `entitlement` and `roles` mappers read; see +[Keycloak](./keycloak.md#ldap-mappers). + +# Service accounts + +Two accounts matter to the rest of the deployment: + +| Account | Used by | Variable | +|---------|---------|----------| +| LDAP reader | Keycloak federation bind | `ldap_cn`, `ldap_base_dn`, `ldap_ldap_reader_pw` | +| `portal` | User Portal account creation | portal group variables | + +The reader account only needs read access; give it nothing more. Its bind DN is +assembled from the inventory variables, for example +`uid=ldap_reader,ou=People,`. + +## Creating the portal service account + +The User Portal needs an LDAP identity to create accounts with. Create it from an +LDIF, then set its password separately so the password never appears in a file: + +```ldif +dn: uid=portal,ou=People, +objectClass: inetOrgPerson +objectClass: posixAccount +objectClass: shadowAccount +uid: portal +mail: portal@ +sn: ServiceAccount +givenName: PORTAL +cn: portal +title: Other +o: N/A +departmentNumber: N/A +uidNumber: 40003 +gidNumber: 10003 +homeDirectory: /home/portal +``` + +```bash +ldapadd -x -D "cn=Manager," -W -f portal-user.ldif +ldappasswd -x -D "cn=Manager," -W -S "uid=portal,ou=People," +``` + +`-W` and `-S` prompt for the passwords instead of taking them on the command +line, where they would land in shell history and process listings. + +Then add the account to the DE administrators group: + +```ldif +dn: cn=de_admins,ou=Groups, +changetype: modify +add: memberUid +memberUid: portal +``` + +```bash +ldapmodify -x -D "cn=Manager," -W -f portal-de_admin.ldif +``` + +See [User Portal](../06-applications/user-portal.md) for the iRODS and database +accounts the portal also needs. + +# Adding the community group later + +If an existing deployment is missing the community group, the deployment +repository has an idempotent playbook for it: + +```bash +ansible-playbook -i /path/to/inventory openldap_community_group.yml +``` + +It is safe to run when you are not sure whether it is needed. + +# Related + +* [Keycloak](./keycloak.md) +* [Grouper](./grouper.md) +* [User Portal](../06-applications/user-portal.md) diff --git a/docs/deployment/05-core-services/opensearch.md b/docs/deployment/05-core-services/opensearch.md new file mode 100644 index 0000000..fb20fa8 --- /dev/null +++ b/docs/deployment/05-core-services/opensearch.md @@ -0,0 +1,71 @@ +--- +type: Deployment Procedure +title: "OpenSearch" +description: "Deploying the search cluster that indexes Data Store contents and metadata for the DE." +tags: [deployment, core-services, opensearch, search] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: opensearch-docs + resource: https://opensearch.org/docs/latest/ + title: OpenSearch documentation + author: team:opensearch +--- + +# Role in the deployment + +The search cluster indexes Data Store paths, names, and AVU metadata so the DE +can answer data searches without walking the catalog. It is populated by +`infosquito2`, which consumes iRODS data events from +[RabbitMQ](../01-foundation/rabbitmq.md) and can also be asked for a full +reindex. + +OpenSearch is what new deployments use. Older deployments run Elasticsearch; that +path is kept in [Elasticsearch (legacy)](./elasticsearch.md) for the clusters +still on it. + +# Install + +```bash +ansible-playbook -i /path/to/inventory --tags opensearch kubernetes.yml +``` + +Requires a working storage class — see [storage](../04-kubernetes/storage.md). +Index data is regenerable, but a full reindex of a large zone is measured in +hours, so give it durable volumes rather than node-local scratch. + +# Verify + +```bash +kubectl -n get pods -l app=opensearch +kubectl -n exec -it -- curl -s localhost:9200/_cluster/health?pretty +``` + +On a small cluster expect `yellow` rather than `green` when replicas are +configured but there are not enough nodes to place them. That is a capacity +statement, not a failure; either add nodes or lower the replica count. + +# Populating the index + +1. Confirm `infosquito2` is running and consuming from the DE exchange. +2. Trigger a full reindex through the message bus: + [reindex search](../01-foundation/rabbitmq.md#operations-reindex-search). +3. Restart `infosquito2` and the `search` service so they pick up the request: + + ```bash + kubectl rollout restart deployment infosquito2 search -n + ``` + +A reindex reads the entire catalog and loads both PostgreSQL and the search +cluster. Run it deliberately, not as a first troubleshooting step. + +# Related + +* [Elasticsearch (legacy)](./elasticsearch.md) +* [RabbitMQ](../01-foundation/rabbitmq.md) +* [iRODS integration for the DE](../03-data-store/de-integration.md) diff --git a/docs/deployment/05-core-services/redis-ha.md b/docs/deployment/05-core-services/redis-ha.md new file mode 100644 index 0000000..95acea4 --- /dev/null +++ b/docs/deployment/05-core-services/redis-ha.md @@ -0,0 +1,87 @@ +--- +type: Deployment Procedure +title: "Redis HA" +description: "Deploying the Redis server, Sentinel, and Redis HAProxy used for caching and sessions." +tags: [deployment, core-services, redis, cache, sessions] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# Role in the deployment + +Redis backs DE caching and session state. It is deployed as three replicas with +Sentinel for failover, plus a Redis HAProxy that gives clients one address to +connect to instead of tracking which replica is currently primary. All of it runs +in the same namespace as the DE services. + +The DE reads its Redis configuration from the `Redis` section of the deployment +group variables: host, port, HA service name, password, and database number. + +# Values + +Create a values file that overrides the persistent volume and sets the +credentials. Keep it in your **private** inventory — it contains two secrets. + +```yaml +## replicas for each component +replicas: 3 + +persistentVolume: + enabled: true + ## storage class for redis-ha data + ## use your cluster's replicated class; see the storage document + storageClass: + +## Sentinel +sentinel: + auth: true + authkey: + password: + +## Redis +auth: true +authkey: +redisPassword: +``` + +!!! note "Storage class" + + Older deployments set `storageClass: openebs-hostpath`, which is node-local: + a replica pinned to a lost node loses its data. Use your Longhorn class in a + new deployment — see [storage](../04-kubernetes/storage.md). + +# Deploy Redis and Sentinel + +```bash +helm repo add dandydev https://dandydeveloper.github.io/charts +helm repo update + +helm upgrade --install redis-ha dandydev/redis-ha \ + --namespace --values values.yaml +``` + +# Deploy Redis HAProxy + +The HAProxy in front of Redis reads the configuration and secrets loaded by +[cluster resources](../04-kubernetes/resources.md), so load those first, then: + +```bash +kubectl apply -f resources/deployments/redis-haproxy.yml -n +``` + +# Verify + +```bash +kubectl -n get pods -l app=redis-ha +kubectl -n exec -it redis-ha-server-0 -c redis -- \ + redis-cli -a "$REDIS_PASSWORD" info replication +``` + +Expect one `master` and two `slave` roles. Three replicas all reporting `master` +means Sentinel is not forming a quorum — check that the Sentinel auth key matches +across replicas. + +# Related + +* [Storage](../04-kubernetes/storage.md) +* [Cluster resources](../04-kubernetes/resources.md) diff --git a/docs/deployment/05-core-services/unleash.md b/docs/deployment/05-core-services/unleash.md new file mode 100644 index 0000000..2a3abb7 --- /dev/null +++ b/docs/deployment/05-core-services/unleash.md @@ -0,0 +1,46 @@ +--- +type: Deployment Procedure +title: "Unleash" +description: "Deploying the Unleash feature-flag service the DE reads toggles from." +tags: [deployment, core-services, unleash, feature-flags] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# Role in the deployment + +Unleash holds the DE's feature toggles, including the maintenance flag that puts +the DE into a read-only banner state. The DE reads it through the `Unleash` +section of the deployment configuration: base URL, API path, API token, and the +maintenance flag name. + +# Prerequisites + +* [Unleash database](../02-databases/unleash.md) created. +* The `Unleash` group variables filled in; see + [cluster resources](../04-kubernetes/resources.md). + +# Deploy + +From the [cluster resources](../04-kubernetes/resources.md) checkout, +substituting the namespace the DE runs in: + +```bash +kubectl apply -f resources/deployments/unleash.yml -n +``` + +# Verify + +```bash +kubectl -n get pods -l app=unleash +kubectl -n logs deploy/unleash --tail=50 +``` + +Unleash runs its own schema migrations at startup, so the first start after a +version bump takes longer than usual. A pod that restarts repeatedly on first +boot is normally failing to reach the database. + +# Related + +* [Unleash database](../02-databases/unleash.md) +* [Cluster resources](../04-kubernetes/resources.md) diff --git a/docs/deployment/06-applications/discovery-environment.md b/docs/deployment/06-applications/discovery-environment.md new file mode 100644 index 0000000..29b2ceb --- /dev/null +++ b/docs/deployment/06-applications/discovery-environment.md @@ -0,0 +1,96 @@ +--- +type: Deployment Procedure +title: "Discovery Environment" +description: "Deploying the DE service set and the nginx front end that proxies it." +tags: [deployment, applications, discovery-environment] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# Prerequisites + +Everything in phases 1 through 5. Specifically: + +* [Databases](../02-databases/index.md) created and migrated. +* [iRODS integration](../03-data-store/de-integration.md) complete — specific + queries installed and the `de-irods` account created. +* [Keycloak](../05-core-services/keycloak.md) configured, with every client + secret written into `group_vars/all.yml`. +* Service signing keys generated (`./scripts/generate-secrets.sh`) and the + printed YAML snippet added to the group variables. +* [Cluster resources](../04-kubernetes/resources.md) loaded — in particular the + `harbor-registry-credentials` and `de-nginx-tls` secrets in the DE namespace. +* If HAProxy terminates TLS against a private CA, that CA is in the HAProxy + host's trust bundle (`/etc/ssl/certs/ca-bundle.crt` on RPM-based hosts). + +# Deploy the service set + +The whole DE — Terrain, apps, analyses, metadata, notifications, search, the +Sonora UI, and the VICE backend — is deployed by one tag: + +```bash +ansible-playbook -i /path/to/inventory --tags=deploy-all-services kubernetes.yml +``` + +Watch for anything that does not settle: + +```bash +kubectl get pods -A | grep -Ev 'Running|Completed' +``` + +`ImagePullBackOff` on an image that exists points at the registry pull secret, +not the registry. `CrashLoopBackOff` on a service that starts and immediately +exits is usually a missing configuration key or an unreachable database. + +# The nginx front end + +`de-nginx` proxies the DE's services behind a single hostname. Its manifests are +in the [cluster resources](../04-kubernetes/resources.md) repository as a +kustomize base with per-environment overlays. + +Two values in the base are site-specific and have to match your deployment: + +**`resources/kustomize/de-nginx/base/nginx.conf`** — the server name regex: + +```diff +- server_name ~^[^.]+[.]example[.]org$; ++ server_name ~^[^.]+[.]$; +``` + +The dots are escaped as `[.]` because the value is a regex. `` +written literally would match more hostnames than you intend. + +**`resources/kustomize/de-nginx/base/kustomization.yaml`** — the namespace: + +```diff +- namespace: prod ++ namespace: +``` + +Then apply the overlay and the service definition for your site: + +```bash +kubectl apply -k resources/kustomize/de-nginx/overlays// -n +kubectl apply -f resources/services/.yml -n +``` + +# Verify + +1. `https://de.` loads and redirects to Keycloak for sign-in. +2. Sign-in returns you to the DE with your account name in the UI. +3. The data browser lists your Data Store home directory. + +If sign-in loops back to Keycloak, the client's redirect URIs or web origins are +wrong; see [Keycloak](../05-core-services/keycloak.md#clients). + +# Next + +* [VICE](./vice.md) — interactive analyses +* [User Portal](./user-portal.md) — account management +* [Bootstrap](../07-post-install/bootstrap.md) — first administrator and apps diff --git a/docs/deployment/06-applications/index.md b/docs/deployment/06-applications/index.md new file mode 100644 index 0000000..be1e3c1 --- /dev/null +++ b/docs/deployment/06-applications/index.md @@ -0,0 +1,11 @@ +# Phase 6: applications + +The user-facing platform, deployed once every dependency below it is verified. + +* [Discovery Environment](discovery-environment.md) - the DE service set and its nginx front end +* [VICE](vice.md) - the namespace, isolation, and credentials interactive analyses need +* [User Portal](user-portal.md) - account creation and access requests + +# Next + +* [Phase 7: post-install](../07-post-install/) diff --git a/docs/deployment/06-applications/user-portal.md b/docs/deployment/06-applications/user-portal.md new file mode 100644 index 0000000..7fb165a --- /dev/null +++ b/docs/deployment/06-applications/user-portal.md @@ -0,0 +1,91 @@ +--- +type: Deployment Procedure +title: "User Portal" +description: "Deploying the User Portal and creating the LDAP, iRODS, and database accounts it needs." +tags: [deployment, applications, user-portal, accounts] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: portal2 + resource: https://gitlab.com/cyverse/portal2 + title: CyVerse User Portal (portal2) + author: team:cyverse + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# Role in the deployment + +The User Portal is where accounts are created and access to services is +requested and granted. It writes to three systems, which is why it needs an +identity in each of them: + +| System | Identity | Why | +|--------|----------|-----| +| LDAP | `uid=portal,ou=People,` | Creates and updates user entries | +| iRODS | `portal` (`rodsadmin`) | Creates home collections for new accounts | +| PostgreSQL | `portal_db_reader` | Owns the portal database | + +# Prerequisites + +* [OpenLDAP](../05-core-services/openldap.md) running, with the `portal` service + account created and added to `de_admins`. +* [Portal database](../02-databases/portal.md) created, restored, and seeded. +* [Keycloak](../05-core-services/keycloak.md) configured with the + `portal-` client, and its ID and secret in the group variables. + +# iRODS account + +The portal creates users' home collections, so it needs an iRODS admin account of +its own — not the DE's: + +```bash +# as the irods service account +iadmin mkuser portal rodsadmin +iadmin moduser portal password '' +``` + +# Images + +The portal deployment pulls two images: + +| Image | Notes | +|-------|-------| +| `nginx:1.20-alpine` | Static front end; pull through your own registry rather than Docker Hub to avoid rate limits | +| `/portal:` | The portal application itself | + +Build the portal image from +[portal2](https://gitlab.com/cyverse/portal2)[^portal2] and push it to your own +[Harbor](../04-kubernetes/harbor.md) project. Older notes reference a personal +Docker Hub image; do not deploy from one — an image nobody at your site controls +is an unpinned dependency in your authentication path. + +# Deploy + +```bash +kubectl create ns user-portal +kubectl apply -k portal/user-portal/base -n user-portal +``` + +The portal is also deployed as part of the `deploy-all-services` tag; deploy it +by hand only when you are iterating on the portal specifically. + +# Verify + +1. `https://user.` loads. +2. Sign-in redirects to Keycloak and back. +3. A test account request appears in the admin panel. + +Account creation exercises all three identities above. A request that is accepted +but never produces a usable account is normally the LDAP or iRODS credential, not +the portal. + +# Next + +* [Bootstrap the first administrator](../07-post-install/bootstrap.md) +* [User Portal administration](../../operations/user-portal.md) + +[^portal2]: CyVerse User Portal (portal2) diff --git a/docs/deployment/06-applications/vice.md b/docs/deployment/06-applications/vice.md new file mode 100644 index 0000000..89ede16 --- /dev/null +++ b/docs/deployment/06-applications/vice.md @@ -0,0 +1,139 @@ +--- +type: Deployment Procedure +title: "VICE" +description: "Preparing the vice-apps namespace, service accounts, network policies, and secrets for interactive apps." +tags: [deployment, applications, vice, interactive] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# What VICE needs + +VICE (the Visual Interactive Computing Environment) runs user-supplied +interactive containers — JupyterLab, RStudio, Shiny, Cloud Shell — inside the +cluster, each with its own ingress and its own mount of the user's Data Store +home. That means it needs more isolation than any other part of the deployment: +a dedicated namespace, dedicated service accounts, and a network policy that +keeps user containers away from cluster infrastructure. + +Interactive analyses run on the workers in the `k8s_vice_workers` inventory +group, kept separate from the DE service set so a runaway analysis cannot starve +the platform. + +# Namespace and image pulls + +```bash +kubectl create ns vice-apps +``` + +Interactive app images come from [Harbor](../04-kubernetes/harbor.md), so the +namespace needs a pull secret: + +```bash +kubectl create secret generic vice-image-pull-secret \ + --from-file=.dockerconfigjson= \ + --type=kubernetes.io/dockerconfigjson -n vice-apps +``` + +Point `--from-file` at a Docker config containing **only** the registry +credentials VICE needs. A personal `~/.docker/config.json` often holds +credentials for several registries, and everything in the file becomes readable +to anything that can read the secret. + +# Service accounts, bindings, and roles + +From the [cluster resources](../04-kubernetes/resources.md) checkout. Check the +namespace in each manifest before applying if you are not deploying into the +default environment: + +```bash +kubectl apply -f resources/serviceaccounts/app-exposer.yml +kubectl apply -f resources/serviceaccounts/vice-app-runner.yml +kubectl apply -f resources/clusterrolebindings/app-exposer.yml +kubectl apply -f resources/roles/vice-apps.yml +``` + +`app-exposer` creates and tears down per-analysis deployments, services, and +ingresses; `vice-app-runner` is the identity the user's container itself runs as. + +# Network policy + +`resources/networkpolicies/vice-apps.yml` is what stops interactive containers +from reaching cluster infrastructure. It allows egress broadly and then subtracts +the addresses user containers must not reach — the control plane and the worker +nodes themselves: + +```yaml + except: + - # control plane + - /32 # one entry per node +``` + +Add an entry for **every** node in the cluster, control plane and workers alike, +and revisit the list whenever a node is added. A node missing from the exception +list is a node that user containers can reach directly. + +```bash +kubectl apply -f resources/networkpolicies/vice-apps.yml +``` + +# Data transfer credentials + +Analyses stage data in and out of the Data Store with `porklock`, which reads its +configuration from a secret in the `vice-apps` namespace. Create +`irods-config.properties`: + +```properties +porklock.irods-home=//home +porklock.irods-user=de-irods +porklock.irods-pass= +porklock.irods-host= +porklock.irods-port=1247 +porklock.irods-zone= +porklock.irods-resc= +``` + +```bash +kubectl -n vice-apps create secret generic porklock-config \ + --from-file=irods-config.properties +``` + +Use the `de-irods` credentials from +[iRODS integration](../03-data-store/de-integration.md), then delete the local +file — it holds an iRODS admin password in plain text. + +# Ingress + +Interactive analyses get per-analysis hostnames under `*.vice.`, +served today through ingress-nginx. See [ingress](../04-kubernetes/ingress.md) +for the controller and [cert-manager](../04-kubernetes/cert-manager.md) for the +wildcard certificate those hostnames need. + +# Apply configuration changes + +After changing VICE configuration or secrets, restart the services that read +them: + +```bash +kubectl rollout restart deployment \ + app-exposer templeton-incremental templeton-periodic -n +``` + +# Before analyses will launch + +A VICE **operator** has to be registered through the DE admin UI, which is a +post-install step rather than a deployment step: +[register a VICE operator](../07-post-install/bootstrap.md#register-a-vice-operator). + +# Related + +* [iRODS CSI driver](../05-core-services/irods-csi-driver.md) — how home + directories are mounted into analysis pods +* [Namespaces](../../architecture/namespaces.md) +* [Discovery Environment](./discovery-environment.md) diff --git a/docs/deployment/07-post-install/bootstrap.md b/docs/deployment/07-post-install/bootstrap.md new file mode 100644 index 0000000..ce6618f --- /dev/null +++ b/docs/deployment/07-post-install/bootstrap.md @@ -0,0 +1,134 @@ +--- +type: Playbook +title: "Bootstrap" +description: "Creating the first administrator, registering a VICE operator, and importing a starting set of apps." +tags: [deployment, post-install, bootstrap, vice, apps] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: uv + resource: https://docs.astral.sh/uv/ + title: uv, the Python package and project manager + author: team:astral +--- + +# Where this fits + +At this point every service is deployed but the deployment has no users, no +registered VICE operator, and no apps. Three steps make it usable. + +# Bootstrap portal administrator + +```bash +ansible-playbook -i /path/to/inventory bootstrap_portal_admin.yml +``` + +This creates the `-bootstrap` account — the administrator you use for every +step below. + +!!! warning "Run this from the right host" + + The playbook does two things that constrain where it can run: it + `kubectl port-forward`s to OpenLDAP and portal-conductor, and it connects to + the portal database directly with `psql`. So the host needs cluster access + **and** a `pg_hba.conf` rule of its own. + + Getting this far means the port-forwarding half already works. The database + half is the one that fails — see + [PostgreSQL access control](../01-foundation/postgresql.md#access-control). + +Sign in at `https://de.` as `-bootstrap` and confirm the +account has the admin navigation before continuing. + +# Register a VICE operator + +Interactive analyses do not launch until an operator exists. This is a one-time +registration through the DE UI. + +1. Sign in to `https://de.` as `-bootstrap`. +2. Open the left navigation from the hamburger menu in the upper left, then + **Admin → VICE**, and select the **Operators** tab. +3. Click **+New** and set: + + | Field | Value | + |-------|-------| + | Name | `prod` | + | URL | `http://vice-operator.vice-apps:10000` | + | Public base URL | `https://vice.` | + | Priority | `0` | + +4. Click **Register operator**. + +The operator URL is plain `http` on purpose: it is an in-cluster service address +that never leaves the cluster network. The **public** base URL is the HTTPS +hostname users' browsers connect to. + +# Import a starting set of apps + +`scripts/appei` in the deployment repository exports and imports apps and tools +between DE deployments, and manages its own dependencies with +[uv](https://docs.astral.sh/uv/).[^uv] From that directory: + +```bash +uv run appei login --server de. --username -bootstrap +uv run appei import --server de. -i de-word-count.json --publish +uv run appei import --server de. -i cloudshell.json --featured +uv run appei import --server de. -i portal-delete-user.json +``` + +The visibility each flag produces: + +| App | Flag | Result | +|-----|------|--------| +| DE Word Count | `--publish` | Public, not featured | +| Cloud Shell | `--featured` | Featured, and therefore public | +| portal-delete-user | none | Private to `-bootstrap` | + +`--featured` implies public. There is no way to feature a private app, and +attempting it is usually a sign the wrong flag was chosen. + +## portal-delete-user + +This administrative app needs a configuration file attached before it will run. +Generate the file from your group variables: + +```bash +ansible-playbook -i /path/to/inventory portal_delete_user_config.yml +``` + +That writes `portal-delete-user.json` into the `ansible` directory. Upload it to +the bootstrap user's home directory in the Data Store, keeping the filename. + +!!! danger "Delete the local copy afterwards" + + The generated file is populated from the group variables and contains + credentials. Remove it from your working copy as soon as it is uploaded — it + sits in a git working tree and is easy to commit by accident. + +Then attach it in the DE: + +1. **Apps** listing → select **Apps Under Development** from the dropdown. +2. On the `portal-delete-user` row, open the three-dot menu → **Edit App**. +3. Go to the **Parameters** page of the Edit App wizard. +4. Edit the greyed-out **Config File** parameter (the pencil icon on the right). +5. Near the top of the page that opens is another **Config File** box. If it + holds a value, clear it with the **X**. +6. Click **Browse**, select the uploaded file, and confirm with **OK**. +7. Click **Done**, then **Save** in the top right of the app editor. + +Step 7 is not optional — leaving the editor without saving discards the +attachment silently. + +If the app or tool imported with the wrong visibility, see +[troubleshooting](./troubleshooting.md#repairing-the-portal-delete-user-import). + +# Next + +[Verification](./verification.md) — confirm the deployment end to end. + +[^uv]: uv, the Python package and project manager diff --git a/docs/deployment/07-post-install/index.md b/docs/deployment/07-post-install/index.md new file mode 100644 index 0000000..b3b0bae --- /dev/null +++ b/docs/deployment/07-post-install/index.md @@ -0,0 +1,12 @@ +# Phase 7: post-install + +Everything is deployed; nothing is usable yet. These three documents create the +first administrator, prove the deployment works, and fix what commonly breaks. + +* [Bootstrap](bootstrap.md) - portal administrator, VICE operator, and a starting set of apps +* [Verification](verification.md) - end-to-end checks, from cluster health to an interactive analysis +* [Troubleshooting](troubleshooting.md) - fixes for problems seen during first deployments + +# Next + +* [operations/](../../operations/) - running the deployment day to day diff --git a/docs/deployment/07-post-install/troubleshooting.md b/docs/deployment/07-post-install/troubleshooting.md new file mode 100644 index 0000000..fe97fa4 --- /dev/null +++ b/docs/deployment/07-post-install/troubleshooting.md @@ -0,0 +1,138 @@ +--- +type: Playbook +title: "Troubleshooting a new deployment" +description: "Fixes for the problems that show up during and just after a first CyVerse deployment." +tags: [deployment, post-install, troubleshooting] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 +--- + +# Migrations were skipped + +**Symptom.** DE services start but fail on queries against columns or tables that +should exist. + +**Cause.** The DE database expects `standard_conforming_strings = off`. When it is +left on, some migrations do not apply. + +**Fix.** Correct the setting in `postgresql.conf` +([PostgreSQL tuning](../01-foundation/postgresql.md#tuning)), restart PostgreSQL, +then re-run the migrations: + +```bash +ansible-playbook -i /path/to/inventory --tags=update-databases kubernetes.yml +``` + +# OpenLDAP is missing the community group + +**Symptom.** Group-based entitlements do not resolve for community data, or a +playbook expects a group that is not in the directory. + +**Fix.** The deployment repository has an idempotent playbook for it: + +```bash +ansible-playbook -i /path/to/inventory openldap_community_group.yml +``` + +Safe to run when you are not sure whether you need it. + +# NATS will not reinstall + +**Symptom.** Reinstalling NATS fails because resources already exist. + +**Cause.** Helm still holds a release record for the previous install. + +**Fix.** + +```bash +helm -n prod uninstall nats +``` + +Then re-run the tag that installs it; see [NATS](../05-core-services/nats.md). + +# iRODS logs nothing + +**Symptom.** `/var/log/irods/irods.log` is missing or empty even though iRODS is +serving requests. + +**Cause.** The rsyslog snippet is in the wrong directory. rsyslog includes +`/etc/rsyslog.d/*.conf`; a file in `/etc/rsyslog/` is silently ignored. + +**Fix.** Move the file to `/etc/rsyslog.d/00-irods.conf` and restart rsyslog. See +[iRODS provider](../03-data-store/irods-provider.md#configure-logging-first). + +# k0sctl cannot write the kubeconfig + +**Symptom.** `k0sctl apply` fails writing the kubeconfig, or a directory named +`config` appears where the file should be. + +**Cause.** Creating the *basename* of `$KUBECONFIG` instead of its *dirname*. + +**Fix.** + +```bash +mkdir -p "$(dirname "$KUBECONFIG")" +``` + +# Pods cannot reach PostgreSQL + +**Symptom.** Services in the cluster fail to connect to the database while `psql` +from an admin host works. + +**Cause.** The pod CIDR is not in `pg_hba.conf`. It cannot be added before the +cluster exists, so it is easy to skip. + +**Fix.** [Add the pod CIDR](../01-foundation/postgresql.md#access-control) and +restart PostgreSQL. + +# ImagePullBackOff on an image that exists + +**Cause.** A missing or wrong registry pull secret in the namespace, rather than a +registry problem — `harbor-registry-credentials` for DE services, +`vice-image-pull-secret` for interactive apps. + +**Fix.** See [Harbor](../04-kubernetes/harbor.md) and +[VICE](../06-applications/vice.md). + +# Repairing the portal-delete-user import + +**Symptom.** The `portal-delete-user` app or tool imported with the wrong +visibility, or has to be re-imported cleanly. + +The app and tool are deleted **by ID**, and the simplest way to learn both IDs is +to re-run the import: it is a no-op when they already exist, and it prints the +IDs. From `scripts/appei` in the deployment repository: + +```bash +uv run appei login --server de. --username -bootstrap + +# no-op re-import; prints the tool and app IDs +uv run appei import --server de. -i portal-delete-user.json + +# hard delete the app, then the tool +uv run appei shred-app --server de. --id +uv run appei delete-tool --server de. --id + +# re-import, private this time +uv run appei import --server de. -i portal-delete-user.json +``` + +Both should now be visible to `-bootstrap` and private to that account. + +`shred-app` is a permanent hard delete, not a soft delete. Confirm the app ID +before running it. + +Then re-attach the configuration file: +[portal-delete-user](./bootstrap.md#portal-delete-user). + +# Related + +* [Verification](./verification.md) +* [FAQ](../../operations/faq.md) +* [Deploying from scratch](../from-scratch.md) diff --git a/docs/deployment/07-post-install/verification.md b/docs/deployment/07-post-install/verification.md new file mode 100644 index 0000000..1f374db --- /dev/null +++ b/docs/deployment/07-post-install/verification.md @@ -0,0 +1,110 @@ +--- +type: Playbook +title: "Verification" +description: "The end-to-end checks that tell you a fresh CyVerse deployment actually works." +tags: [deployment, post-install, verification, testing] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# How to use this + +Work down the list. Each check depends on the ones above it, so a failure is +usually caused by the last thing that passed rather than by the check that broke. + +# Cluster + +```bash +kubectl get nodes -o wide +kubectl get pods -A | grep -Ev 'Running|Completed' +kubectl get pvc -A +kubectl get certificates -A +``` + +* [ ] Every node `Ready`. +* [ ] No pod outside `Running` or `Completed`. +* [ ] No PVC `Pending` — a pending claim means the storage class cannot satisfy + it; see [storage](../04-kubernetes/storage.md). +* [ ] Every certificate `True` / `Ready`; see + [cert-manager](../04-kubernetes/cert-manager.md). + +# Foundation + +* [ ] `psql` connects from an admin host and from a pod. +* [ ] The RabbitMQ management UI is reachable from an admin host and **not** from + outside. +* [ ] `iinit` and `ils` succeed as an ordinary user. +* [ ] `ils //home/public` succeeds as `anonymous`, with no password. + +# Authentication + +* [ ] `https://keycloak.` serves a valid certificate. +* [ ] The realm's LDAP federation **Test connection** and **Test authentication** + both pass. +* [ ] A synced user appears under **Users** in the realm. +* [ ] A token issued for the `de-` client carries both `name` and + `entitlement` claims — this is what the client scope mappers exist for; see + [Keycloak](../05-core-services/keycloak.md#client-scope-mappers). + +# Discovery Environment + +* [ ] `https://de.` loads and redirects to Keycloak. +* [ ] Sign-in returns to the DE with your account name shown. +* [ ] The data browser lists your Data Store home directory. +* [ ] Upload a small file through the DE, then confirm it with `ils` — this + exercises the DE, iRODS, and the transfer path together. +* [ ] Search finds the uploaded file after a few moments. If not, follow the + event chain in + [iRODS integration](../03-data-store/de-integration.md#event-flow). + +# Batch analysis + +* [ ] Run **DE Word Count** on a small text file. +* [ ] The analysis completes and writes an output folder to the Data Store. +* [ ] A completion notification appears in the DE. + +A batch analysis touches Argo (or HTCondor), the transfer credentials, the message +bus, and notifications, which is why it is the single most informative check here. + +# Interactive analysis + +* [ ] Launch **Cloud Shell**. +* [ ] The analysis URL under `*.vice.` resolves and serves a valid + certificate. +* [ ] The shell opens and the user's Data Store home is mounted. +* [ ] Terminating the analysis removes the deployment, service, and ingress: + + ```bash + kubectl -n vice-apps get deploy,svc,ingress + ``` + +If the URL resolves but the certificate is wrong, the wildcard certificate is the +suspect; if the shell opens without the home directory, look at the +[iRODS CSI driver](../05-core-services/irods-csi-driver.md). + +# User Portal + +* [ ] `https://user.` loads and signs in. +* [ ] A test account request appears in the admin panel. +* [ ] Approving it produces a working account: it can sign in to the DE and has + an iRODS home collection. +* [ ] Account verification mail arrives; see + [mail](../05-core-services/mail.md). + +# Operational readiness + +Not required for the deployment to work, and required before anyone relies on it: + +* [ ] Database backups scheduled and a restore tested. +* [ ] iRODS vault backup or replication in place. +* [ ] Certificate renewal verified against the staging issuer. +* [ ] Log rotation confirmed for iRODS + ([provider](../03-data-store/irods-provider.md#configure-logging-first)), + and index rollover confirmed if [Jaeger](../05-core-services/jaeger.md) is + deployed. +* [ ] Deployment secrets stored only in the private inventory repository. + +# Related + +* [Troubleshooting](./troubleshooting.md) +* [FAQ](../../operations/faq.md) diff --git a/docs/deployment/from-scratch.md b/docs/deployment/from-scratch.md new file mode 100644 index 0000000..0a95809 --- /dev/null +++ b/docs/deployment/from-scratch.md @@ -0,0 +1,582 @@ +--- +type: Playbook +title: "Deploying CyVerse from scratch" +description: "End-to-end walkthrough of a two-node CyVerse pilot deployment, from bare servers to a Discovery Environment running VICE analyses." +tags: [deployment, runbook, pilot, irods, kubernetes, discovery-environment] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: pilot-record + resource: ../references/pilot-deployment-record.md + title: Pilot CyVerse deployment record + author: process:cyverse-devops + last_modified: 2026-07-29 + - id: irods-install + resource: https://docs.irods.org/4.3.3/getting_started/installation/ + title: iRODS 4.3.3 installation guide + author: team:irods-consortium + - id: irods-packages + resource: https://packages.irods.org/ + title: iRODS package repository setup + author: team:irods-consortium + - id: ds-collection + resource: https://github.com/cyverse/ds-collection + title: CyVerse Data Store collection (playbooks and iRODS policy) + author: team:cyverse-devops +--- + +# Scope + +This runbook builds a complete, self-contained CyVerse deployment on **two +servers** — the shape used for the most recent pilot. It is the narrative +companion to the per-component procedures under +[deployment](./index.md); every phase below links to the document that +carries the full detail. + +A deployment of this size supports a working Discovery Environment (DE) with +VICE analyses, a single-provider iRODS zone, and one PostgreSQL instance shared +by the Data Store and the DE. Scaling out means adding worker nodes and +splitting roles onto more hosts; the order of operations does not change. + +!!! info "Placeholders and secrets" + + Every site-specific value in this document is a placeholder. Substitute + your own and keep the real values in a **private** inventory repository — + never in this documentation and never in a public repo. + + | Placeholder | Meaning | Example | + |-------------|---------|---------| + | `` | DNS domain the deployment is served under | `example.org` | + | `` | Short site name used in realm, client, and account names | `pilot` | + | `core-1` | Node hosting HAProxy, PostgreSQL, RabbitMQ, iRODS, k8s control plane, DE services | `core-1.` | + | `analysis-1` | Node hosting VICE analyses | `analysis-1.` | + | `` | iRODS zone name | `pilotZone` | + | `` | LDAP base DN | `dc=example,dc=org` | + | `` | Your private inventory and group_vars repo | `-deployment` | + | `` | A value you generate per install, never reuse | output of `openssl rand -base64 36` | + + Generate every password, salt, zone key, and negotiation key at install + time, for example with `openssl rand -base64 36` (passwords) or + `openssl rand -hex 16` (alphanumeric keys and salts). Store them in the + private inventory, and let Ansible template them into place rather than + typing them into shells where they land in history. + + Deployment secrets that Ansible generates for you — the GPG and PEM files + used by Terrain, apps, and iplant-groups — belong in the **private** + inventory repository, not in this one. + +# Node layout + +Two servers, split by workload rather than by service: + +| Node | Runs | +|------|------| +| `core-1` | HAProxy, PostgreSQL, RabbitMQ, iRODS provider, Kubernetes control plane, DE services (everything except analyses) | +| `analysis-1` | Kubernetes worker dedicated to VICE analyses | + +Sizing, per-component reservations, and the dependency graph are in +[component inventory](../architecture/component-inventory.md). The ports that +must be open between the two nodes, to administrators, and to the public +internet are in +[network requirements](../architecture/network-requirements.md). + +Reading the dependency graph from the bottom up gives the order the rest of +this document follows: + +```mermaid +graph TD + NET[Phase 0: network and firewalls] --> FOUND[Phase 1: HAProxy, PostgreSQL, RabbitMQ] + FOUND --> DB[Phase 2: service databases] + DB --> IRODS[Phase 3: iRODS provider and DE integration] + IRODS --> K8S[Phase 4: Kubernetes and cluster add-ons] + K8S --> CORE[Phase 5: LDAP, Keycloak, Grouper, OpenSearch, NATS, CSI driver] + CORE --> APPS[Phase 6: DE services, VICE, User Portal] + APPS --> POST[Phase 7: bootstrap admin, VICE operator, apps] +``` + +# Phase 0: network and firewalls + +Do this first. Every later phase fails in a confusing way if a port is closed. + +1. Confirm host firewalls on both nodes and any institutional firewall allow + the traffic listed in + [network requirements](../architecture/network-requirements.md). +2. Confirm `80/tcp` and `443/tcp` on `core-1` are reachable from everywhere + users and VICE clients connect from. +3. Confirm iRODS (`1247/tcp` plus the `20000-20199` data range) is reachable + from `analysis-1` and from any host that will move data in or out. +4. Confirm outbound HTTPS to your DNS provider's API is allowed, so + cert-manager can complete Let's Encrypt DNS-01 challenges. This is optional + but strongly recommended; without it, certificate issuance is manual. +5. Create the DNS records you will need: `de.`, + `keycloak.`, `user.`, `vice.`, and a + wildcard `*.vice.` for interactive apps. + +# Phase 1: foundation services on `core-1` + +Full procedures: [HAProxy](./01-foundation/haproxy.md), +[PostgreSQL](./01-foundation/postgresql.md), +[RabbitMQ](./01-foundation/rabbitmq.md). + +## 1.1 HAProxy + +Install HAProxy on `core-1`. It terminates public HTTPS on `443/tcp` and +forwards to the node ports Traefik listens on inside the cluster. Its +configuration depends on the Traefik node ports chosen in Phase 4, so install +the package now and configure it with the `haproxy` playbook tag later in that +phase. + +## 1.2 PostgreSQL + +One PostgreSQL instance backs both the iRODS catalog (iCAT) and every DE +service database. + +1. Disable transparent huge pages: append `transparent_hugepage=never` to + `GRUB_CMDLINE_LINUX_DEFAULT` in `/etc/default/grub`, run `update-grub`, and + reboot. +2. Set kernel parameters persistently (`/etc/sysctl.d/`): + `vm.nr_hugepages = 0.6 * (memory_available_to_postgres_in_kiB / 2048)` and + `vm.swappiness = 5`. +3. Install `postgresql`, `postgresql-client`, and `python3-psycopg2`. +4. Apply the tuned `postgresql.conf` settings from + [PostgreSQL](./01-foundation/postgresql.md#tuning) — they derive from the + cores and memory you reserved for the database, so compute them from your + own sizing rather than copying literals. +5. Restart PostgreSQL, then follow the iRODS project's instructions for + preparing PostgreSQL for iRODS.[^irods-install] +6. Create a role for the DE with `SELECT` on all tables in the iCAT database, + with a generated password. + +!!! warning "`standard_conforming_strings`" + + The DE database expects `standard_conforming_strings = off`. If it is left + on, some migrations do not apply; see + [troubleshooting](./07-post-install/troubleshooting.md). + +## 1.3 RabbitMQ + +1. Install `rabbitmq-server`, enable the `rabbitmq_management` plugin, restart. +2. Create an administrator account with a generated password, grant it full + configure/write/read permissions, and tag it `administrator`. +3. **Delete the default `guest` account.** +4. Create the vhost `/data-store` and grant the administrator full + configure/write/read on it. +5. Create a separate account for iRODS with a generated password and full + configure/write/read on `/data-store`. +6. Create a `topic` exchange named `irods` on `/data-store`. + +The DE's own RabbitMQ objects are created later, in Phase 6, by the +`rabbitmq_configure.yml` playbook. + +# Phase 2: service databases + +Full procedures: [databases](./02-databases/index.md). + +The iCAT database itself is created by the iRODS installer in Phase 3. The DE's +databases are created and migrated by Ansible in Phase 4 (the +`setup-databases` and `update-databases` tags), so at this point you only need +to make PostgreSQL reachable: + +1. Set a password for the `postgres` role. +2. Edit `pg_hba.conf` to allow connections from your admin hosts and from both + nodes, using method `scram-sha-256`. +3. Restart PostgreSQL. + +The pod network cannot be allowed yet — its CIDR does not exist until the +cluster is created. Phase 4 comes back to `pg_hba.conf` for that. + +# Phase 3: iRODS provider and DE integration + +Full procedures: [iRODS provider](./03-data-store/irods-provider.md), +[DE integration](./03-data-store/de-integration.md). + +## 3.1 Logging first + +Create the rsyslog configuration **before** installing iRODS, so the first +server start is captured. Write `/etc/rsyslog.d/00-irods.conf`: + +``` +$FileCreateMode 0644 +$DirCreateMode 0755 +$Umask 0000 +$template irods_format,"%msg%\n" +:programname,startswith,"irodsServer" /var/log/irods/irods.log;irods_format +& stop +:programname,startswith,"irodsDelayServer" /var/log/irods/irods.log;irods_format +& stop +:programname,startswith,"irodsAgent" /var/log/irods/irods.log;irods_format +& stop +``` + +Then add log rotation in `/etc/logrotate.d/irods`: + +``` +/var/log/irods/irods.log { + weekly + rotate 26 + copytruncate + delaycompress + compress + dateext + notifempty + missingok + su root root +} +``` + +## 3.2 Install the iRODS 4.3.3 catalog provider + +1. Set TCP keepalive to 120 seconds with `sysctl`. +2. Install `python-is-python3` and `python3-pika`. +3. Add the iRODS apt repository[^irods-packages] and pin `irods-*` to + `4.3.3` so an unattended upgrade cannot move the catalog provider. +4. Run the iRODS setup script[^irods-install] with the answers in + [iRODS provider](./03-data-store/irods-provider.md#setup-answers) — + service account `irods`, role `provider`, ODBC driver + `PostgreSQL Unicode`, catalog on `localhost:5432`, zone ``, + port `1247`, data port range `20000-20199`. + + Three answers deserve care: + + * **Password salt** — a generated alphanumeric string. Never leave it + empty; an empty salt makes stored passwords recoverable. + * **Zone key** and **negotiation key** — generated alphanumeric strings; + the zone key must be under 40 characters. + * **Default resource name** — anything but `demoResc`, with its vault + directory at the root of the filesystem that will hold the data. + +## 3.3 Install CyVerse policy + +From the [Data Store collection](https://github.com/cyverse/ds-collection) +branch that matches your site,[^ds-collection] as the `irods` service account: + +1. Copy `playbooks/files/irods/var/lib/irods/msiExecCmd_bin/*` into + `/var/lib/irods/msiExecCmd_bin/` and make them executable. +2. Render `playbooks/templates/irods/etc/irods/cyverse-env.re.j2` to + `/etc/irods/cyverse-env.re`, setting `cyverse_RE_HOST` to the FQDN of + `core-1` and `cyverse_ZONE` to ``. +3. Copy `playbooks/files/etc/irods/*` into `/etc/irods/`. + +Then edit `/etc/irods/server_config.json`: + +```json +{ + "advanced_settings": { + "number_of_concurrent_delay_rule_executors": 12 + }, + "environment_variables": { + "IRODS_AMQP_URI": "amqp://:@localhost:5672/%2Fdata-store" + }, + "plugin_configuration": { + "rule_engines": [ + { "re_rulebase_set": ["cve", "cyverse_core", "core"] } + ] + } +} +``` + +The rule base order is significant: `cve` overrides `cyverse_core`, which +overrides `core`. + +Enable the service so it starts at boot, and start it. + +## 3.4 Runtime initialization + +As the `irods` service account, with `` substituted throughout: + +1. Create the `rodsadmin` group and add `rods` to it. Remove the collections + `//home/rodsadmin` and + `//trash/home/rodsadmin` that group creation leaves behind. +2. Remove `//trash/home/public`. +3. Ensure the predefined collections `/`, `//home`, + `//home/public`, `//home/rods`, + `//trash`, `//trash/home`, and + `//trash/home/rods` all carry time-based (version 1) UUIDs. +4. Grant `rodsadmin` `write` on `/`, `//home`, + `//trash`, and `//trash/home`; grant it `own` on + `//home/rods` and `//trash/home/rods`. +5. Create the `anonymous` `rodsuser` **without** a password and grant it + `read` on `/` and `//home`. This is what makes + public data public. + +## 3.5 Prepare iRODS for the DE + +1. Install the DE's specific queries from + `playbooks/files/irods/specific-queries` in the Data Store collection. Each + file name is the query alias and the file contents are the query: + + ```bash + iadmin asq "$(cat IPCCountCollectionsUnderPath.sql)" IPCCountCollectionsUnderPath + ``` + +2. Create the DE's iRODS admin account: + + ```bash + iadmin mkuser de-irods rodsadmin + iadmin moduser de-irods password '' + iadmin atg rodsadmin de-irods + ``` + +# Phase 4: Kubernetes and cluster add-ons + +Full procedures: [cluster](./04-kubernetes/cluster.md), +[resources](./04-kubernetes/resources.md), +[cert-manager](./04-kubernetes/cert-manager.md), +[ingress](./04-kubernetes/ingress.md), [storage](./04-kubernetes/storage.md), +[Harbor](./04-kubernetes/harbor.md), [Argo](./04-kubernetes/argo.md). + +## 4.1 Inventory and group variables + +Start from the example inventory in `ansible/example/inventory` in the +deployment playbook repository and copy it into your private +``: + +| Inventory group | Contents for a two-node pilot | +|-----------------|-------------------------------| +| `01_condor` | empty — HTCondor is not used in this deployment | +| `02_dbms` | the database host; both groups may name the same host | +| `03_gocd` | empty — no GoCD in this deployment | +| `04_haproxy` | `core-1` | +| `05_k8s` | `core-1` in `k8s_api_proxy`, `k8s_controllers`, and `k8s_de_workers`; `analysis-1` in `k8s_vice_workers` | + +`group_vars/all.yml` in the example inventory is organized in phases, because +some variables can only be filled in from values produced by earlier phases +(Keycloak client secrets, for example). Fill in phases 1 through 4 now. + +## 4.2 Databases, node prep, HAProxy + +Run from the `ansible` directory of the deployment playbook repository, with +`-i` pointing at your private inventory: + +```bash +ansible-playbook -i /path/to/inventory --tags setup-databases,update-databases kubernetes.yml +ansible-playbook -i /path/to/inventory --tags prep-nodes kubernetes.yml +ansible-playbook -i /path/to/inventory --tags haproxy kubernetes.yml +``` + +The PostgreSQL installation role is skipped here on purpose: the Data Store's +DBMS from Phase 1 is being reused. + +## 4.3 Create the k0s cluster + +Write a `k0sctl.yaml` listing both hosts and the cluster settings — see +[cluster](./04-kubernetes/cluster.md#k0sctlyaml) for a sanitized example — then: + +```bash +export K0S_SSH_USER= +export K0S_SSH_KEY_PATH=/path/to/private-key +export KUBECONFIG="$HOME/.kube/config" +mkdir -p "$(dirname "$KUBECONFIG")" +k0sctl apply --config /path/to/k0sctl.yaml +``` + +Because `core-1` is both control plane and DE worker, remove the control-plane +taint if k0s applied one. On the control node: + +```bash +k0s kubectl taint node core-1 node-role.kubernetes.io/control-plane:NoSchedule- +``` + +If the node was never tainted, the command reports that there is nothing to +remove; that is the expected outcome, not an error to chase. + +## 4.4 Cluster add-ons, in order + +```bash +ansible-playbook -i /path/to/inventory --tags cert-manager kubernetes.yml +ansible-playbook -i /path/to/inventory --tags cert-issuers kubernetes.yml +ansible-playbook -i /path/to/inventory --tags argo kubernetes.yml +ansible-playbook -i /path/to/inventory argo_resources.yml +ansible-playbook -i /path/to/inventory --tags ingress-nginx kubernetes.yml +ansible-playbook -i /path/to/inventory --tags traefik kubernetes.yml +ansible-playbook -i /path/to/inventory --tags longhorn kubernetes.yml +ansible-playbook -i /path/to/inventory --tags harbor kubernetes.yml +``` + +!!! note "ingress-nginx is transitional" + + Traefik is the ingress the deployment is standardizing on. ingress-nginx is + still installed for VICE ingresses and will be removed once that migration + completes. See [ingress](./04-kubernetes/ingress.md). + +## 4.5 Let the pods reach PostgreSQL + +The cluster now has a pod network, so add it to `pg_hba.conf`: + +```bash +kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' && echo +``` + +For each distinct CIDR, add a line to `pg_hba.conf` (in a Debian packaged +PostgreSQL 16 install, `/etc/postgresql/16/main/pg_hba.conf`): + +``` +host all all scram-sha-256 +``` + +Restart PostgreSQL afterwards. In a single-node-per-role pilot the CIDRs are +usually identical, but check rather than assume. + +# Phase 5: core services + +Full procedures: [OpenLDAP](./05-core-services/openldap.md), +[Keycloak](./05-core-services/keycloak.md), +[Grouper](./05-core-services/grouper.md), +[OpenSearch](./05-core-services/opensearch.md), +[NATS](./05-core-services/nats.md), +[iRODS CSI driver](./05-core-services/irods-csi-driver.md). + +## 5.1 DE prerequisites and directory services + +```bash +ansible-playbook -i /path/to/inventory --tags de-reqs kubernetes.yml +ansible-playbook -i /path/to/inventory --tags openldap-docker kubernetes.yml +``` + +## 5.2 Keycloak + +```bash +ansible-playbook -i /path/to/inventory --tags keycloak kubernetes.yml +``` + +Add a DNS record for `keycloak.` if you have not already. An +`/etc/hosts` entry will get you through the next few steps, but it is not a +deployment. + +The realm, LDAP federation, mappers, roles, and eight OAuth clients are then +configured in the Keycloak UI. That configuration is long, exact, and produces +the client secrets that later phases need in `group_vars/all.yml`, so it lives +in its own document: [Keycloak](./05-core-services/keycloak.md). Complete it +before continuing. + +## 5.3 Service signing keys + +From the same directory you run `ansible-playbook` from: + +```bash +./scripts/generate-secrets.sh /path/to/inventory +``` + +This writes the GPG and PEM files Terrain, apps, and iplant-groups expect, and +prints a YAML snippet to add to `group_vars/all.yml`. Commit the generated key +material to your **private** inventory repository only. + +## 5.4 Configuration, ingress, networking, messaging + +```bash +ansible-playbook -i /path/to/inventory \ + --tags=configure-services,ingress,networking,nats kubernetes.yml +``` + +If NATS has to be reinstalled, Helm may still hold a release record; remove it +with `helm -n prod uninstall nats` before re-running. + +## 5.5 Scheduling, caching, search, groups, storage + +```bash +ansible-playbook -i /path/to/inventory \ + --tags=feature-discovery,image-cache,grouper kubernetes.yml +ansible-playbook -i /path/to/inventory --tags=opensearch kubernetes.yml +ansible-playbook -i /path/to/inventory --tags=irods-csi-driver kubernetes.yml +``` + +Node feature discovery labels nodes by capability (GPUs, for instance) so apps +that need specific hardware land on the right worker. Image caching pre-pulls +frequently used VICE images so interactive apps start promptly. + +## 5.6 DE messaging objects + +```bash +ansible-playbook -i /path/to/inventory rabbitmq_configure.yml +``` + +# Phase 6: applications + +Full procedures: +[Discovery Environment](./06-applications/discovery-environment.md), +[VICE](./06-applications/vice.md), +[User Portal](./06-applications/user-portal.md). + +```bash +ansible-playbook -i /path/to/inventory --tags=deploy-all-services kubernetes.yml +``` + +This deploys the full DE service set, the User Portal, and the VICE backend. +Watch for pods that never reach `Running`: + +```bash +kubectl get pods -A | grep -Ev 'Running|Completed' +``` + +# Phase 7: bootstrap and first analyses + +Full procedures: [bootstrap](./07-post-install/bootstrap.md), +[verification](./07-post-install/verification.md), +[troubleshooting](./07-post-install/troubleshooting.md). + +## 7.1 Bootstrap portal administrator + +```bash +ansible-playbook -i /path/to/inventory bootstrap_portal_admin.yml +``` + +Run this from a host that can both `kubectl port-forward` into the cluster and +reach the portal database with `psql`. The playbook port-forwards to OpenLDAP +and portal-conductor and connects to the portal database directly, so a +`pg_hba.conf` rule for that host has to exist. Getting this far means the +port-forwarding half already works; the database half is the one that bites. + +This creates the `-bootstrap` user, the account you use for everything +below. + +## 7.2 Register a VICE operator + +Interactive analyses will not launch until an operator is registered. + +1. Log in to `https://de.` as `-bootstrap`. +2. Open the left navigation from the hamburger menu, then + **Admin → VICE → Operators**. +3. Click **+New** and set: + * **Name**: `prod` + * **URL**: `http://vice-operator.vice-apps:10000` — plain HTTP is correct + here; the connection never leaves the cluster network. + * **Public base URL**: `https://vice.` + * **Priority**: `0` +4. Click **Register operator**. + +## 7.3 Import a starting set of apps + +The `scripts/appei` directory of the deployment repository exports and imports +apps and tools between DEs, and manages its dependencies with +[uv](https://docs.astral.sh/uv/). From that directory: + +```bash +uv run appei login --server de. --username -bootstrap +uv run appei import --server de. -i de-word-count.json --publish +uv run appei import --server de. -i cloudshell.json --featured +uv run appei import --server de. -i portal-delete-user.json +``` + +The result, as `-bootstrap`: + +| App | Visibility | +|-----|------------| +| DE Word Count | public, not featured | +| Cloud Shell | featured, and therefore public | +| portal-delete-user | private to the bootstrap account | + +`portal-delete-user` needs a configuration file attached before it will run; +[bootstrap](./07-post-install/bootstrap.md#portal-delete-user) covers that, +and [troubleshooting](./07-post-install/troubleshooting.md) covers repairing +the import if it lands with the wrong visibility. + +## 7.4 Verify + +Work through [verification](./07-post-install/verification.md): sign in +through Keycloak, browse the Data Store, run the DE Word Count app on a small +input, and launch Cloud Shell to confirm VICE end to end. + +[^irods-install]: iRODS 4.3.3 installation guide +[^irods-packages]: iRODS package repository setup +[^ds-collection]: CyVerse Data Store collection (playbooks and iRODS policy) +[^pilot-record]: Pilot CyVerse deployment record diff --git a/docs/deployment/index.md b/docs/deployment/index.md new file mode 100644 index 0000000..aadc9a7 --- /dev/null +++ b/docs/deployment/index.md @@ -0,0 +1,76 @@ +# Deployment + +* [Deploying CyVerse from scratch](from-scratch.md) - end-to-end walkthrough of a two-node deployment, phase by phase + +The phases below are ordered by dependency, not by preference. iRODS needs +PostgreSQL and RabbitMQ; the Discovery Environment needs all three plus a cluster; +VICE needs the DE. Verify each phase before starting the next. + +# Phase 0: planning + +* [planning/](planning/) - prerequisites, Ansible, and Docker setup +* [Prerequisites](planning/prerequisites.md) - hardware, skills, tooling, and access +* [Component inventory and sizing](../architecture/component-inventory.md) - what to provision +* [Network requirements](../architecture/network-requirements.md) - ports to open before you begin + +# Phase 1: foundation + +* [01-foundation/](01-foundation/) - the host services everything else depends on +* [HAProxy](01-foundation/haproxy.md) - the public entry point +* [PostgreSQL](01-foundation/postgresql.md) - catalog and service databases +* [RabbitMQ](01-foundation/rabbitmq.md) - the AMQP message bus + +# Phase 2: databases + +* [02-databases/](02-databases/) - one schema per service +* [Database migrations](02-databases/migrations.md) - the shared migration procedure + +# Phase 3: Data Store + +* [03-data-store/](03-data-store/) - the iRODS zone +* [iRODS catalog provider](03-data-store/irods-provider.md) - install, policy, and zone initialization +* [iRODS integration for the DE](03-data-store/de-integration.md) - specific queries and service account + +# Phase 4: Kubernetes + +* [04-kubernetes/](04-kubernetes/) - the cluster and its add-ons +* [Cluster](04-kubernetes/cluster.md) - control plane and workers +* [Cluster resources](04-kubernetes/resources.md) - configuration, secrets, and manifests +* [cert-manager](04-kubernetes/cert-manager.md) - TLS issuance +* [Ingress](04-kubernetes/ingress.md) - Traefik and the legacy ingress-nginx path +* [Storage](04-kubernetes/storage.md) - persistent volumes +* [Harbor](04-kubernetes/harbor.md) - container registry +* [Argo Workflows](04-kubernetes/argo.md) - batch analysis execution + +# Phase 5: core services + +* [05-core-services/](05-core-services/) - directory, authentication, search, messaging, storage plumbing +* [OpenLDAP](05-core-services/openldap.md) - accounts and groups +* [Keycloak](05-core-services/keycloak.md) - realm, federation, and OAuth clients +* [Grouper](05-core-services/grouper.md) - group management +* [OpenSearch](05-core-services/opensearch.md) - data search index +* [NATS](05-core-services/nats.md) - internal messaging +* [Redis HA](05-core-services/redis-ha.md) - caching and sessions +* [Unleash](05-core-services/unleash.md) - feature flags +* [iRODS CSI driver](05-core-services/irods-csi-driver.md) - Data Store mounts for pods +* [Mail](05-core-services/mail.md) - outbound mail +* [Jaeger](05-core-services/jaeger.md) - distributed tracing + +# Phase 6: applications + +* [06-applications/](06-applications/) - the user-facing platform +* [Discovery Environment](06-applications/discovery-environment.md) - the DE service set +* [VICE](06-applications/vice.md) - interactive analyses +* [User Portal](06-applications/user-portal.md) - account management + +# Phase 7: post-install + +* [07-post-install/](07-post-install/) - making the deployment usable and confirming it works +* [Bootstrap](07-post-install/bootstrap.md) - first administrator, VICE operator, starting apps +* [Verification](07-post-install/verification.md) - end-to-end checks +* [Troubleshooting](07-post-install/troubleshooting.md) - fixes for first-deployment problems + +# After deployment + +* [operations/](../operations/) - day-to-day administration +* [FAQ](../operations/faq.md) - common questions and recipes diff --git a/docs/deployment/planning/ansible.md b/docs/deployment/planning/ansible.md new file mode 100644 index 0000000..3e0678a --- /dev/null +++ b/docs/deployment/planning/ansible.md @@ -0,0 +1,149 @@ +--- +type: Deployment Procedure +title: "Ansible" +description: "Installing Ansible and the inventory, group_vars, and playbook conventions the CyVerse deployment expects." +tags: [deployment, planning, ansible, automation] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: ansible-docs + resource: https://docs.ansible.com/ansible/latest/ + title: Ansible documentation + author: team:ansible +--- + +# How CyVerse uses Ansible + +[Ansible](https://www.ansible.com/) is agentless: it runs from your workstation +over `ssh` and needs nothing installed on the servers. CyVerse uses it for +everything from host preparation to deploying the DE service set, driven by tags +on a single `kubernetes.yml` playbook plus a handful of standalone playbooks. + +Read the [Ansible documentation](https://docs.ansible.com/ansible/latest/) if you +have not used it before — this document covers only the CyVerse conventions. + +# Install + +Any of these works; pick whichever your workstation already manages packages +with: + +```bash +# pip +pip install --user ansible + +# homebrew +brew install ansible + +# distribution package +apt-get install ansible +``` + +## Third-party roles + +Some playbooks depend on roles from Ansible Galaxy. From the playbook repository: + +```bash +ansible-galaxy install --force -r requirements.yaml +``` + +# Configuration + +CyVerse's variables are structured as YAML hashes, and overriding a single key +inside one requires hash merging rather than replacement: + +```bash +export ANSIBLE_HASH_BEHAVIOUR="merge" +``` + +Set it in your shell profile, or set `hash_behaviour = merge` in a local +`ansible.cfg` — the second is more reliable, because it travels with the +repository instead of with your shell. + +# Inventory layout + +Inventories are **not** in the public repository, and files matching `*.cfg` in +the inventories directory are gitignored deliberately: an inventory names every +host in the deployment. + +Keep yours in a private repository. Start from the example inventory shipped with +the playbooks and fill in the groups; for the two-node pilot shape, see +[cluster](../04-kubernetes/cluster.md#inventory). + +## group_vars + +The default variables live in `inventories/group_vars/all` in the playbook +repository, with every variable the roles use set to a default. You create your +own `group_vars/all.yml` in your inventory and override only what differs. + +The example file is organized **in phases**, and that ordering is meaningful: some +variables cannot be filled in until earlier phases have produced their values. +The clearest case is Keycloak — its eight client secrets do not exist until the +clients have been created in the running Keycloak, so phase 5 variables are filled +in partway through the deployment. Fill in phases 1 through 4 before starting, and +expect to come back. + +# Playbook layout + +The repository diverges from Ansible's recommended layout in two documented ways: + +* **`group_vars` location.** In `inventories/group_vars` rather than beside the + playbooks, so that inventories carry their own variables. +* **Role-separated playbooks.** Kept in `playbooks/`; what sits in the top level + of `ansible/` is composite or one-off playbooks. To run a single role, use the + `single-role.yaml` playbook — its own comments document its use. + +# Running playbooks + +```bash +# check every host is reachable +ansible -i /path/to/inventory -m ping all + +# run a phase by tag +ansible-playbook -i /path/to/inventory --tags prep-nodes kubernetes.yml + +# a standalone playbook +ansible-playbook -i /path/to/inventory rabbitmq_configure.yml +``` + +Useful flags: `-K` to prompt for the `sudo` password, `-u ` to override the +remote user, `--check` for a dry run, and `--diff` to see what would change. + +# Preparing servers + +Modern targets need little preparation beyond what `prep-nodes` does. Two +requirements are worth knowing about because their failure mode is a confusing +Ansible error rather than a clear one: + +* **Python 3 on every host**, which is what Ansible's modules run under. +* **`python3-psycopg2` on the database host**, which the PostgreSQL modules + import. + +Older CyVerse notes also call for `python-simplejson`, `python-httplib2`, and +`curl` on CentOS 5/6 and early Ubuntu hosts. Those instructions apply only to +distributions that are now end of life; on a current OS the packages above are +enough. + +# SSH access + +```bash +# generate a key if you do not have one +ssh-keygen -t ed25519 + +# install it on each host +ssh-copy-id -i ~/.ssh/id_ed25519.pub @ +``` + +Then `ssh` to each host once by fully qualified name, so its host key lands in +`~/.ssh/known_hosts` — Ansible fails on an unknown host key, and doing this +up front turns a run-time failure into a five-minute setup step. An `~/.ssh/config` +entry per host saves typing later. + +`ed25519` is the current default; older notes generate RSA keys, which still work +but are no longer the recommendation. + +# Related + +* [Docker-based setup](./docker.md) +* [Cluster](../04-kubernetes/cluster.md) +* [Cluster resources](../04-kubernetes/resources.md) +* [Database migrations](../02-databases/migrations.md) diff --git a/docs/deployment/planning/docker.md b/docs/deployment/planning/docker.md new file mode 100644 index 0000000..1d6599f --- /dev/null +++ b/docs/deployment/planning/docker.md @@ -0,0 +1,56 @@ +--- +type: Deployment Procedure +title: "Docker" +description: "Installing Docker and running the deployment playbooks from inside a container." +tags: [deployment, planning, docker, containers] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +sources: + - id: docker-docs + resource: https://docs.docker.com/engine/install/ + title: Docker Engine installation + author: team:docker +--- + +# Install + +Follow the [Docker Engine installation +guide](https://docs.docker.com/engine/install/) for your platform. On macOS and +Windows that means Docker Desktop; on Linux, the distribution packages or Docker's +own repository. + +You need Docker for two unrelated reasons: building service and tool images, and — +optionally — running the deployment playbooks in a container instead of installing +Ansible on your workstation. + +# Running the playbooks in a container + +The deployment repository includes a `Dockerfile` at the top level that builds an +image capable of running the playbooks. It is useful when your workstation cannot +easily run the Ansible version the playbooks expect. + +```bash +# generate the SSH configuration the image expects +./create-ssh-configs.sh + +# build the image +docker build -t cyverse-ansible . + +# run the playbooks with the working tree mounted +docker run --rm -it -v "$(pwd)":/ansible -w /ansible cyverse-ansible /bin/bash +``` + +From the shell inside the container, run `ansible-playbook` as usual; see +[Ansible](./ansible.md). + +!!! danger "Never push this image" + + The build embeds your SSH configuration, and a run mounts your inventory. That + image is a copy of your credentials, and pushing it to any registry publishes + them. Build it locally, use it locally, and do not tag it for a registry. + +# Related + +* [Ansible](./ansible.md) +* [Harbor](../04-kubernetes/harbor.md) +* [Prerequisites](./prerequisites.md) diff --git a/docs/deployment/planning/index.md b/docs/deployment/planning/index.md new file mode 100644 index 0000000..0063e52 --- /dev/null +++ b/docs/deployment/planning/index.md @@ -0,0 +1,10 @@ +# Planning + +* [Prerequisites](prerequisites.md) - hardware, skills, tooling, and access to have in place first +* [Ansible](ansible.md) - installation, inventory layout, and group_vars conventions +* [Docker](docker.md) - Docker setup, and running the playbooks in a container + +# Related + +* [Component inventory and sizing](../../architecture/component-inventory.md) +* [Network requirements](../../architecture/network-requirements.md) diff --git a/docs/deployment/planning/prerequisites.md b/docs/deployment/planning/prerequisites.md new file mode 100644 index 0000000..604ba00 --- /dev/null +++ b/docs/deployment/planning/prerequisites.md @@ -0,0 +1,108 @@ +--- +type: Reference +title: "Prerequisites" +description: "Hardware, skills, tooling, and access you need in place before deploying CyVerse." +tags: [deployment, planning, prerequisites] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# Before you start + +CyVerse is a full-stack cyberinfrastructure: a data platform, an authentication +stack, a Kubernetes cluster, and a dozen services on top. Deploying it is a +project, not an afternoon. This document is the readiness check; the ordered +procedure is in [deployment](../index.md), and the narrative walkthrough of a +real two-node build is [deploying from scratch](../from-scratch.md). + +# Infrastructure + +!!! success "Hardware or cloud" + + * Bare metal, an OpenStack cloud, or a commercial provider. + * Enough capacity for the components in + [component inventory](../../architecture/component-inventory.md). The + smallest useful deployment is two well-provisioned servers; a production + deployment is a cluster. + * Storage for the iRODS vault, sized to the science rather than to a + recommendation, plus persistent volumes for the cluster's stateful + services. + * Public IP addresses, and DNS you can create records in — including a + wildcard for VICE. + * API access to your DNS provider, for automated certificate issuance. + +!!! success "Network" + + * The ports in + [network requirements](../../architecture/network-requirements.md), open + before you begin rather than debugged later. + * Institutional firewall changes agreed in advance. On a university network + this is usually the longest lead time in the whole project. + * Experience operating in a + [Science DMZ](https://en.wikipedia.org/wiki/Science_DMZ_Network_Architecture) + architecture helps: high-throughput data transfer and segmented services + are exactly what this stack needs. + +# Skills + +* Linux system administration, including filesystem permissions and systemd. +* Kubernetes cluster operation — not just `kubectl apply`, but reading events, + debugging pod networking, and understanding storage classes. +* Ansible for configuration management. +* PostgreSQL administration and tuning. +* Container fundamentals and registry management. +* DNS, TLS, load balancing, and ingress. +* iRODS concepts: zones, resources, vaults, and the catalog. + +Nobody has all of these in equal measure. What matters is that the deployment +team as a whole does, and that whoever runs it can tell a configuration error +from a network error from a policy error. + +# Tooling + +On the workstation you deploy from: + +| Tool | Used for | +|------|----------| +| `ansible` | Everything the playbooks do | +| `kubectl`, `helm` | Cluster operations, chart installs | +| `k0sctl` | Creating and upgrading the cluster | +| `git` | The deployment and inventory repositories | +| `psql` | Database work, including bootstrap steps | +| `uv` | The `appei` app import and export tool | +| `gomplate` | Rendering the configuration and secret templates | +| `skaffold` | Building and deploying individual services during development | + +Setup instructions: [Ansible](./ansible.md), [Docker](./docker.md). + +# Access + +* `ssh` to every node, key-based, with `sudo`. +* A **private** git repository for the inventory, `group_vars`, and generated key + material. Nothing in this list belongs in a public repository. +* A container registry you control — see [Harbor](../04-kubernetes/harbor.md). +* Administrative access to the DNS zone. + +!!! danger "Secrets discipline, decided up front" + + A CyVerse deployment generates a lot of secrets: database passwords, an iRODS + zone key, a negotiation key, a password salt, eight Keycloak client secrets, + GPG and PEM signing keys, registry credentials, and DNS API tokens. + + Decide where they live before you generate the first one. The pattern that + works is: generate at install time, store in the private inventory repository, + template into place with Ansible, and never paste into a shell. + +# Temperament + +Distributed systems fail in layers, and this one has many. The single most useful +habit is to verify each phase before starting the next — which is what the phase +order in [deployment](../index.md) is for, and why +[verification](../07-post-install/verification.md) exists as its own document. + +# Next + +* [Deployment overview and order](../index.md) +* [Deploying from scratch](../from-scratch.md) +* [Component inventory and sizing](../../architecture/component-inventory.md) +* [Network requirements](../../architecture/network-requirements.md) diff --git a/docs/deployments/DiscoveryEnvironment.md b/docs/deployments/DiscoveryEnvironment.md deleted file mode 100644 index adfcb21..0000000 --- a/docs/deployments/DiscoveryEnvironment.md +++ /dev/null @@ -1,44 +0,0 @@ -# Discovery Environment - -!!! success "Prerequisites" - - * Make sure you have the `harbor-registry-credentials` secrets in your NAMESPACE, see also [k8s-resources](k8s-resources.md) - - * Make sure you have `de-nginx-tls` secret created, also see [k8s-resources](k8s-resources.md) - - * Make sure your Haproxy has the CA cert, Add `/docker-tugraz-data/ca/ca.pem` to `HAPROXY_DOMAIN`:/etc/ssl/certs/ca-bundle.crt` - - -**remember** `nginx` is proxying all the services to the DE in CyVerse. - -* Use the manifest files from the [k8s-resources](k8s-resources.md) - -### Change hardcoded - -If you are using a diffrent domain instead of `cyverse.tugraz.at`, e.g. `cyverse.at` - change these two files. - -**/k8s-resources/resources/kustomize/de-nginx/base/nginx.conf** - -```diff -- server_name ~^[^.]+[.]cyverse[.]tugraz[.]at$; -+ server_name ~^[^.]+[.]cyverse[.]at$; -``` - -**k8s-resources/resources/kustomize/de-nginx/base/kustomization.yaml** - -```diff -- namespace: prod -+ namespace: discover -``` - -### Deploy - -```bash -## For prod env -kubectl apply -k resources/kustomize/de-nginx/overlays/prod/ -n prod -kubectl apply -f resources/services/tugraz.yml -n prod - -## For discover env -# kubectl apply -k resources/kustomize/de-nginx/overlays/prod/ -n discover -# kubectl apply -f resources/services/tugraz.yml -n discover -``` diff --git a/docs/deployments/RabbitMQ.md b/docs/deployments/RabbitMQ.md deleted file mode 100644 index b716eec..0000000 --- a/docs/deployments/RabbitMQ.md +++ /dev/null @@ -1,60 +0,0 @@ -# TODO - -!!! success "Prerequisites" - - -## Reindex RabbitMQ jobs - -### access the vm where the RabbitMQ is installed. - -```bash -ssh root@RABBITMQ_HOST -``` - -### Configure rabbitmqadmin - -This step you have to do only once, if the rabbitmqadmin is not present. - -```bash -mkdir adm -cd adm -wget http://localhost:15672/cli/rabbitmqadmin -chmod +x rabbitmqadmin -``` - -### Commands - -#### discover - -```bash -# check status -systemctl status rabbitmq-server.service -l - -## add your password to a temp var -read -s PASSWORD && export PASSWORD - -# list exchange -./rabbitmqadmin -V /cyverse/de list exchanges -u cyverse -p $PASSWORD - -# publish the message to reindex all -./rabbitmqadmin publish -V /cyverse/de -u cyverse -p $PASSWORD exchange=de routing_key=index.all payload="" - -# restart infosquito2 -kubectl rollout restart deployment infosquito2 -n discover -# IF NOT deployed -# ./deploy.py -Bn discover -p infosquito2 -C -``` - -#### prod -```bash -## add your password to a temp var -read -s PASSWORD && export PASSWORD - -# list -./rabbitmqadmin -V /tugraz/de list exchanges -u tugraz -p $PASSWORD - -# publish the message to reindex all -./rabbitmqadmin publish -V /tugraz/de -u tugraz -p $PASSWORD exchange=de routing_key=index.all payload="" - -``` - diff --git a/docs/deployments/deployment_overview.md b/docs/deployments/deployment_overview.md deleted file mode 100644 index 3b12127..0000000 --- a/docs/deployments/deployment_overview.md +++ /dev/null @@ -1,45 +0,0 @@ -# Deployment Overview - -All deployments of CyVerse Products and Services are managed through Kubernetes. - -Each service is maintained in its own GitHub Repository in the core [:simple-github: CyVerse Organization](https://github.com/cyverse){target=_blank} or [:simple-github: CyVerse Discovery Environment Organization](https://github.com/cyverse-de){target=_blank} - -## :simple-kubernetes: Deployments - -All of CyVerse primary services and database deployments are containers, controled via fully managed [:simple-kubernetes: Kubernetes](https://kubernetes.io/) - -[:simple-kubernetes: Discovery Environment](../deployments/DiscoveryEnvironment.md) - deploy primary data science workbench site - -[:simple-kubernetes: Kubernetes (K8s)](../deployments/kubernetes-deploy.md) - deploy the main K8s cluster for running DE applications - -[:simple-kubernetes: K8s Resources](../deployments/k8s-resources.md) - deploy the various resources in DE managed by K8s - -[:simple-kubernetes: K8s NameSpaces](../deployments/k8s-namespace.md) - list of namespaces used in DE - -[:simple-kubernetes: User Portal](../deployments/userportal.md) - deploy the User Portal website via K8s - -[:simple-kubernetes: OpenEBS](../deployments/openebs.md) - deploy K8s stateful workloads that require container attached storage - -[:simple-kubernetes: KeyCloak](../deployments/keycloak.md) - deploy K8s KeyCloak configuration - -[:simple-kubernetes: Exim4 Mail](../deployments/exim4.md) - deploy `exim4` (MTA) running as a smarthost via K8s - -[:simple-kubernetes: Redis HA](../deployments/redis-ha.md) - installing the Redis Server and Redis Haproxy - -[:simple-kubernetes: ElasticSearch](../deployments/elasticsearch.md) - deploy stateful set ES cluster - -[:simple-kubernetes: RabbitMQ](../deployments/RabbitMQ.md) - deploy RabbitMQ services - -[:simple-kubernetes: Unleash](../deployments/unleash.md) - deploy the Unleash database - -[:simple-kubernetes: Grouper](../deployments/grouper.md) - Internet2 Grouper Service - -[:simple-kubernetes: iRODS CSI Driver](../deployments/irods-csi-driver.md) - K8s Container Storage Interface (CSI Driver) for iRODS - -[:simple-kubernetes: Local Exim](../deployments/local-exim.md) - verification of email using Exim - -[:simple-kubernetes: VICE](../deployments/vice.md) - Manage K8s interactive jobs in DE - -[:simple-kubernetes: Jaeger](../deployments/jaeger.md) - open-source end-to-end distributed tracing - -[:simple-github: :simple-ansible: Data Store](https://github.com/cyverse/ds-playbooks) - deploy the CyVerse data Store diff --git a/docs/deployments/exim4.md b/docs/deployments/exim4.md deleted file mode 100644 index 883c82e..0000000 --- a/docs/deployments/exim4.md +++ /dev/null @@ -1,44 +0,0 @@ -# Mail - -!!! success "Prerequisites" - - [exim4-helm](https://github.com/mb-wali/exim4-helm) - - A Helm chart to provide a exim4 deployment, exim4 (MTA) running as a smarthost. - -## Deploy - -### Add & update helm chart - -```bash -helm repo add exim4 https://mb-wali.github.io/exim4-helm -helm repo update -``` - -### Install - -```bash -# replace the secrets with yours -helm install exim4 --set secrets.EXIM_SMARTHOST='localhost',secrets.EXIM_PASSWORD='passw0rd',secrets.EXIM_ALLOWED_SENDERS='*' exim4/exim4 --namespace mail --create-namespace --wait -``` - -### Debugging - -Once the pod is running - -```bash -# execute shell -kubectl exec -it exim4-6ff546fb9f-ff47m -- bash - -# send a test mail -echo "This is test" | mail -s "The subject" receiver@myhost.com -aFrom:sender@myhost.com -``` - -### Usage - -Use your deployed exim4 to send mails, -e.g. connect from a another service. - -```bash -SMTP_HOST=exim4.mail.svc.cluster.local -``` diff --git a/docs/deployments/grouper.md b/docs/deployments/grouper.md deleted file mode 100644 index adcefee..0000000 --- a/docs/deployments/grouper.md +++ /dev/null @@ -1,27 +0,0 @@ -# Grouper - -!!! success "Prerequisites" - - * Make sure `grouper` database is setup, see also [grouper-db](../database/grouper-db.md) - -## Deploy -Grouper consist of two deployments, `grouper-loader` & `grouper-ws`. -### grouper-loader - -```bash -## Deploy for prod env -kubectl apply -f resources/deployments/grouper-loader.yml -n prod - -## Deploy for discover env -# kubectl apply -f resources/deployments/grouper-loader.yml -n discover -``` - -### grouper-ws - -```bash -## Deploy for prod env -kubectl apply -f resources/deployments/grouper-ws.yml -n prod - -## Deploy for discover env -# kubectl apply -f resources/deployments/grouper-ws.yml -n discover -``` diff --git a/docs/deployments/ingress-nginx.md b/docs/deployments/ingress-nginx.md deleted file mode 100644 index d16c346..0000000 --- a/docs/deployments/ingress-nginx.md +++ /dev/null @@ -1,30 +0,0 @@ -# ingress-nginx - -!!! success "Prerequisites" - - -**ingress-nginx** is used to give vice-apps ingresses, using nodeport and what not. - - -## deploy - -The deployment manifests are in [k8s-resources](k8s-resources.md). - -### (optional) changing the env - -Modify file `resources/kustomize/ingress-nginx/overlays/prod/args.yaml`, to change the namespace. - -```diff -- --default-backend-service=prod/vice-default-backend -+ --default-backend-service=discover/vice-default-backend -``` - -### Deploy kustomize - -This script will create a namespace `ingress-nginx`. - -```bash -kubectl apply -k resources/kustomize/ingress-nginx/overlays/prod -``` - - diff --git a/docs/deployments/irods-csi-driver.md b/docs/deployments/irods-csi-driver.md deleted file mode 100644 index 5eaf808..0000000 --- a/docs/deployments/irods-csi-driver.md +++ /dev/null @@ -1,115 +0,0 @@ -# iRODS CSI Driver - -iRODS Container Storage Interface (CSI) Driver implements the CSI Specification to provide container orchestration engines (like Kubernetes) iRODS access. - -!!! success "Prerequisites" - - - -Before we install `irods-csi-driver`, we need to create a `values.yaml` file. - -**Change the values accordingly and save the file as `values.yaml`.** - -```yaml -globalConfig: - secret: - stringData: - client: "irodsfuse" - host: - port: "1247" - zone: "TUG" - user: - password: - retainData: "false" - enforceProxyAccess: "true" - mountPathWhitelist: "/TUG/home" -nodeService: - irodsPool: - extraArgs: - - --cache_size_max=10737418240 - - --cache_root=/irodsfs_pool_cache - - '--cache_timeout_settings=[{"path":"/","timeout":"-1ns","inherit":false},{"path":"/TUG","timeout":"-1ns","inherit":false},{"path":"/TUG/home","timeout":"5m","inherit":false},{"path":"/TUG/home/shared","timeout":"5m","inherit":true}]' - -``` - -## Deploy - -we use helm to deploy irods-csi-driver. - -```bash -# Add the Helm repository. -helm repo add irods-csi-driver-repo https://cyverse.github.io/irods-csi-driver-helm/ - -# Update the local repository caches. -helm repo update - -# create namespace -kubectl create namespace irods-csi-driver - -# install csi-driver -# make sure to edit values-cyverse.at.yaml -helm install -n irods-csi-driver irods-csi-driver irods-csi-driver-repo/irods-csi-driver -f ./values.yaml - -# or upgrade -helm upgrade -n irods-csi-driver irods-csi-driver irods-csi-driver-repo/irods-csi-driver -f ./values.yaml - -``` - -## Upgrading to a newer version - -When we want to upgrade the `irods-csi-driver` to a newer version, we need to stop all running vice-apps and delete all the pvcs. - -```bash -# update helm repo -helm repo update - -# delete the pvc -kubectl delete pvc -l app-type=interactive -n vice-apps - -# uninstall the irods-csi-driver -helm uninstall irods-csi-driver -n irods-csi-driver - -# delete all the vice-apps deployments -## see below for the content of this file -./nuke-vice-analysis.sh $(kubectl get deployments -n vice-apps -l app-type=interactive -o name) - -# install again -helm install -n irods-csi-driver irods-csi-driver irods-csi-driver-repo/irods-csi-driver -f values.yaml -``` - -## install specific version -```bash -helm install -n irods-csi-driver irods-csi-driver --version 0.8.7 irods-csi-driver-repo/irods-csi-driver -f values.yaml - -## try the latest -helm install -n irods-csi-driver irods-csi-driver --version 0.9.2 irods-csi-driver-repo/irods-csi-driver -f values.yaml -``` - -## NOTE -With `irods-csi-driver` **version > 0.8.7**, there’s a small change on the configuration file `user_config.yaml` for driver installation. -**You will need to delete --cache_root and --temp_root flags if you used it.** - - - -# nuke-vice-analysis.sh - -```sh -function delete_resources() { - local external_id="$1" - kubectl -n vice-apps delete deployment "${external_id}" - kubectl -n vice-apps delete service "vice-${external_id}" - kubectl -n vice-apps delete ingress "${external_id}" - kubectl -n vice-apps delete configmap "excludes-file-${external_id}" - kubectl -n vice-apps delete configmap "input-path-list-${external_id}" -} - -function remove_deployment_prefix() { - local external_id="$1" - echo -n "$external_id" | sed 's;^deployment.apps/;;' -} - -# Iterate over all arguments on the command line. -for id in "$@"; do - delete_resources $(remove_deployment_prefix "$id") -done -``` \ No newline at end of file diff --git a/docs/deployments/jaeger.md b/docs/deployments/jaeger.md deleted file mode 100644 index e61afe5..0000000 --- a/docs/deployments/jaeger.md +++ /dev/null @@ -1,37 +0,0 @@ -# jaeger - -Jaeger: open source, end-to-end distributed tracing - -!!! success "Prerequisites" - - -## Deploy - -The deployment manifests are in [k8s-resources](k8s-resources.md). - -**Create namespace** -```bash -kubectl create ns jaeger -``` - - -**(optional) changing the env** - -Modify files to change the namespace: - -* `resources/addons/jaeger/collector.yaml` -* `resources/addons/jaeger/query.yaml` -* `resources/addons/jaeger/rollover-cron.yaml` - -```diff -- "http://elasticsearch.prod:9200" -+ "http://elasticsearch.discover:9200" -``` - -**Apply manifests** -```bash -kubectl apply -f resources/addons/jaeger/rollover-cron.yaml -n jaeger -kubectl apply -f resources/addons/jaeger/query.yaml -n jaeger -kubectl apply -f resources/addons/jaeger/collector.yaml -n jaeger -``` - diff --git a/docs/deployments/k8s-namespace.md b/docs/deployments/k8s-namespace.md deleted file mode 100644 index 67076ec..0000000 --- a/docs/deployments/k8s-namespace.md +++ /dev/null @@ -1,41 +0,0 @@ -# Namespaces - -List of namespaces used for CyVerse deployment in kubernetes. - -!!! success "Prerequisites" - -**Note**: this documentation is relavant only for single/current environment. - -![Namespaces](../assets/namespaces.png) - -## prod - -This namespace is dedicated to the [core services](deployment_overview.md) & some of non-core services such as : - -* [redis-ha](redis-ha.md) -* redis-haproxy -* [elasticsearch](elasticsearch.md) - -Except the **irods-csi-driver** which runs in a specific namespace. - - -## ingress-nginx - -This namespace is dedicated to the [ingress-nginx](https://docs.nginx.com/nginx-ingress-controller/) deployment. - -## vice-apps - -This namespace is dedicated to the [VICE](https://cyverse.org/node/50) related deployment and configurations. - -## keycloak - -This namespace is dedicated to Keycloak related deployment and configurations. - -## openebs - -This namespace is dedicated to the [openebs](https://openebs.io/) deployment. -OpenEBS turns any storage available on the Kubernetes worker nodes into local or distributed Kubernetes Persistent Volumes. - -## irods-csi-driver - -This namespace is dedicated to the [irods-csi-driver](https://github.com/cyverse/irods-csi-driver) deployment and configurations. diff --git a/docs/deployments/k8s-resources.md b/docs/deployments/k8s-resources.md deleted file mode 100644 index 7ffdbb8..0000000 --- a/docs/deployments/k8s-resources.md +++ /dev/null @@ -1,455 +0,0 @@ -# k8s-resources - -This repository includes all the manifests and resources for kubernetes deployment. - -!!! success "Prerequisites" - - -## Clone - -```bash -git clone git@gitlab.cyverse.org:tugraz/k8s-resources.git -``` - -## Generate config/secrets for **prod** env - -Make sure [gomplate](https://docs.gomplate.ca/installing/#manual-install) is installed in your OS. - -**Here is an example on ubuntu:** - -```bash -sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.10.0/gomplate_linux-amd64 - -sudo chmod 755 /usr/local/bin/gomplate - -gomplate --help -``` - -**Generate configs/secrets** -```bash -./generate_configs.py -e prod -./generate_secrets.py -e prod -``` - -## Load config/secrets in your cluster for **prod** env - -```bash -./load_configs.py -e prod -n prod -./load_secrets.py -e prod -n prod -``` - -## Deploy services - -### preq - -Before we start to deploy the services we need to create these secrets, and install some dependencies. - -* `vice-image-pull-secret` TODO: kubectl apply. -* `harbor-registry-credentials` TODO: kubectl apply. - -**These secrets below can be deployed using:** `git clone -b discover https://gitlab.cyverse.org/tugraz/docker-tugraz-data` see also [cyverse.at](https://github.com/mb-wali/cyverse.at/tree/main/k8s/discoveryEnvironment) repo. - -* `gpg-keys` -* `ui-nginx-tls` -* `gpg-keys` -* `pgpass-files` -* `signing-keys` -* `accepted-keys` -* `ssl-files` -* Make sure [elasticsearch](elasticsearch.md) is deployed. -* Make sure [skaffold](https://docs.gomplate.ca/installing/#manual-install) is installed in your OS. - ```bash - # For Linux x86_64 (amd64) - curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 - - sudo install skaffold /usr/local/bin/ - - # check - skaffold --help - ``` -* Make sure ServiceAccounts are created: - ```bash - # For prod env - kubectl apply -f /k8s-resources/resources/serviceaccounts/app-exposer.yml -n prod - - ## For discover env - # kubectl apply -f /k8s-resources/resources/serviceaccounts/app-exposer.yml -n discover - ``` - -#### TODO above - -### Deploy - -This command will deploy all the services listed on `k8s-resources/repos` - -```bash -# deploy services for prod env -./deploy.py -n prod -BCa - -## deploy services for discover env -# ./deploy.py -n discover -BCa -``` - -#### Deploy single service - -If you want to deploy a single service, e.g. `search` -```bash -## Run for the prod env -./deploy.py -Bn prod -p search -C - -## Run for the discover env -# ./deploy.py -Bn discover -p search -C -``` - -# create a new env -Currently we have `prod` environment which is our productive instance. - -Create your environment: `cp config_values/prod.yaml config_values/discover.yaml` -This will create a new config file for your environment. You will have to update all the values as you see fit for your environment. - -**fill in all these values:** -```yaml -# discover.yaml ---- -Environment: - -Agave: - Key: - Secret: - RedirectURI: - StorageSystem: - CallbackBaseURI: - ReadTimeout: - Enabled: - JobsEnabled: - -AMQP: - URI: - -AnonFiles: - BaseURI: - -AppExposer: - BaseURI: - -BaseURLs: - Analyses: - Apps: - AsyncTasks: - DashboardAggregator: - DataInfo: - GrouperWebServices: - IplantEmail: - IplantGroups: - JexAdapter: - Metadata: - Notifications: - Permissions: - Requests: - Search: - Terrain: - UserInfo: - -CAS: - BaseURI: - ServerName: - UIDDomain: - -DashboardAggregator: - PublicGroup: - LogLevel: - -DataOne: - BaseURI: - -DE: - Version: - VersionName: - AMQP: - URI: - Host: - BaseURI: - Legacy: - BaseURI: - Subscriptions: - CheckoutURL: - KeepAlive: - Service: - Target: - ContextMenu: - Enabled: - BaseTrash: - Path: - ProdDeployment: - DefaultOutputFolder: - WSO2: - JWTHeader: - Coge: - BaseURI: - Tools: - Admin: - MaxCpuLimit: - MaxMemoryLimit: - MaxDiskLimit: - -Docker: - TrustedRegistries: - Tag: - -Elasticsearch: - BaseURI: - Username: - Password: - Index: - -Email: - AppDeletion: - Src: - Dest: - AppPublicationRequest: - Src: - Dest: - ToolRequest: - Src: - Dest: - PermIDRequest: - Src: - Dest: - Support: - Src: - Dest: - -Grouper: - Environment: - MorphString: - WebService: - Password: - Password: - DB: - User: - Password: - Host: - Port: - Name: - FolderNamePrefix: - Loader: - URI: - User: - Password: - SubjectSource: - ID: - Name: - SearchBase: - -ICAT: - Host: - Port: - User: - Password: - -Infosquito: - DayNum: - PrefixLength: - -InteractiveApps: - BaseURI: - ServiceSuffix: - -Intercom: - AppID: - CompanyID: - CompanyName: - Intercom: - -IRODS: - AMQP: - URI: - Host: - User: - Zone: - Password: - AdminUsers: - PermsFilter: - ExternalHost: - QuotaRootResources: - -Jobs: - DataTransferImage: - -JobStatusListener: - BaseURI: - -Keycloak: - ServerURI: - Realm: - ClientID: - ClientSecret: - VICE: - ClientID: - ClientSecret: - -Kifshare: - ExternalUri: - -PGP: - KeyPassword: - -PermanentID: - CuratorsGroup: - DataCite: - BaseURI: - User: - Password: - DOIPrefix: - -Redis: - Host: - Port: - HA: - Name: - Password: - DB: - Number: - -TimeZone: - -Vault: - Token: - URL: - IRODS: - MountPath: - ChildToken: - UseLimit: - -VICE: - DB: - User: - Password: - Host: - Port: - Name: - FileTransfers: - Image: - Tag: - JobStatus: - BaseURI: - K8sEnabled: - BackendNamespace: - ImagePullSecret: - ImageCache: - UseCSIDriver: - DefaultImage: - DefaultName: - DefaultCasUrl: - DefaultCasValidate: - ConcurrentJobs: - UseCaseCharsMin: - DefaultBackend: - LoadingPageTemplateString: - -Sonora: - BaseURI: - -Terrain: - CASClientID: - CASClientSecret: - JWT: - SigningKey: - Password: - -Unleash: - BaseUrl: - APIPath: - APIToken: - MaintenanceFlag: - -UserPortal: - BaseURI: - -DEDB: - User: - Password: - Host: - Port: - Name: - -NewNotificationsDB: - User: - Password: - Host: - Port: - Name: - -NotificationsDB: - User: - Password: - Host: - Port: - Name: - -PermissionsDB: - User: - Password: - Host: - Port: - Name: - -QMSDB: - User: - Password: - Host: - Port: - Name: - Reinitialize: - -MetadataDB: - User: - Password: - Host: - Port: - Name: - -UnleashDB: - User: - Password: - Host: - Port: - Name: - -Admin: - Groups: - Attribute: - -FileIdentifier: - HtPathList: - MultiInputPathList: - -Analytics: - Enabled: - Id: - -Harbor: - URL: - ProjectQARobotName: - ProjectQARobotSecret: - -QMS: - Enabled: - Base: - Usage: - -Jaeger: - Endpoint: -``` - -## Generate config/secrets for **discover** env -```bash -./generate_configs.py -e discover -./generate_secrets.py -e discover -``` - -## Load config/secrets in your cluster for **discover** env -```bash -./load_configs.py -e discover -n discover -./load_secrets.py -e discover -n discover -``` diff --git a/docs/deployments/keycloak.md b/docs/deployments/keycloak.md deleted file mode 100644 index ca58362..0000000 --- a/docs/deployments/keycloak.md +++ /dev/null @@ -1,70 +0,0 @@ -# Keycloak - -!!! success "Prerequisites" - -### configure database - -To setup and configure database please have a look at [keycloak database](../database/keycloak-db.md). - - -### create namsespace - -```bash -# create ns -kubectl create ns keycloak -``` - -### Create required secrets & configmap - -Keycloak deployment requires secrets and configmaps, which can be done via a `kustomization.yaml` file, please see below for an example of this template: - -```yaml -secretGenerator: -- name: dbuser # Database - literals: - - username= - - password= -- name: kcadmin # keyclaok - literals: - - username= - - password= -configMapGenerator: -- name: keycloak-config - literals: - - KEYCLOAK_HOSTNAME=keycloak.example.com - - KEYCLOAK_LOGLEVEL=INFO - - DB_VENDOR=postgres - - DB_ADDR= - - DB_PORT=5432 - - PROXY_ADDRESS_FORWARDING=true - - JDBC_PARAMS=connectTimeout=21600 - - JAVA_OPTS=-server - -Xms4096m - -Xmx8192m - -XX:MetaspaceSize=96m - -XX:MaxMetaspaceSize=256m - -Djboss.modules.system.pkgs=org.jboss.byteman - -Djava.awt.headless=true - -Dkeycloak.profile.feature.token_exchange=enabled - -Djava.security.egd=file:/dev/urandom -namespace: keycloak -resources: -- deployment.yaml -- service.yaml -generatorOptions: - disableNameSuffixHash: true -``` - -* Update the values of the `kustomization.yaml` file. -* place the `kustomization.yaml`, `deployment.yaml` and `service.yaml` inside a directory. e.g. `base` - -### deployment & service YAML files: - -**TODO:** find a way to add those files. - -## Deploy - -```bash -# apply kustomize -kubectl apply -k ./base/ -n keycloak -``` diff --git a/docs/deployments/kubernetes-deploy.md b/docs/deployments/kubernetes-deploy.md deleted file mode 100644 index a3052bc..0000000 --- a/docs/deployments/kubernetes-deploy.md +++ /dev/null @@ -1,101 +0,0 @@ -# Kubernetes Cluster - -!!! success "Prerequisites" - - * Make sure you have at least **6** VMs configured with Centos7 - - * 1 Master node - - * 4 worker nodes - - * 1 worker node dedicated only for **vice-apps** - - * Make sure you have Ansible installed, and you can reach your VMs via `ssh` - - -# TODO rewrite the docs once [ansible-k8s-centos7](https://github.com/cyverse-austria/ansible-k8s-centos7) is ready. - -We are using **Ansible** to set up a kubernetes cluster. - -Current kubernetes cluster is configured using ansible playbooks from [deployments/ansible/kubernetes](https://github.com/cyverse-de/deployments/tree/main/ansible/kubernetes). - -# Looking to setup your own k8s cluster? - -If your are intrested to setup your own kubernetes cluster on Centos 7, please follow the steps bellow. - - -## Steps - -### Clone the repo - -```bash -# clone repo -git clone https://github.com/cyverse-de/deployments.git - -# navigate to ansible playbooks for k8s -cd /ansible/kubernetes - -``` - -### create your inventory - -Create your inventory file under `/inventory/cyverse` and replace the host names as yours. -```bash -[k8s:children] -k8s-control-plane -k8s-worker - -[kube-apiserver-haproxy] -k8s-reverse-proxy.example.com - -[k8s-control-plane] -k8s-c1.example.com - -[k8s-storage:children] -k8s-worker - -[k8s-worker] -k8s-w1.example.com -k8s-w2.example.com -k8s-w3.example.com -k8s-w4.example.com -vice-w1.example.com - - -[outward-facing-proxy] -vice-haproxy.example.com - -[haproxy] -vice-haproxy.example.com - -[vice-workers] -vice-w1.example.com -``` - -### Run playbooks -**Note:** we are using `--user root`, if your virtual machines have a diffrent user you could change this. - -#### Check if your hosts are reachable - -```bash -ansible -i inventory/ -m ping all --user root -``` - -#### Setup firewall configs - -```bash -ansible-playbook -i inventory/ firewalld-config.yml --user root -``` - -#### Provision your nodes - -```bash -ansible-playbook -i inventory/ provision-nodes.yml --user root -``` - -#### Tainting and Labeling VICE Worker Nodes - -```bash -kubectl label nodes vice-w1.example.com vice=true -kubectl taint nodes vice-w1.example.com vice=only:NoSchedule -``` diff --git a/docs/deployments/local-exim.md b/docs/deployments/local-exim.md deleted file mode 100644 index 2c5183e..0000000 --- a/docs/deployments/local-exim.md +++ /dev/null @@ -1,16 +0,0 @@ -# Local Exim (exim-sender) - -**local-exim** or also known as **exim-sender** - -!!! success "Prerequisites" - - -## Deploy - -```bash -## deploy for prod env -kubectl apply -f resources/deployments/exim-sender.yml -n prod - -## deploy for prod discover -# kubectl apply -f resources/deployments/exim-sender.yml -n discover -``` diff --git a/docs/deployments/openebs.md b/docs/deployments/openebs.md deleted file mode 100644 index 8e9fa36..0000000 --- a/docs/deployments/openebs.md +++ /dev/null @@ -1,14 +0,0 @@ -# [OpenEBS](https://openebs.io/) - -!!! success "Prerequisites" - - -## Deploy - -```bash -# create namespace -kubectl create ns openebs - -# deploy -kubectl -n openebs apply -f https://openebs.github.io/charts/openebs-operator.yaml -``` diff --git a/docs/deployments/redis-ha.md b/docs/deployments/redis-ha.md deleted file mode 100644 index ed631ac..0000000 --- a/docs/deployments/redis-ha.md +++ /dev/null @@ -1,69 +0,0 @@ -# Redis HA - -In this Document we will cover installing the **Redis Server** and **Redis Haproxy**, due to the `k8s-resources` configurations both would be installed and configured inside the **prod** namespace. - -!!! success "Prerequisites" - - * Make sure to create a **values.yaml** and replace add your credentials - - -**values.yaml** -Overrides the PersistentVolume, adds secrets and uses **openebs** as storage class. - -```yaml -## replicas number for each component -replicas: 3 - -persistentVolume: - enabled: true - ## redis-ha data Persistent Volume Storage Class - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, AWS & OpenStack) - ## - storageClass: openebs-hostpath - -## Sentinel specific configuration options -sentinel: - auth: true - authkey: - password: - -## Redis specific configuration options -auth: true -authkey: -redisPassword: -``` - -## Redis Server & Sentinel - -```bash -# add helm repo -helm repo add dandydev https://dandydeveloper.github.io/charts -helm repo update - -## deploy redis servers for prod env -helm upgrade --install --namespace prod redis-ha dandydev/redis-ha --values values.yaml - -## deploy redis server for discover env -# helm upgrade --install --namespace discover redis-ha dandydev/redis-ha --values values.yaml -``` - -# Redis Haproxy - -## Preq - -* **Make sure to load the configs/secrets** see also [k8s-resources](k8s-resources.md) - -## Deploy redis-haproxy - - -```bash -## deploy for prod env -kubectl apply -n prod -f resources/deployments/redis-haproxy.yml - -## deploy for discover env -# kubectl apply -n discover -f resources/deployments/redis-haproxy.yml -``` diff --git a/docs/deployments/unleash.md b/docs/deployments/unleash.md deleted file mode 100644 index ed7fcde..0000000 --- a/docs/deployments/unleash.md +++ /dev/null @@ -1,15 +0,0 @@ -# Unleash - -!!! success "Prerequisites" - - * Make sure `unleash` database is setup, see also [unleash-db](../database/unleash-db.md) - -## Deploy - -```bash -## Deploy for prod env -kubectl apply -f resources/deployments/unleash.yml -n prod - -## Deploy for discover env -# kubectl apply -f resources/deployments/unleash.yml -n discover -``` diff --git a/docs/deployments/userportal.md b/docs/deployments/userportal.md deleted file mode 100644 index 673b8e9..0000000 --- a/docs/deployments/userportal.md +++ /dev/null @@ -1,93 +0,0 @@ -# User Portal - -!!! success "Prerequisites" - - -### TODO: add source location. - -### LDAP -Setup required ldap user. - -#### create portal user -##### Create portal-user.ldif - -```ldif -dn: uid=portal,ou=People,dc=tugraz,dc=at -objectClass: inetOrgPerson -objectClass: posixAccount -objectClass: shadowAccount -uid: portal -mail: portal@example.com -sn: SeviceAccount -givenName: PORTAL -cn: portal -title: Other -o: N/A -departmentNumber: N/A -uidNumber: 40003 -gidNumber: 10003 -homeDirectory: /home/portal -``` - -##### Apply ldap file - -```bash -# replace LDAP_PASSWORD -ldapadd -x -D "cn=Manager,dc=tugraz,dc=at" -w "LDAP_PASSWORD" -f portal-user.ldif - -# create a password for the user -## replace PORTAL_PASSWORD & LDAP_PASSWORD -ldappasswd -x -D cn=Manager,dc=tugraz,dc=at -w "LDAP_PASSWORD" -s "PORTAL_PASSWORD" "uid=portal,ou=People,dc=tugraz,dc=at" -``` - -#### Add portal user to admim group - -##### Create portal-de_admin.ldif - -```ldif -dn: cn=de_admins,ou=Groups,dc=tugraz,dc=at -changetype: modify -add: memberUid -memberUid: portal -``` - -##### Apply ldap file - -```bash -# replace LDAP_PASSWORD -## add user to the de_admin group -ldapmodify -x -D "cn=Manager,dc=tugraz,dc=at" -w "LDAP_PASSWORD" -f portal-de_admin.ldif -``` - - -### iRODs - -For user portal we would need to create an irods `rodsadmin` user. - -```bash -# replace YOURPASSWORDHERE -## su - irods -iadmin mkuser portal rodsadmin -iadmin moduser portal password YOURPASSWORDHERE -``` - -### Database - -For Database configuration and setup vist [portal-db docs](../database/portal-db.md). - -### Images - -1. harbor.cyverse.org/hub/library/nginx:1.20-alpine **Replaced with**:nginx:1.20-alpine - -2. mbwali/portal:tug-stable - * Todo: tekton should build this via the URL. - -## Deploy - -```bash -# create ns -kubectl create ns user-portal - -# deploy -kubectl apply -k portal/user-portal/base -n user-portal -``` diff --git a/docs/deployments/vice.md b/docs/deployments/vice.md deleted file mode 100644 index 441734d..0000000 --- a/docs/deployments/vice.md +++ /dev/null @@ -1,103 +0,0 @@ -# vice apps - -!!! success "Prerequisites" - - -## Create namespace -```bash -kubectl create ns vice-apps -``` - -## Create a secret on namespace vice-apps `vice-image-pull-secret` - -This secret has the `harbor.org` which allows the pod to pull docker images. - -```bash -kubectl create secret generic vice-image-pull-secret \ - --from-file=.dockerconfigjson=/root/.docker/config.json \ - --type=kubernetes.io/dockerconfigjson -n vice-apps -``` - -## Create serviceAccounts - -```bash -## edit the namespace if you are running in a diffrent env -## vi /k8s-resources/resources/serviceaccounts/app-exposer.yml -kubectl apply -f /k8s-resources/resources/serviceaccounts/app-exposer.yml -kubectl apply -f /k8s-resources/resources/serviceaccounts/vice-app-runner.yml -``` - -## Apply clusterrolebindings - -```bash -## edit the namespace if you are running in a diffrent env -## vi /k8s-resources/resources/clusterrolebindings/app-exposer.yml -kubectl apply -f /k8s-resources/resources/clusterrolebindings/app-exposer.yml -``` - -## Apply networkpolicies - -To apply networkpolicies we need to edit the file `/k8s-resources/resources/networkpolicies/vice-apps.yml`, and add all the worker nodes, and master nodes, to it if we are using a diffrent **env** rather than -**prod**. - -**e.g.** - -```diff - -- except: - - ******** - - ******** - -+ except: - - 10.0.10.0/24 # k8s master CIDR - - ****************/32 # c1 - - ****************/32 # w1 - - ***************/32 # w2 - - ***************/32 # w3 - - **************/32 # w4 - - *************/32 # w5 - - ************/32 # vice-w1 -``` - -**Run policy** - -```bash -kubectl apply -f /k8s-resources/resources/networkpolicies/vice-apps.yml -``` - -## Apply roles - -```bash -kubectl apply -f /k8s-resources/resources/roles/vice-apps.yml -``` - -## Create porklock-config secrert for irods - -### Create irods-config.properties - -Create a file `irods-config.properties` and add the values. - -```properties -porklock.irods-home= -porklock.irods-user= -porklock.irods-pass= -porklock.irods-host= -porklock.irods-port= -porklock.irods-zone= -porklock.irods-resc= -``` - -### Create secret from file - -```bash -kubectl -n vice-apps create secret generic porklock-config --from-file=irods-config.properties -``` - -## Restart the services -```bash -kubectl rollout restart apps app-exposer templeton-incremental templeton-periodic -n NAMESPACE -``` - -## Install/configure ingress-nginx - -For installing and configuring the **ingress-nginx** have a look at [ingress-nginx](ingress-nginx.md) diff --git a/docs/development/developer-guide.md b/docs/development/developer-guide.md new file mode 100644 index 0000000..759f501 --- /dev/null +++ b/docs/development/developer-guide.md @@ -0,0 +1,558 @@ +--- +type: Guide +title: "Developer guide" +description: "Development environment, service layout, and contribution workflow for CyVerse code." +tags: [development, contributing] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- +# Developer Guide + +Welcome to the CyVerse Developer Guide. This documentation is for developers who want to: + +- Integrate applications with CyVerse APIs +- Contribute code to the CyVerse platform +- Build custom tools and services on top of CyVerse infrastructure +- Understand CyVerse architecture for research or collaboration + +## Who This Guide Is For + +**API Integrators**: You want to connect your application to CyVerse's Data Store, submit computational jobs, or leverage CyVerse authentication. + +**Platform Contributors**: You want to contribute code, fix bugs, or add features to CyVerse's open-source codebase. + +**Tool Developers**: You're packaging scientific software to run in the Discovery Environment or VICE. + +**Researchers & Data Scientists**: You want to automate workflows, build custom interfaces, or extend CyVerse functionality for your research needs. + +--- + +## Getting Started + +### Development Environment Setup + +#### Required Tools + +Before you begin, install these tools: + +```bash +# Version control +git --version # Git 2.30+ + +# Container tools +docker --version # Docker 20.10+ +docker-compose --version # Docker Compose 1.29+ + +# Kubernetes tools (for platform development) +kubectl version # kubectl 1.23+ +helm version # Helm 3.8+ + +# Programming languages (depending on services) +python --version # Python 3.9+ +go version # Go 1.19+ (for DE services) +node --version # Node.js 16+ (for UI development) +``` + +#### Get Access + +1. **CyVerse Account**: Create a free account at [https://user.cyverse.org](https://user.cyverse.org){target=_blank} +2. **GitHub Access**: Most CyVerse code is at [github.com/cyverse-de](https://github.com/cyverse-de){target=_blank} +3. **API Credentials**: Generate API tokens through the Discovery Environment or User Portal +4. **Development Environment**: Set up a test environment or use the public CyVerse production instance + +#### Development Accounts + +For testing and development: + +- **Production**: `https://de.cyverse.org` - Use for API testing with real data (be cautious!) +- **Development/Staging**: Contact CyVerse team for access to dev environments +- **Local Development**: Set up local services using Docker Compose (see repository README files) + +--- + +## CyVerse Architecture Overview + +Understanding CyVerse's architecture will help you integrate effectively or contribute meaningfully. + +### System Design + +CyVerse follows a **microservices architecture** with services communicating via REST APIs and message queues: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ Users & Applications │ +└───────────────────────┬─────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Frontend Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ Discovery │ │ User Portal │ │ Custom Apps │ │ +│ │ Environment │ │ │ │ (Your Code) │ │ +│ │ (Sonora UI) │ │ │ │ │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└───────────────────────┬─────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ API Gateway Layer │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ Terrain API (Aggregator) │ │ +│ │ https://de.cyverse.org/terrain/docs │ │ +│ └───────────────────────────────────────────────────────┘ │ +└───────────────────────┬─────────────────────────────────────┘ + │ + ┌───────────────┼───────────────┐ + │ │ │ + ▼ ▼ ▼ +┌──────────────┐ ┌──────────────┐ ┌──────────────┐ +│ Microservices Layer │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ apps │ │ data-info│ │ analyses │ │ +│ │ │ │ │ │ │ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ metadata │ │ notif- │ │ permiss- │ │ +│ │ │ │ ications │ │ ions │ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +└──────────────────────────────────────────────────┘ + │ │ │ + └───────────────┼───────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ Infrastructure Layer │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ PostgreSQL │ │ RabbitMQ │ │ iRODS │ │ +│ │ (Databases) │ │ (Messages) │ │ (Storage) │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ ElasticSrch │ │ KeyCloak │ │ Kubernetes │ │ +│ │ (Search) │ │ (Auth) │ │ (Orchestr.) │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +└─────────────────────────────────────────────────────────────┘ +``` + +### Key Components + +| Component | Technology | Purpose | +|-----------|------------|---------| +| **Terrain** | Clojure | API gateway aggregating all DE services | +| **apps** | Go | App/tool metadata and management | +| **data-info** | Clojure | Data Store filesystem operations | +| **analyses** | Go | Job submission and monitoring | +| **metadata** | Go | User-defined metadata (AVUs) | +| **notifications** | Go | User notification system | +| **permissions** | Go | Fine-grained permission management | +| **Sonora** | React/TypeScript | Modern DE web UI | +| **iRODS** | C++/Python | Data storage backend | +| **KeyCloak** | Java | Authentication and identity | + +### Service Interactions + +**Example: Submitting a Job** + +1. User clicks "Run" in Sonora UI +2. Sonora calls Terrain API: `POST /terrain/analyses` +3. Terrain validates request and calls `analyses` service +4. `analyses` service: + - Validates app exists (via `apps` service) + - Checks user permissions (via `permissions` service) + - Stages input data (via `data-info` service) + - Submits job to HTCondor or Kubernetes + - Sends notification (via `notifications` service) + - Publishes events to RabbitMQ +5. Job status updates flow back through the same chain + +--- + +## API Integration + +### Authentication + +CyVerse uses **OAUTH 2.0** via KeyCloak for authentication. + +#### Getting an Access Token + +**Method 1: Interactive (Browser-based)** + +Use OAUTH 2.0 authorization code flow: + +```python +import requests + +# Step 1: Direct user to authorization URL +auth_url = "https://kc.cyverse.org/auth/realms/CyVerse/protocol/openid-connect/auth" +params = { + "client_id": "your-client-id", + "redirect_uri": "https://yourapp.com/callback", + "response_type": "code", + "scope": "openid" +} +# User visits this URL and authorizes + +# Step 2: Exchange authorization code for token +token_url = "https://kc.cyverse.org/auth/realms/CyVerse/protocol/openid-connect/token" +token_response = requests.post(token_url, data={ + "grant_type": "authorization_code", + "code": "authorization-code-from-callback", + "redirect_uri": "https://yourapp.com/callback", + "client_id": "your-client-id", + "client_secret": "your-client-secret" +}) +access_token = token_response.json()["access_token"] +``` + +**Method 2: Command-line (for scripts)** + +For automated scripts, use resource owner password credentials flow (requires user consent): + +```bash +#!/bin/bash +# Get access token for scripting + +USERNAME="your-username" +PASSWORD="your-password" +CLIENT_ID="de-client" + +TOKEN=$(curl -s -X POST \ + https://kc.cyverse.org/auth/realms/CyVerse/protocol/openid-connect/token \ + -d "grant_type=password" \ + -d "client_id=${CLIENT_ID}" \ + -d "username=${USERNAME}" \ + -d "password=${PASSWORD}" \ + | jq -r '.access_token') + +echo $TOKEN +``` + +#### Using the Access Token + +Include the token in the `Authorization` header: + +```bash +curl -H "Authorization: Bearer ${TOKEN}" \ + https://de.cyverse.org/terrain/filesystem/list +``` + +### Making API Calls + +#### Example: List Files + +```python +import requests + +def list_directory(access_token, path): + """List files in a Data Store directory.""" + url = f"https://de.cyverse.org/terrain/secured/filesystem/paged-directory" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + params = { + "path": path, + "limit": 100, + "offset": 0 + } + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + return response.json() + +# Usage +files = list_directory(access_token, "/iplant/home/username") +for item in files["files"]: + print(f"{item['label']} ({item['file-size']} bytes)") +``` + +#### Example: Submit a Job + +```python +def submit_analysis(access_token, app_id, input_files, parameters): + """Submit a job to run an analysis.""" + url = "https://de.cyverse.org/terrain/analyses" + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + } + payload = { + "app_id": app_id, + "name": "My Analysis", + "description": "Running analysis via API", + "notify": True, + "config": { + "input1": input_files, + "param1": parameters + } + } + + response = requests.post(url, headers=headers, json=payload) + response.raise_for_status() + return response.json() + +# Usage +result = submit_analysis( + access_token, + app_id="12345678-1234-1234-1234-123456789abc", + input_files=["/iplant/home/username/data.txt"], + parameters={"threshold": 0.05} +) +print(f"Job submitted: {result['id']}") +``` + +#### Example: Upload a File + +```python +def upload_file(access_token, local_file, dest_path): + """Upload a file to the Data Store.""" + url = "https://de.cyverse.org/terrain/secured/fileio/upload" + headers = { + "Authorization": f"Bearer {access_token}" + } + files = { + "file": open(local_file, "rb") + } + data = { + "dest": dest_path + } + + response = requests.post(url, headers=headers, files=files, data=data) + response.raise_for_status() + return response.json() + +# Usage +upload_file(access_token, "local-data.csv", "/iplant/home/username/uploads/") +``` + +### API Resources + +- **[Live API Documentation](https://de.cyverse.org/terrain/docs/){target=_blank}** - Interactive Swagger UI +- **[Terrain Endpoint Reference](../api/endpoint-index.md)** - Comprehensive endpoint list +- **[API Error Codes](../api/errors.md)** - Error handling guide +- **[Filesystem API](../api/endpoints/filesystem/directory-listing.md)** - Data Store operations +- **[Tapis Migration Guide](../api/tapis-v2-v3-migration.md)** - Upgrading from Tapis v2 to v3 + +--- + +## Building Custom Applications + +### Best Practices + +1. **Use Access Tokens Securely** + - Never commit tokens to version control + - Use environment variables or secure vaults + - Implement token refresh logic + +2. **Handle Rate Limits** + - Implement exponential backoff for retries + - Cache responses when appropriate + - Use bulk operations when available + +3. **Error Handling** + - Check HTTP status codes + - Parse error responses for details + - Provide user-friendly error messages + +4. **Respect User Privacy** + - Only access data the user has authorized + - Follow CyVerse data policies + - Implement proper permission checks + +### Example: Python SDK Wrapper + +Create a simple SDK wrapper for common operations: + +```python +# cyverse_sdk.py +import requests +from typing import List, Dict + +class CyVerseClient: + def __init__(self, access_token: str, base_url: str = "https://de.cyverse.org/terrain"): + self.access_token = access_token + self.base_url = base_url + self.session = requests.Session() + self.session.headers.update({ + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json" + }) + + def list_files(self, path: str, limit: int = 100) -> List[Dict]: + """List files in a directory.""" + url = f"{self.base_url}/secured/filesystem/paged-directory" + response = self.session.get(url, params={"path": path, "limit": limit}) + response.raise_for_status() + return response.json()["files"] + + def search_data(self, query: str) -> List[Dict]: + """Search for data in the Data Store.""" + url = f"{self.base_url}/secured/filesystem/search" + response = self.session.post(url, json={"query": query}) + response.raise_for_status() + return response.json()["files"] + + def get_analyses(self, limit: int = 10) -> List[Dict]: + """Get user's recent analyses.""" + url = f"{self.base_url}/analyses" + response = self.session.get(url, params={"limit": limit}) + response.raise_for_status() + return response.json()["analyses"] + +# Usage +client = CyVerseClient(access_token="your-token") +files = client.list_files("/iplant/home/username") +``` + +--- + +## Contributing to CyVerse + +### GitHub Workflow + +CyVerse development follows standard Git/GitHub workflows: + +1. **Fork the Repository** + ```bash + # Navigate to the service repository on GitHub and click "Fork" + git clone https://github.com/YOUR-USERNAME/repo-name.git + cd repo-name + git remote add upstream https://github.com/cyverse-de/repo-name.git + ``` + +2. **Create a Feature Branch** + ```bash + git checkout -b feature/your-feature-name + ``` + +3. **Make Changes and Commit** + ```bash + git add . + git commit -m "Add feature: description of changes" + ``` + +4. **Push and Create Pull Request** + ```bash + git push origin feature/your-feature-name + # Then create a Pull Request on GitHub + ``` + +### Code Review Process + +1. Submit a Pull Request with: + - Clear description of changes + - Reference to any related issues + - Screenshots for UI changes + - Test results + +2. Automated checks will run: + - Unit tests + - Linting/formatting checks + - Security scans + +3. Maintainers will review your code: + - Respond to feedback promptly + - Make requested changes + - Engage in technical discussions + +4. Once approved, maintainers will merge your PR + +### Testing Requirements + +All contributions should include appropriate tests: + +**Go Services:** +```bash +# Run unit tests +go test ./... + +# Run with coverage +go test -cover ./... +``` + +**Clojure Services:** +```bash +# Run tests with Leiningen +lein test +``` + +**React/TypeScript UI:** +```bash +# Run Jest tests +npm test + +# Run with coverage +npm test -- --coverage +``` + +### Development Standards + +- **Code Style**: Follow existing code style in the repository +- **Documentation**: Update README and inline comments for significant changes +- **Commit Messages**: Use clear, descriptive commit messages +- **Breaking Changes**: Discuss breaking changes with maintainers before implementing + +--- + +## CyVerse Repositories + +### Core Services + +| Repository | Language | Description | +|------------|----------|-------------| +| [terrain](https://github.com/cyverse-de/terrain){target=_blank} | Clojure | Main API gateway | +| [apps](https://github.com/cyverse-de/apps){target=_blank} | Go | App metadata service | +| [data-info](https://github.com/cyverse-de/data-info){target=_blank} | Clojure | Data Store operations | +| [analyses](https://github.com/cyverse-de/analyses){target=_blank} | Go | Job management | +| [sonora](https://github.com/cyverse-de/sonora){target=_blank} | React/TS | Modern DE UI | + +### Infrastructure + +| Repository | Language | Description | +|------------|----------|-------------| +| [deployments](https://github.com/cyverse-de/deployments){target=_blank} | YAML/HCL | Kubernetes manifests | +| [ansible](https://github.com/cyverse/ansible){target=_blank} | Ansible | Configuration management | + +### Tools & Utilities + +| Repository | Language | Description | +|------------|----------|-------------| +| [gocommands](https://github.com/cyverse/gocommands){target=_blank} | Go | iRODS CLI tools | +| [irods-csi-driver](https://github.com/cyverse/irods-csi-driver){target=_blank} | Go | Kubernetes iRODS integration | + +Browse the full list at [github.com/cyverse-de](https://github.com/cyverse-de){target=_blank} + +--- + +## Community & Support + +### Get Help + +- **Documentation**: You're reading it! Search this site for answers +- **GitHub Issues**: Report bugs or request features in the relevant repository +- **CyVerse Support**: [support.cyverse.org](https://cyverse.org/help){target=_blank} + +### Stay Updated + +- **GitHub**: Watch repositories for updates +- **CyVerse Blog**: [cyverse.org/blog](https://cyverse.org){target=_blank} + +### Contributing Beyond Code + +Not a developer? You can still contribute: + +- Report bugs you encounter +- Suggest feature improvements +- Write documentation +- Answer questions in forums +- Share your CyVerse integration as an example + +--- + +## Additional Resources + +- **[API Overview](../api/terrain.md)** - Detailed Terrain API documentation +- **[Platform services](../platform/index.md)** - What CyVerse offers, service by service +- **[Deployment Guide](../deployment/index.md)** - For platform operators +- **[Getting Started](../deployment/planning/prerequisites.md)** - Prerequisites and initial setup + +**Ready to build?** Start with the [API Endpoint Index](../api/endpoint-index.md) to explore available functionality, or dive into the [live API documentation](https://de.cyverse.org/terrain/docs/){target=_blank} for interactive testing. diff --git a/docs/development/index.md b/docs/development/index.md index 24dedcd..f6a4530 100644 --- a/docs/development/index.md +++ b/docs/development/index.md @@ -1,550 +1,9 @@ -# Developer Guide +# Development -Welcome to the CyVerse Developer Guide. This documentation is for developers who want to: +* [Developer guide](developer-guide.md) - development environment, service layout, and contribution workflow -- Integrate applications with CyVerse APIs -- Contribute code to the CyVerse platform -- Build custom tools and services on top of CyVerse infrastructure -- Understand CyVerse architecture for research or collaboration +# Related -## Who This Guide Is For - -**API Integrators**: You want to connect your application to CyVerse's Data Store, submit computational jobs, or leverage CyVerse authentication. - -**Platform Contributors**: You want to contribute code, fix bugs, or add features to CyVerse's open-source codebase. - -**Tool Developers**: You're packaging scientific software to run in the Discovery Environment or VICE. - -**Researchers & Data Scientists**: You want to automate workflows, build custom interfaces, or extend CyVerse functionality for your research needs. - ---- - -## Getting Started - -### Development Environment Setup - -#### Required Tools - -Before you begin, install these tools: - -```bash -# Version control -git --version # Git 2.30+ - -# Container tools -docker --version # Docker 20.10+ -docker-compose --version # Docker Compose 1.29+ - -# Kubernetes tools (for platform development) -kubectl version # kubectl 1.23+ -helm version # Helm 3.8+ - -# Programming languages (depending on services) -python --version # Python 3.9+ -go version # Go 1.19+ (for DE services) -node --version # Node.js 16+ (for UI development) -``` - -#### Get Access - -1. **CyVerse Account**: Create a free account at [https://user.cyverse.org](https://user.cyverse.org){target=_blank} -2. **GitHub Access**: Most CyVerse code is at [github.com/cyverse-de](https://github.com/cyverse-de){target=_blank} -3. **API Credentials**: Generate API tokens through the Discovery Environment or User Portal -4. **Development Environment**: Set up a test environment or use the public CyVerse production instance - -#### Development Accounts - -For testing and development: - -- **Production**: `https://de.cyverse.org` - Use for API testing with real data (be cautious!) -- **Development/Staging**: Contact CyVerse team for access to dev environments -- **Local Development**: Set up local services using Docker Compose (see repository README files) - ---- - -## CyVerse Architecture Overview - -Understanding CyVerse's architecture will help you integrate effectively or contribute meaningfully. - -### System Design - -CyVerse follows a **microservices architecture** with services communicating via REST APIs and message queues: - -``` -┌─────────────────────────────────────────────────────────────┐ -│ Users & Applications │ -└───────────────────────┬─────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Frontend Layer │ -│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ Discovery │ │ User Portal │ │ Custom Apps │ │ -│ │ Environment │ │ │ │ (Your Code) │ │ -│ │ (Sonora UI) │ │ │ │ │ │ -│ └──────────────┘ └──────────────┘ └──────────────┘ │ -└───────────────────────┬─────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ API Gateway Layer │ -│ ┌───────────────────────────────────────────────────────┐ │ -│ │ Terrain API (Aggregator) │ │ -│ │ https://de.cyverse.org/terrain/docs │ │ -│ └───────────────────────────────────────────────────────┘ │ -└───────────────────────┬─────────────────────────────────────┘ - │ - ┌───────────────┼───────────────┐ - │ │ │ - ▼ ▼ ▼ -┌──────────────┐ ┌──────────────┐ ┌──────────────┐ -│ Microservices Layer │ -│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ -│ │ apps │ │ data-info│ │ analyses │ │ -│ │ │ │ │ │ │ │ -│ └──────────┘ └──────────┘ └──────────┘ │ -│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ -│ │ metadata │ │ notif- │ │ permiss- │ │ -│ │ │ │ ications │ │ ions │ │ -│ └──────────┘ └──────────┘ └──────────┘ │ -└──────────────────────────────────────────────────┘ - │ │ │ - └───────────────┼───────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────┐ -│ Infrastructure Layer │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ PostgreSQL │ │ RabbitMQ │ │ iRODS │ │ -│ │ (Databases) │ │ (Messages) │ │ (Storage) │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ -│ │ ElasticSrch │ │ KeyCloak │ │ Kubernetes │ │ -│ │ (Search) │ │ (Auth) │ │ (Orchestr.) │ │ -│ └─────────────┘ └─────────────┘ └─────────────┘ │ -└─────────────────────────────────────────────────────────────┘ -``` - -### Key Components - -| Component | Technology | Purpose | -|-----------|------------|---------| -| **Terrain** | Clojure | API gateway aggregating all DE services | -| **apps** | Go | App/tool metadata and management | -| **data-info** | Clojure | Data Store filesystem operations | -| **analyses** | Go | Job submission and monitoring | -| **metadata** | Go | User-defined metadata (AVUs) | -| **notifications** | Go | User notification system | -| **permissions** | Go | Fine-grained permission management | -| **Sonora** | React/TypeScript | Modern DE web UI | -| **iRODS** | C++/Python | Data storage backend | -| **KeyCloak** | Java | Authentication and identity | - -### Service Interactions - -**Example: Submitting a Job** - -1. User clicks "Run" in Sonora UI -2. Sonora calls Terrain API: `POST /terrain/analyses` -3. Terrain validates request and calls `analyses` service -4. `analyses` service: - - Validates app exists (via `apps` service) - - Checks user permissions (via `permissions` service) - - Stages input data (via `data-info` service) - - Submits job to HTCondor or Kubernetes - - Sends notification (via `notifications` service) - - Publishes events to RabbitMQ -5. Job status updates flow back through the same chain - ---- - -## API Integration - -### Authentication - -CyVerse uses **OAUTH 2.0** via KeyCloak for authentication. - -#### Getting an Access Token - -**Method 1: Interactive (Browser-based)** - -Use OAUTH 2.0 authorization code flow: - -```python -import requests - -# Step 1: Direct user to authorization URL -auth_url = "https://kc.cyverse.org/auth/realms/CyVerse/protocol/openid-connect/auth" -params = { - "client_id": "your-client-id", - "redirect_uri": "https://yourapp.com/callback", - "response_type": "code", - "scope": "openid" -} -# User visits this URL and authorizes - -# Step 2: Exchange authorization code for token -token_url = "https://kc.cyverse.org/auth/realms/CyVerse/protocol/openid-connect/token" -token_response = requests.post(token_url, data={ - "grant_type": "authorization_code", - "code": "authorization-code-from-callback", - "redirect_uri": "https://yourapp.com/callback", - "client_id": "your-client-id", - "client_secret": "your-client-secret" -}) -access_token = token_response.json()["access_token"] -``` - -**Method 2: Command-line (for scripts)** - -For automated scripts, use resource owner password credentials flow (requires user consent): - -```bash -#!/bin/bash -# Get access token for scripting - -USERNAME="your-username" -PASSWORD="your-password" -CLIENT_ID="de-client" - -TOKEN=$(curl -s -X POST \ - https://kc.cyverse.org/auth/realms/CyVerse/protocol/openid-connect/token \ - -d "grant_type=password" \ - -d "client_id=${CLIENT_ID}" \ - -d "username=${USERNAME}" \ - -d "password=${PASSWORD}" \ - | jq -r '.access_token') - -echo $TOKEN -``` - -#### Using the Access Token - -Include the token in the `Authorization` header: - -```bash -curl -H "Authorization: Bearer ${TOKEN}" \ - https://de.cyverse.org/terrain/filesystem/list -``` - -### Making API Calls - -#### Example: List Files - -```python -import requests - -def list_directory(access_token, path): - """List files in a Data Store directory.""" - url = f"https://de.cyverse.org/terrain/secured/filesystem/paged-directory" - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json" - } - params = { - "path": path, - "limit": 100, - "offset": 0 - } - - response = requests.get(url, headers=headers, params=params) - response.raise_for_status() - return response.json() - -# Usage -files = list_directory(access_token, "/iplant/home/username") -for item in files["files"]: - print(f"{item['label']} ({item['file-size']} bytes)") -``` - -#### Example: Submit a Job - -```python -def submit_analysis(access_token, app_id, input_files, parameters): - """Submit a job to run an analysis.""" - url = "https://de.cyverse.org/terrain/analyses" - headers = { - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json" - } - payload = { - "app_id": app_id, - "name": "My Analysis", - "description": "Running analysis via API", - "notify": True, - "config": { - "input1": input_files, - "param1": parameters - } - } - - response = requests.post(url, headers=headers, json=payload) - response.raise_for_status() - return response.json() - -# Usage -result = submit_analysis( - access_token, - app_id="12345678-1234-1234-1234-123456789abc", - input_files=["/iplant/home/username/data.txt"], - parameters={"threshold": 0.05} -) -print(f"Job submitted: {result['id']}") -``` - -#### Example: Upload a File - -```python -def upload_file(access_token, local_file, dest_path): - """Upload a file to the Data Store.""" - url = "https://de.cyverse.org/terrain/secured/fileio/upload" - headers = { - "Authorization": f"Bearer {access_token}" - } - files = { - "file": open(local_file, "rb") - } - data = { - "dest": dest_path - } - - response = requests.post(url, headers=headers, files=files, data=data) - response.raise_for_status() - return response.json() - -# Usage -upload_file(access_token, "local-data.csv", "/iplant/home/username/uploads/") -``` - -### API Resources - -- **[Live API Documentation](https://de.cyverse.org/terrain/docs/){target=_blank}** - Interactive Swagger UI -- **[Terrain Endpoint Reference](../services/api/endpoint-index.md)** - Comprehensive endpoint list -- **[API Error Codes](../services/api/errors.md)** - Error handling guide -- **[Filesystem API](../services/api/endpoints/filesystem/directory-listing.md)** - Data Store operations -- **[Tapis Migration Guide](../services/api/tapis-v2-v3-migration.md)** - Upgrading from Tapis v2 to v3 - ---- - -## Building Custom Applications - -### Best Practices - -1. **Use Access Tokens Securely** - - Never commit tokens to version control - - Use environment variables or secure vaults - - Implement token refresh logic - -2. **Handle Rate Limits** - - Implement exponential backoff for retries - - Cache responses when appropriate - - Use bulk operations when available - -3. **Error Handling** - - Check HTTP status codes - - Parse error responses for details - - Provide user-friendly error messages - -4. **Respect User Privacy** - - Only access data the user has authorized - - Follow CyVerse data policies - - Implement proper permission checks - -### Example: Python SDK Wrapper - -Create a simple SDK wrapper for common operations: - -```python -# cyverse_sdk.py -import requests -from typing import List, Dict - -class CyVerseClient: - def __init__(self, access_token: str, base_url: str = "https://de.cyverse.org/terrain"): - self.access_token = access_token - self.base_url = base_url - self.session = requests.Session() - self.session.headers.update({ - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json" - }) - - def list_files(self, path: str, limit: int = 100) -> List[Dict]: - """List files in a directory.""" - url = f"{self.base_url}/secured/filesystem/paged-directory" - response = self.session.get(url, params={"path": path, "limit": limit}) - response.raise_for_status() - return response.json()["files"] - - def search_data(self, query: str) -> List[Dict]: - """Search for data in the Data Store.""" - url = f"{self.base_url}/secured/filesystem/search" - response = self.session.post(url, json={"query": query}) - response.raise_for_status() - return response.json()["files"] - - def get_analyses(self, limit: int = 10) -> List[Dict]: - """Get user's recent analyses.""" - url = f"{self.base_url}/analyses" - response = self.session.get(url, params={"limit": limit}) - response.raise_for_status() - return response.json()["analyses"] - -# Usage -client = CyVerseClient(access_token="your-token") -files = client.list_files("/iplant/home/username") -``` - ---- - -## Contributing to CyVerse - -### GitHub Workflow - -CyVerse development follows standard Git/GitHub workflows: - -1. **Fork the Repository** - ```bash - # Navigate to the service repository on GitHub and click "Fork" - git clone https://github.com/YOUR-USERNAME/repo-name.git - cd repo-name - git remote add upstream https://github.com/cyverse-de/repo-name.git - ``` - -2. **Create a Feature Branch** - ```bash - git checkout -b feature/your-feature-name - ``` - -3. **Make Changes and Commit** - ```bash - git add . - git commit -m "Add feature: description of changes" - ``` - -4. **Push and Create Pull Request** - ```bash - git push origin feature/your-feature-name - # Then create a Pull Request on GitHub - ``` - -### Code Review Process - -1. Submit a Pull Request with: - - Clear description of changes - - Reference to any related issues - - Screenshots for UI changes - - Test results - -2. Automated checks will run: - - Unit tests - - Linting/formatting checks - - Security scans - -3. Maintainers will review your code: - - Respond to feedback promptly - - Make requested changes - - Engage in technical discussions - -4. Once approved, maintainers will merge your PR - -### Testing Requirements - -All contributions should include appropriate tests: - -**Go Services:** -```bash -# Run unit tests -go test ./... - -# Run with coverage -go test -cover ./... -``` - -**Clojure Services:** -```bash -# Run tests with Leiningen -lein test -``` - -**React/TypeScript UI:** -```bash -# Run Jest tests -npm test - -# Run with coverage -npm test -- --coverage -``` - -### Development Standards - -- **Code Style**: Follow existing code style in the repository -- **Documentation**: Update README and inline comments for significant changes -- **Commit Messages**: Use clear, descriptive commit messages -- **Breaking Changes**: Discuss breaking changes with maintainers before implementing - ---- - -## CyVerse Repositories - -### Core Services - -| Repository | Language | Description | -|------------|----------|-------------| -| [terrain](https://github.com/cyverse-de/terrain){target=_blank} | Clojure | Main API gateway | -| [apps](https://github.com/cyverse-de/apps){target=_blank} | Go | App metadata service | -| [data-info](https://github.com/cyverse-de/data-info){target=_blank} | Clojure | Data Store operations | -| [analyses](https://github.com/cyverse-de/analyses){target=_blank} | Go | Job management | -| [sonora](https://github.com/cyverse-de/sonora){target=_blank} | React/TS | Modern DE UI | - -### Infrastructure - -| Repository | Language | Description | -|------------|----------|-------------| -| [deployments](https://github.com/cyverse-de/deployments){target=_blank} | YAML/HCL | Kubernetes manifests | -| [ansible](https://github.com/cyverse/ansible){target=_blank} | Ansible | Configuration management | - -### Tools & Utilities - -| Repository | Language | Description | -|------------|----------|-------------| -| [gocommands](https://github.com/cyverse/gocommands){target=_blank} | Go | iRODS CLI tools | -| [irods-csi-driver](https://github.com/cyverse/irods-csi-driver){target=_blank} | Go | Kubernetes iRODS integration | - -Browse the full list at [github.com/cyverse-de](https://github.com/cyverse-de){target=_blank} - ---- - -## Community & Support - -### Get Help - -- **Documentation**: You're reading it! Search this site for answers -- **GitHub Issues**: Report bugs or request features in the relevant repository -- **CyVerse Support**: [support.cyverse.org](https://cyverse.org/help){target=_blank} - -### Stay Updated - -- **GitHub**: Watch repositories for updates -- **CyVerse Blog**: [cyverse.org/blog](https://cyverse.org){target=_blank} - -### Contributing Beyond Code - -Not a developer? You can still contribute: - -- Report bugs you encounter -- Suggest feature improvements -- Write documentation -- Answer questions in forums -- Share your CyVerse integration as an example - ---- - -## Additional Resources - -- **[API Overview](../services/api_overview.md)** - Detailed Terrain API documentation -- **[System Architecture](../services/services_overview.md)** - Deep dive into CyVerse design -- **[Deployment Guide](../deployments/deployment_overview.md)** - For platform operators -- **[Getting Started](../services/getting_started.md)** - Prerequisites and initial setup - -**Ready to build?** Start with the [API Endpoint Index](../services/api/endpoint-index.md) to explore available functionality, or dive into the [live API documentation](https://de.cyverse.org/terrain/docs/){target=_blank} for interactive testing. +* [Terrain API](../api/terrain.md) +* [Cluster resources](../deployment/04-kubernetes/resources.md) - how services are configured and deployed +* [Docker](../deployment/planning/docker.md) diff --git a/docs/guides/devops.md b/docs/guides/devops.md deleted file mode 100644 index 8adfa1e..0000000 --- a/docs/guides/devops.md +++ /dev/null @@ -1,57 +0,0 @@ -[Ansible](http://www.ansible.com/) is an open source, agentless automation tool. The DE development team uses Ansible to provision/update our servers and deploy the DE. However, for this repository, we only expose the Ansible scripts that we use for deploying the DE. - -If you intend to use our ansible scripts, we highly suggest that you read the [Ansible documentation](http://docs.ansible.com/ansible/index.html). - -## Installations - -[:simple-ansible: Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html){target=_blank} - -[:simple-docker: Docker](https://docs.docker.com/engine/install/){target=_blank} - -[:simple-kubernetes: Kubernetes (K8s)](https://kubernetes.io/docs/tasks/tools/){target=_blank} - `kubectl` for managing K8s clusters - -[:simple-terraform: Terraform](https://developer.hashicorp.com/terraform/downloads){target=_blank} - -[:simple-visualstudiocode: VS Code Kubernetes Extension](https://code.visualstudio.com/docs/azure/kubernetes){target=_blank} - optional (recommended) - -## Setup - -* [Ansible Setup](setup/ansible.md) - -* [Docker-based Ansible Setup](setup/docker.md) - -## Design - -We have strived to follow Ansible's [best practices](http://docs.ansible.com/ansible/playbooks_best_practices.html). - -However, we have slightly diverged on the topics of [directory layout](http://docs.ansible.com/ansible/playbooks_best_practices.html#directory-layout) and [role-separated top level playbooks](http://docs.ansible.com/ansible/playbooks_best_practices.html#top-level-playbooks-are-separated-by-role). - -## Directory Layout - -The ansible best practices for directory layout suggests using a `group_vars` folder, but you may have noticed that the repo doesn't contain a `ansible/group_vars` folder. Our default `group_vars` folder resides in the `inventories` folder `ansible/inventories/group_vars`. - -Also, this folder contains a single file, `all`, which contains all of the variables used by the provided roles and playbooks with default values. - -This is done with the intent that developers will create their own `group_vars/all` file, which will override any of the defaults set in the `inventories/group_vars/all` file. - -You only need to include the variables you wish to override. - -## Role-Separated Playbooks - -We maintain role-separated playbooks, but they are kept in the `ansible/playbooks/` folder. The remaining playbooks in the root of the `ansible/` folder are composite (utilize more than one role) or one off playbooks (do not use any roles). - -If you wish to use these playbooks, we have created the `single-role.yaml` playbook. - -The documentation on its use is contained within the playbook. - -## Inventories - -We have provided an example inventory file; `example.cfg`. Our roles and playbooks are written against the host groups in this inventory. Files with a `.cfg` extension are ignored by git in the `inventories` folder. This is done to prevent us from accidentally exposing our inventories to the public. - -The host groups within the example inventory reference the host machines for the DE's underlying architecture, as well as host groups for the application itself. Each micro-service has a corresponding host group in the inventory. - -Refer to the `example.cfg` file for more info. - -# Playbooks - -* [Updating Databases](setup/database.md) diff --git a/docs/guides/ds.md b/docs/guides/ds.md deleted file mode 100644 index c4b47f2..0000000 --- a/docs/guides/ds.md +++ /dev/null @@ -1,23 +0,0 @@ -## :simple-kubernetes: Container Storage Interface (CSI) Driver - -## :fontawesome-brands-golang: GoCommands - -## 3rd Party Applications - -### :material-duck: Cyberduck - -### :simple-filezilla: FileZilla - -### :material-folder-network: SFTP - -### :material-folder-network: WebDAV - -### :material-file-tree: File Explorers - -## :material-share-all: Sharing Folders - -## :material-folder-open: Community Released Folder Creation - -## :material-folder-open: Curated Folders - -DataCite DOI \ No newline at end of file diff --git a/docs/guides/setup/ansible.md b/docs/guides/setup/ansible.md deleted file mode 100644 index 48c1012..0000000 --- a/docs/guides/setup/ansible.md +++ /dev/null @@ -1,133 +0,0 @@ - -## Installing Ansible -See http://docs.ansible.com/intro_installation.html for the long instructions. - -There's multiple ways to install ansible. I used Python's 'pip' command to install it on OS X, but -you can also use homebrew or a git checkout. - -__pip__ - - sudo pip install ansible - -__homebrew__ - - brew install ansible - -__checkout__ - - git clone git://github.com/ansible/ansible.git - cd ./ansible - source ./hacking/env-setup - -If you use the checkout method, add a line to your ~/.profile (on OS X) or your ~/.bashrc file (all -other sane OSes). - -__You only need ansible on your local machine. It does not need to be installed on the servers.__ - -## Installing Third Party Ansible Roles -See http://docs.ansible.com/galaxy.html for more information about managing third party roles. - -In brief, you'll need to sign up for galaxy at https://galaxy.ansible.com and execute the following -command in the de-ansible repo. - - ansible-galaxy install --force -r requirements.yaml - -## Learning About Ansible - -If you just want to use ansible for DE related tasks, move on to the other sections. - -If you want to make your own playbooks for existing inventories of servers, read: -http://docs.ansible.com/playbooks_intro.html. - -If you want to create a new inventory or modify an existing one, read: -http://docs.ansible.com/intro_inventory.html. - -Each playbook uses one or more modules. Modules encapsulate operations that run on servers and make -them idempotent. There are a lot of existing modules, which you can read about at -http://docs.ansible.com/modules_by_category.html. - -## Ansible Config Settings - -The design of the DE's ansible variables makes use of YAML -["hashes"](http://docs.ansible.com/ansible/YAMLSyntax.html#yaml-basics). If you ever intend to -override a single value in a hash, you will need to set ansible's hash behaviour to _"merge"_. -The easiest way to set these are with environment variables. -For example, add these lines to your `~/.bash_profile` or `~/.bashrc` file: - - export ANSIBLE_HASH_BEHAVIOUR="merge" - -Or these settings can be updated in a local `ansible.cfg`. - -## Preparing Servers - -### simplejson - -Each server needs to have the simplejson Python library available for the default Python -installation. - -__CentOS 5/6__ - - sudo yum install python-simplejson - -This should already be done for the development servers. - -### curl - -Each server needs to have curl on it so ansible can send messages to chat. curl is provided by -default on CentOS distributions, It needs to be manually installed on Ubuntu systems. - -__Ubuntu__ - - sudo apt-get install curl - -### httplib2 - -Each server needs to have httplib2 Python library available for the default Python installation. It -needs to be manually installed on Ubuntu and CentOS 6 systems. - -##### TODO: Is manual installation still necessary? The private-registry-image-builder role installs it with pip (on CentOS7 systems at least). - -__Ubuntu__ - - sudo apt-get install python-httplib2 - -__CentOS 6__ - - sudo yum install python-httplib2 - -## Setting Up Your Accounts - -You will need passwordless ssh access to each of the servers listed in the inventory for the -environment you're working in. - -### Generate RSA private and public keys - -If you do not already have a ~/.ssh/id_rsa.pub file generated, then run this command to create it: - - ssh-keygen -t rsa - -### Install ssh-copy-id - -See https://github.com/beautifulcode/ssh-copy-id-for-OSX for instructions on setting up ssh-copy-id -on OS X. - -### Copying ssh keys - - ssh-copy-id -i - -For example: - - ssh-copy-id -i ~/.ssh/id_rsa.pub user@example.iplantcollaborative.org - -Do that for each of the servers in the inventory. I'd recommend setting up an SSH config that has -entries for each of the servers (include the fully-qualified domain name) first; it will allow you -to skip a lot of typing. - -Next, you need to generate entries in ~/.ssh/known_hosts for each of the servers. The easiest way to -do this is to ssh into each of the servers once **after** you've set up passwordless ssh access. You -need to SSH into the fully-qualified domain name of the host. For example: - - ssh example.iplantcollaborative.org - -You will also need sudo access on those servers for some operations. You should already have this if -you're in the dev group on the servers. \ No newline at end of file diff --git a/docs/guides/setup/database.md b/docs/guides/setup/database.md deleted file mode 100644 index b67b452..0000000 --- a/docs/guides/setup/database.md +++ /dev/null @@ -1,6 +0,0 @@ - -## Updating the DE database - - ansible-playbook -i inventories/... -K [-u ] db-migrations.yaml - -The `-i`, `-K`, and `-u` options are the same as in the other Ansible commands. \ No newline at end of file diff --git a/docs/guides/setup/docker.md b/docs/guides/setup/docker.md deleted file mode 100644 index 84ca570..0000000 --- a/docs/guides/setup/docker.md +++ /dev/null @@ -1,25 +0,0 @@ - -## OSX - -To build on OS X, see [Get Started with Docker for Mac OS X](http://docs.docker.com/mac/started/) -for instructions on installing [Docker Toolbox](https://www.docker.com/toolbox) - -## Running ansible inside a Docker container - -The Dockerfile at the top-level of the de-ansible checkout can be used to create a personal container that is able to run the de-ansible playbooks. - -DO NOT PUSH YOUR PERSONAL de-ansible CONTAINER! - -To build the container, first run the create-ssh-configs.sh script at the top-level of the de-ansible checkout: - - de-ansible> ./create-ssh-configs.sh - -Then run docker build: - - de-ansible> docker build . - -Make a note of the image ID and use it in a docker run command: - - de-ansible> docker run --rm -it -v $(pwd):/de-ansible -w /de-ansible /bin/bash - -You should be able to run the ansible commands inside the container you created. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index becd05c..e484c41 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,151 +1,43 @@ -![](assets/cyverse_logo_2022.svg) - -# CyVerse Core Software Documentation - -Welcome to the comprehensive technical documentation for deploying and maintaining the CyVerse cyberinfrastructure. This documentation is designed to support DevOps engineers, system administrators, and application developers working with CyVerse's open-source scientific cyberinfrastructure. - -## Documentation Overview - -CyVerse is a Software as a Service (SaaS) platform built on modern cloud-native technologies, providing researchers with powerful computational infrastructure for data-intensive science. Our documentation covers the complete stack from infrastructure deployment to API integration. - -
- ![layercake](assets/layerCake.svg){width=800} -
The CyVerse infrastructure layer cake is stacked upon bare-metal Hardware, then Services, and finally Products
-
- --- - -## Find Documentation by Your Role - -### :material-server: DevOps Engineer - -**You're deploying and maintaining CyVerse infrastructure** - -**Start here:** - -- [Getting Started Guide](services/getting_started.md) - Prerequisites and deployment roadmap -- [System Overview](services/system_overview.md) - Architecture and technology stack -- [Deployment Guide](deployments/deployment_overview.md) - Complete deployment walkthrough - -**Common tasks:** - -- [Deploy Kubernetes cluster](deployments/kubernetes-deploy.md) - Core infrastructure setup -- [Configure authentication](deployments/keycloak.md) - KeyCloak deployment and LDAP integration -- [Set up storage](deployments/openebs.md) - OpenEBS and iRODS CSI driver -- [Deploy Discovery Environment](deployments/DiscoveryEnvironment.md) - Main application platform -- [Database management](database/main.md) - PostgreSQL schemas and migrations - -### :octicons-people-24: System Administrator - -**You're managing users, apps, and operational tasks** - -**Start here:** - -- [Administration Guide](guides/devops.md) - Daily operational procedures -- [Discovery Environment Admin](guides/de.md) - User management, app publishing, VICE access -- [Data Store Admin](guides/ds.md) - Data management and permissions - -**Common tasks:** - -- [Grant VICE access](guides/de.md#vice-access) - Enable interactive computing for users -- [Process DOI/Permanent ID requests](services/api/endpoints/permanent-id-requests.md) - Data publishing workflows -- [Manage user groups](deployments/grouper.md) - Grouper administration -- [Publish apps to Discovery Environment](guides/de.md#app-publication) - Tool integration -- [FAQ](guides/faq.md) - Frequently asked questions and troubleshooting - -### :material-code-braces: Application Developer - -**You're integrating with CyVerse APIs or contributing code** - -**Start here:** - -- [Developer Guide](development/index.md) - Development environment and contribution workflow -- [API Overview](services/api_overview.md) - Terrain API introduction -- [API Endpoint Index](services/api/endpoint-index.md) - Complete API reference - -**Common tasks:** - -- [Authentication](services/keycloak.md) - OAUTH 2.0 integration -- [Filesystem operations](services/api/endpoints/filesystem/directory-listing.md) - Data Store API -- [Job submission](services/api/endpoints/endpoints.md) - Analysis execution -- [Tapis v2 to v3 migration](services/api/tapis-v2-v3-migration.md) - Upgrade guide -- [Error handling](services/api/errors.md) - API error codes and responses - ---- - -## Documentation Sections - -| Section | Description | Audience | -|---------|-------------|----------| -| [**Getting Started**](services/getting_started.md) | Prerequisites, system overview, and deployment roadmap | DevOps, New users | -| [**Deployment Guide**](deployments/deployment_overview.md) | Complete infrastructure deployment: Kubernetes, services, databases | DevOps Engineers | -| [**Administration Guide**](guides/devops.md) | Operational procedures, user management, app publishing | System Administrators | -| [**API Reference**](services/api_overview.md) | Terrain API endpoints, authentication, error handling | Developers | -| [**Developer Guide**](development/index.md) | Development environment, contribution workflow, architecture | Developers | -| [**System Architecture**](services/services_overview.md) | Platform components, services, and design decisions | All audiences | -| [**Database Reference**](database/main.md) | PostgreSQL schemas for all CyVerse services | DevOps, DBAs | - +okf_version: "0.2" --- -## CyVerse Platform Components - -### Core Products - -- **[Discovery Environment (DE)](services/de.md)** - Web-based analysis platform for computational workflows -- **[Data Store](services/ds.md)** - 6+ PB iRODS-based data management system -- **[Data Commons](services/dc.md)** - Data publishing platform with DOI/ARK support -- **[VICE](deployments/vice.md)** - Visual Interactive Computing Environment (Jupyter, RStudio, etc.) - -### Specialized Tools - -- **[BisQue](services/bisque.md)** - Bio-Image Semantic Query User Environment -- **[DNA Subway](services/dnasubway.md)** - Educational genomics platform -- **[Cloud Services (CACAO)](services/cloud.md)** - Cloud automation and multi-cloud orchestration - -### Infrastructure Services - -- **[Authentication (KeyCloak)](services/keycloak.md)** - OAUTH 2.0, LDAP, CILogon integration -- **[Terrain API](services/api_overview.md)** - RESTful API aggregating all DE services -- **[Kubernetes](deployments/kubernetes-deploy.md)** - Container orchestration platform -- **[iRODS](deployments/irods-csi-driver.md)** - Integrated Rule-Oriented Data System for data management - ---- - -## Quick Links - -- :material-frequently-asked-questions: [**FAQ**](guides/faq.md) - Frequently asked questions and troubleshooting -- :simple-github: [**GitHub Organization**](https://github.com/cyverse-de){target=_blank} - Source code repositories -- :material-api: [**Live Terrain API**](https://de.cyverse.org/terrain/docs/){target=_blank} - Interactive API documentation -- :simple-docker: [**Harbor Registry**](https://harbor.cyverse.org/){target=_blank} - Container image repository -- :material-web: [**CyVerse Website**](https://cyverse.org){target=_blank} - Public-facing information and user support - ---- - -??? Question "What is CyVerse?" - - [Public Website](https://cyverse.org){target=_blank} - - CyVerse is a powerful computational infrastructure and the people who support its operations. It is fully open source, and dedicated to furthering open science. - - CyVerse has been funded by the [United States National Science Foundation (NSF)](https://www.nsf.gov/){target=_blank} from 2008 until the present +![](assets/cyverse_logo_2022.svg) - [![nsf](assets/NSF.svg){width=100}](https://www.nsf.gov/){target=_blank} +*Technical documentation for deploying and operating CyVerse — the open-source +cyberinfrastructure for data-intensive science. This bundle follows the +[Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) +v0.2: every document carries frontmatter, and every directory has an index like +this one.* - [![NSF-0735191](https://img.shields.io/badge/NSF-0735191-blue.svg)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=0735191) [![NSF-1265383](https://img.shields.io/badge/NSF-1265383-blue.svg)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1265383) [![NSF-1743442](https://img.shields.io/badge/NSF-1743442-blue.svg)](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1743442) +# Start here +* [About this documentation](about.md) - what CyVerse is, and where each audience should start +* [Deploying CyVerse from scratch](deployment/from-scratch.md) - end-to-end walkthrough of a two-node deployment +* [Update log](log.md) - what changed in this bundle, newest first -!!! Info ":fontawesome-brands-creative-commons-by: SOFTWARE LICENSE" +# Architecture - Copyright (c) 2010-2026, The Arizona Board of Regents on behalf of The University of Arizona +* [architecture/](architecture/) - how the pieces fit together, what they need, and what talks to what +* [platform/](platform/) - the products and services CyVerse offers its users - All rights reserved. +# Deployment - Developed by: CyVerse as a collaboration between participants at BIO5 at The University of Arizona (the primary hosting institution), Cold Spring Harbor Laboratory, The University of Texas at Austin, and individual contributors. Find out more at http://www.cyverse.org/. +Ordered by dependency: nothing in a later phase can start before the phases above it. - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +* [deployment/](deployment/) - the full deployment path, phase by phase +* [deployment/planning/](deployment/planning/) - prerequisites and deployment tooling +* [deployment/01-foundation/](deployment/01-foundation/) - HAProxy, PostgreSQL, RabbitMQ +* [deployment/02-databases/](deployment/02-databases/) - per-service schemas and migrations +* [deployment/03-data-store/](deployment/03-data-store/) - the iRODS zone and its DE integration +* [deployment/04-kubernetes/](deployment/04-kubernetes/) - cluster, certificates, storage, ingress, registry +* [deployment/05-core-services/](deployment/05-core-services/) - directory, authentication, search, messaging +* [deployment/06-applications/](deployment/06-applications/) - Discovery Environment, VICE, User Portal +* [deployment/07-post-install/](deployment/07-post-install/) - bootstrap, verification, troubleshooting - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of CyVerse, BIO5, The University of Arizona, Cold Spring Harbor Laboratory, The University of Texas at Austin, nor the names of other contributors may be used to endorse or promote products derived from this software without specific prior written permission. +# Operating and integrating - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* [operations/](operations/) - running a deployment day to day +* [api/](api/) - the Terrain API and its endpoints +* [development/](development/) - contributing to CyVerse code +* [references/](references/) - provenance notes for material this bundle derives from diff --git a/docs/log.md b/docs/log.md new file mode 100644 index 0000000..a529745 --- /dev/null +++ b/docs/log.md @@ -0,0 +1,17 @@ +# Bundle update log + +## 2026-07-29 + +* **Migration**: Converted the bundle to [Open Knowledge Format](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) v0.2. Every concept document now carries YAML frontmatter with `type`, `title`, `description`, `tags`, `status`, and `generated`; every directory has an `index.md` listing; provenance is recorded in `sources`. +* **Reorganization**: Restructured the bundle in CyVerse deployment order. `deployments/`, `database/`, `guides/`, and `services/` were replaced by [architecture/](architecture/), [platform/](platform/), [deployment/](deployment/) (phases 1 through 7), [operations/](operations/), [api/](api/), and [references/](references/). +* **Creation**: Added [Deploying CyVerse from scratch](deployment/from-scratch.md), an end-to-end runbook for a two-node deployment, derived from an anonymized [pilot deployment record](references/pilot-deployment-record.md). +* **Creation**: Added [component inventory and sizing](architecture/component-inventory.md) and [network requirements](architecture/network-requirements.md), which previously existed only in internal notes. +* **Creation**: Documented deployment steps that had no coverage: [HAProxy](deployment/01-foundation/haproxy.md), [iRODS catalog provider](deployment/03-data-store/irods-provider.md), [DE integration](deployment/03-data-store/de-integration.md), [iCAT database](deployment/02-databases/icat.md), [database migrations](deployment/02-databases/migrations.md), [cert-manager](deployment/04-kubernetes/cert-manager.md), [Harbor](deployment/04-kubernetes/harbor.md), [Argo](deployment/04-kubernetes/argo.md), [OpenLDAP](deployment/05-core-services/openldap.md), [OpenSearch](deployment/05-core-services/opensearch.md), [NATS](deployment/05-core-services/nats.md), [bootstrap](deployment/07-post-install/bootstrap.md), [verification](deployment/07-post-install/verification.md), and [troubleshooting](deployment/07-post-install/troubleshooting.md). +* **Deduplication**: Merged `exim4.md` and `local-exim.md` into [mail](deployment/05-core-services/mail.md); folded `services_overview.md` and `deployment_overview.md` into directory indexes; folded `guides/setup/ansible.md` into [Ansible](deployment/planning/ansible.md) and `guides/setup/database.md` into [migrations](deployment/02-databases/migrations.md); removed the duplicated golang-migrate procedure from each database document and the duplicated database list and deployment-order list that appeared in three places. +* **Correction**: Fixed the deployment order in the prerequisites document, which placed databases after the services that depend on them. +* **Correction**: rsyslog snippet path (`/etc/rsyslog.d/`, not `/etc/rsyslog/`); `dirname` rather than `basename` for the kubeconfig directory; `helm -n prod uninstall nats`; VRRP recorded as IP protocol 112 rather than a TCP port; QMS extensions created in `qms` rather than `de`; recomputed the sizing totals and noted the 100 TB / 110 TB discrepancy in the source. +* **Correction**: Fixed the `zensical.toml` theme name, which pointed at an uninstallable theme and failed the docs build, and the `extra_css` path, which named a stylesheet that does not exist. +* **Update**: Marked [Elasticsearch](deployment/05-core-services/elasticsearch.md) deprecated in favor of OpenSearch, and ingress-nginx transitional in favor of Traefik. Replaced site-specific hostnames, DNs, zone names, and credentials throughout with placeholders. +* **Update**: Replaced the `XXX` placeholders in [system overview](architecture/system-overview.md) with the values that are known and explicit dashes where an aggregate is not tracked. + +*Changes before this date are recorded in the repository's git history.* diff --git a/docs/operations/data-store.md b/docs/operations/data-store.md new file mode 100644 index 0000000..6656a43 --- /dev/null +++ b/docs/operations/data-store.md @@ -0,0 +1,53 @@ +--- +type: Playbook +title: "Data Store administration" +description: "Client tooling, sharing, and curation workflows for the iRODS Data Store." +tags: [operations, administration, data-store, irods] +status: draft +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +!!! warning "Outline, not a complete guide" + + This document was a list of empty headings. It now records what each topic + covers and points at the authoritative source; the procedures themselves are + still to be written. User-facing instructions live at + [learning.cyverse.org](https://learning.cyverse.org/){target=_blank}. + +# Access paths + +| Path | What it is | Reference | +|------|------------|-----------| +| iRODS protocol | Native access with `iCommands` or [GoCommands](https://github.com/cyverse/gocommands) | [Data Store](../platform/data-store.md) | +| WebDAV / HTTPS | `data.cyverse.org`, via Apache and davrods | [Data Store](../platform/data-store.md) | +| SFTP | SFTPGo with the iRODS backend | [Data Store](../platform/data-store.md) | +| CSI driver | Mounts collections into analysis pods | [iRODS CSI driver](../deployment/05-core-services/irods-csi-driver.md) | +| Terrain API | Programmatic filesystem operations | [filesystem endpoints](../api/endpoints/filesystem/directory-listing.md) | + +Third-party clients that speak WebDAV or SFTP work against the endpoints above — +Cyberduck, FileZilla, and most desktop file managers among them. Nothing +CyVerse-specific has to be installed for those. + +# Common administrative tasks + +* **Sharing.** Permissions are set per collection or data object and can be + granted to users or groups; see + [permissions](../api/endpoints/filesystem/permissions.md) and + [sharing](../api/endpoints/filesystem/sharing.md) for what the API exposes. +* **Anonymous access.** Public readability depends on the `anonymous` account's + permissions, established during + [zone initialization](../deployment/03-data-store/irods-provider.md#anonymous-access). +* **Community released folders.** Publishing a folder to all CyVerse users; see + [Data Commons](../platform/data-commons.md). +* **Curated folders and DOIs.** Staff-reviewed publication with a DataCite DOI, + driven by [permanent ID requests](../api/endpoints/permanent-id-requests.md). + The reviewer's checklist is in + [DE administration](./discovery-environment.md). +* **Tickets.** Time- or use-limited anonymous access to a specific path; see + [tickets](../api/endpoints/filesystem/tickets.md). + +# Related + +* [Data Store](../platform/data-store.md) +* [iRODS provider deployment](../deployment/03-data-store/irods-provider.md) +* [FAQ](./faq.md) diff --git a/docs/guides/de.md b/docs/operations/discovery-environment.md similarity index 97% rename from docs/guides/de.md rename to docs/operations/discovery-environment.md index 22438ee..6e8991a 100644 --- a/docs/guides/de.md +++ b/docs/operations/discovery-environment.md @@ -1,3 +1,11 @@ +--- +type: Playbook +title: "Discovery Environment administration" +description: "Day-to-day DE administration: users, VICE access, app publication, and the admin panel." +tags: [operations, administration, discovery-environment] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- [:material-lock: Discovery Environment Admin Panel](https://de.cyverse.org/admin/) ## Administrative Panel diff --git a/docs/guides/faq.md b/docs/operations/faq.md similarity index 86% rename from docs/guides/faq.md rename to docs/operations/faq.md index 645106f..75da18a 100644 --- a/docs/guides/faq.md +++ b/docs/operations/faq.md @@ -1,3 +1,11 @@ +--- +type: Playbook +title: "Frequently asked questions" +description: "Answers and troubleshooting recipes for the questions operators hit most often." +tags: [operations, faq, troubleshooting] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Frequently Asked Questions (FAQ) Common questions about deploying, administering, and developing with CyVerse infrastructure. @@ -8,7 +16,7 @@ Common questions about deploying, administering, and developing with CyVerse inf ### How do I deploy CyVerse on my infrastructure? -Start with the [Getting Started Guide](../services/getting_started.md) to review prerequisites, then follow the [Deployment Guide](../deployments/deployment_overview.md) for step-by-step deployment instructions. The deployment follows this sequence: +Start with the [Getting Started Guide](../deployment/planning/prerequisites.md) to review prerequisites, then follow the [deployment guide](../deployment/index.md) for step-by-step deployment instructions. The deployment follows this sequence: 1. Deploy Kubernetes cluster 2. Set up storage (OpenEBS) and networking (Ingress NGINX) @@ -16,7 +24,7 @@ Start with the [Getting Started Guide](../services/getting_started.md) to review 4. Provision databases 5. Deploy application services (Discovery Environment, User Portal, VICE) -See the [deployment roadmap](../services/getting_started.md#deployment-roadmap) for detailed phase breakdown. +See [deployment](../deployment/index.md) for the phase-by-phase breakdown, and [deploying from scratch](../deployment/from-scratch.md) for a worked example. ### What are the minimum hardware requirements? @@ -35,7 +43,7 @@ See the [deployment roadmap](../services/getting_started.md#deployment-roadmap) - 64GB RAM per node - 1TB storage -See [Prerequisites](../services/getting_started.md#prerequisites) for complete requirements. +See [prerequisites](../deployment/planning/prerequisites.md) for complete requirements. ### Can I deploy CyVerse on commercial cloud providers (AWS, GCP, Azure)? @@ -46,7 +54,7 @@ Yes! CyVerse can be deployed on: - :simple-azuredevops: **Azure** - Use AKS for Kubernetes, Azure Disks for storage - :simple-openstack: **OpenStack** - CyVerse's primary cloud platform -You'll need to adapt storage provisioning and load balancer configurations for your cloud provider. See the [Kubernetes deployment guide](../deployments/kubernetes-deploy.md). +You'll need to adapt storage provisioning and load balancer configurations for your cloud provider. See the [Kubernetes deployment guide](../deployment/04-kubernetes/cluster.md). ### What Kubernetes distribution should I use? @@ -92,7 +100,7 @@ Updates are managed through: 3. **Database migrations** - Run migration scripts before deploying new versions 4. **Rolling updates** - Kubernetes handles zero-downtime deployments -Always test updates in a staging environment first. See [deployment documentation](../deployments/deployment_overview.md). +Always test updates in a staging environment first. See [deployment](../deployment/index.md). --- @@ -108,7 +116,7 @@ VICE (Visual Interactive Computing Environment) access is granted through the Di 4. Grant "VICE Access" permission 5. User must meet requirements: valid account, accepted terms of service, sufficient quota -See [DE Administration Guide](de.md#vice-access) for detailed procedures. +See [DE Administration Guide](./discovery-environment.md#vice-access) for detailed procedures. ### How do I process a DOI/Permanent ID request? @@ -121,7 +129,7 @@ DOI requests for data publishing follow this workflow: 5. Approve request - system creates DOI via DataCite 6. Notify user of published DOI -See [Permanent ID Requests documentation](../services/api/endpoints/permanent-id-requests.md) for complete SOP. +See [Permanent ID Requests documentation](../api/endpoints/permanent-id-requests.md) for complete SOP. ### How do I add a new application to the Discovery Environment? @@ -137,7 +145,7 @@ See [Permanent ID Requests documentation](../services/api/endpoints/permanent-id 4. **Test privately** - Validate app functionality 5. **Publish publicly** - Make available to all users -See [DE Administration Guide](de.md#app-publication) for step-by-step instructions. +See [DE administration](./discovery-environment.md) for step-by-step instructions. ### How do I manage user storage quotas? @@ -171,7 +179,7 @@ Backup all service databases: - Portal database - Unleash database -See [Database documentation](../database/main.md) for detailed backup procedures. +See [Database documentation](../deployment/01-foundation/postgresql.md) for detailed backup procedures. ### How do I troubleshoot service failures? @@ -203,7 +211,7 @@ Administrator permissions are managed through KeyCloak: - `super-admin` for platform-wide access 5. User logs out and back in to activate permissions -See [KeyCloak documentation](../services/keycloak.md) for role management. +See [KeyCloak documentation](../platform/authentication.md) for role management. --- @@ -235,20 +243,20 @@ curl -H "Authorization: Bearer $TOKEN" \ https://de.cyverse.org/terrain/filesystem/list ``` -See [Developer Guide](../development/index.md#authentication) for complete examples. +See [Developer Guide](../development/developer-guide.md#authentication) for complete examples. ### Where can I find the complete API documentation? - **[Live Swagger UI](https://de.cyverse.org/terrain/docs/){target=_blank}** - Interactive API testing -- **[Endpoint Index](../services/api/endpoint-index.md)** - Complete endpoint reference -- **[API Overview](../services/api_overview.md)** - Terrain API introduction -- **[Error Codes](../services/api/errors.md)** - Error handling guide +- **[Endpoint Index](../api/endpoint-index.md)** - Complete endpoint reference +- **[API Overview](../api/terrain.md)** - Terrain API introduction +- **[Error Codes](../api/errors.md)** - Error handling guide All Terrain endpoints are documented with request/response schemas and examples. ### How do I migrate from Tapis v2 to Tapis v3? -Tapis v3 introduces breaking changes. Follow the [Tapis v2 to v3 Migration Guide](../services/api/tapis-v2-v3-migration.md) which covers: +Tapis v3 introduces breaking changes. Follow the [Tapis v2 to v3 Migration Guide](../api/tapis-v2-v3-migration.md) which covers: - API endpoint changes - Authentication updates @@ -309,7 +317,7 @@ def upload_large_file(access_token, local_file, dest_path, chunk_size=10*1024*10 ) ``` -See [File I/O endpoints](../services/api/endpoints/fileio.md) for details. +See [File I/O endpoints](../api/endpoints/fileio.md) for details. ### How do I receive notifications when my job completes? @@ -329,7 +337,7 @@ You'll receive notifications via: - Email (if configured in user preferences) - Webhook callbacks (if configured) -See [Notifications API](../services/api/endpoints/notifications.md) and [Callbacks API](../services/api/endpoints/callbacks.md). +See [Notifications API](../api/endpoints/notifications.md) and [Callbacks API](../api/endpoints/callbacks.md). ### Can I run the API locally for development? @@ -402,7 +410,7 @@ kubectl top pods -n irods iperf3 -c irods-server ``` -See [Data Store documentation](../services/ds.md) for optimization tips. +See [Data Store documentation](../platform/data-store.md) for optimization tips. ### Why are jobs stuck in "Submitted" status? @@ -440,7 +448,7 @@ kubectl logs -n de deployment/analyses - Add resource quotas: Define default/max memory for different app categories - Scale cluster: Add more compute nodes -See [Kubernetes resources documentation](../deployments/k8s-resources.md). +See [Kubernetes resources documentation](../deployment/04-kubernetes/resources.md). ### ElasticSearch is using too much disk space @@ -467,7 +475,7 @@ curl -X PUT "localhost:9200/_ilm/policy/cleanup-policy" -H 'Content-Type: applic }' ``` -See [ElasticSearch deployment documentation](../deployments/elasticsearch.md). +See [ElasticSearch deployment documentation](../deployment/05-core-services/elasticsearch.md). --- @@ -493,7 +501,7 @@ See [ElasticSearch deployment documentation](../deployments/elasticsearch.md). 3. **DOI publication** - Publish to Data Commons with permanent identifier -See [Sharing API](../services/api/endpoints/filesystem/sharing.md) and [Data Store Guide](ds.md). +See [Sharing API](../api/endpoints/filesystem/sharing.md) and [Data Store Guide](./data-store.md). ### How do I search for data across the entire Data Store? @@ -511,7 +519,7 @@ Search supports: - Filters by file type, date, size - User-defined metadata (AVU) searches -See [Search API documentation](../services/api/endpoints/filesystem/search.md). +See [Search API documentation](../api/endpoints/filesystem/search.md). ### How do I attach metadata to data files? @@ -534,7 +542,7 @@ requests.post( Or use the DE interface: Right-click file → Metadata → Add -See [Metadata API](../services/api/endpoints/filesystem/metadata.md). +See [Metadata API](../api/endpoints/filesystem/metadata.md). ### How do I recover deleted data? @@ -555,7 +563,7 @@ curl -H "Authorization: Bearer $TOKEN" \ After 30 days, files are permanently deleted. Implement backups for critical data. -See [Restore API](../services/api/endpoints/filesystem/restore.md). +See [Restore API](../api/endpoints/filesystem/restore.md). ### What's the maximum file size I can store? @@ -571,13 +579,13 @@ For very large files (>100 GB), use: - **GoCommands**: CyVerse's modern iRODS CLI - **Globus**: For scheduled large transfers -See [Data Store documentation](../services/ds.md). +See [Data Store documentation](../platform/data-store.md). --- ## Still Have Questions? - **Search this documentation** - Use the search bar at the top -- **Check service-specific docs** - See [Deployment Guide](../deployments/deployment_overview.md), [API Reference](../services/api_overview.md), or [Admin Guides](devops.md) +- **Check service-specific docs** - See [deployment](../deployment/index.md), [API reference](../api/terrain.md), or [operations](./index.md) - **GitHub Issues** - Report issues or ask questions at [github.com/cyverse-de](https://github.com/cyverse-de){target=_blank} - **CyVerse Support** - Contact support at [cyverse.org/help](https://cyverse.org/help){target=_blank} diff --git a/docs/operations/index.md b/docs/operations/index.md new file mode 100644 index 0000000..49e3473 --- /dev/null +++ b/docs/operations/index.md @@ -0,0 +1,13 @@ +# Operations + +Running a deployed CyVerse: users, apps, data, and the questions that recur. + +* [Discovery Environment administration](discovery-environment.md) - users, VICE access, app publication, admin panel +* [Data Store administration](data-store.md) - client tooling, sharing, and curation workflows +* [User Portal administration](user-portal.md) - accounts, access requests, services, workshops +* [Frequently asked questions](faq.md) - recurring questions and troubleshooting recipes + +# Related + +* [Troubleshooting a new deployment](../deployment/07-post-install/troubleshooting.md) +* [Verification](../deployment/07-post-install/verification.md) diff --git a/docs/guides/user_portal.md b/docs/operations/user-portal.md similarity index 63% rename from docs/guides/user_portal.md rename to docs/operations/user-portal.md index 1798042..cd871fd 100644 --- a/docs/guides/user_portal.md +++ b/docs/operations/user-portal.md @@ -1,4 +1,11 @@ - +--- +type: Playbook +title: "User Portal administration" +description: "Managing accounts, access requests, services, and workshops from the portal admin panel." +tags: [operations, administration, user-portal] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- [:material-lock: CyVerse User Portal Admin Panel](https://user.cyverse.org/administrative) ## Administrative Panel @@ -28,4 +35,4 @@ View, create, and modify workshops. Search across all form submissions and view individual submissions. ### Forms -View and edit forms. \ No newline at end of file +View and edit forms. diff --git a/docs/platform/authentication.md b/docs/platform/authentication.md new file mode 100644 index 0000000..dcc0209 --- /dev/null +++ b/docs/platform/authentication.md @@ -0,0 +1,61 @@ +--- +type: Architecture Overview +title: "Authentication" +description: "How Keycloak, CILogon, and OAuth 2.0 combine to authenticate CyVerse users." +tags: [platform, authentication, keycloak] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- +# How CyVerse authenticates users + +CyVerse authenticates through [Keycloak](https://www.keycloak.org/){target=_blank}, +which brokers to [CILogon](https://cilogon.org/){target=_blank} for federated +institutional identity and to OAuth 2.0 providers such as Google, GitHub, and +ORCID. + +``` mermaid +sequenceDiagram + autonumber + User->>Browser: Click on external Auth + Browser-->>Keycloak: Authentication request (TOKEN) + loop + Keycloak-->>Browser: Browser opened with ../auth?=client_id=de-prod=TOKEN + end + Keycloak-->>CILogon: Auth Request + User->>CILogon: Enter Credentials + Keycloak-->>OAUTH: Auth Response + CILogon-->>Keycloak: Auth Response + Browser-->>OAUTH: Ask for Token + OAUTH-->>Browser: Retrieve Token +``` + +[comment]: <> (![keycloak](../assets/de/keycloak.svg)) + +Users start at the left: the browser is redirected to Keycloak, which brokers the request to CILogon or an OAuth provider, and the token comes back through the browser. + +The public US deployment runs Keycloak at +[kc.cyverse.org](https://kc.cyverse.org){target=_blank}. + +## Deployment + +Keycloak runs in the Kubernetes cluster, in its own namespace. Deploying and +configuring it — realm, LDAP federation, mappers, realm roles, and the OAuth +clients each CyVerse application needs — is covered in +[Keycloak deployment](../deployment/05-core-services/keycloak.md). Its database is +[the Keycloak database](../deployment/02-databases/keycloak.md), and the account +directory it federates is [OpenLDAP](../deployment/05-core-services/openldap.md). + +## Using it from an API client + +Terrain accepts OAuth and OIDC tokens issued by Keycloak; see +[Terrain](../api/terrain.md) for how clients obtain one. + + +The provisioning playbooks live in +[ansible-kubernetes-keycloak](https://github.com/cyverse/ansible-kubernetes-keycloak){target=_blank}. + +## Related + +* [Keycloak deployment](../deployment/05-core-services/keycloak.md) +* [OpenLDAP](../deployment/05-core-services/openldap.md) +* [Terrain](../api/terrain.md) diff --git a/docs/services/bisque.md b/docs/platform/bisque.md similarity index 65% rename from docs/services/bisque.md rename to docs/platform/bisque.md index 35df444..9cf23b1 100644 --- a/docs/services/bisque.md +++ b/docs/platform/bisque.md @@ -1,8 +1,15 @@ - +--- +type: Service +title: "BisQue" +description: "The Bio-Image Semantic Query User Environment fork maintained by CyVerse." +tags: [platform, bisque, imaging] +status: draft +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- The [Bio-Image Semantic Query User Environment (BisQue)](https://bisque2.ece.ucsb.edu/client_service/){target=_blank} was originally developed at [UC Santa Barbara Center for Bio-Image Informatics](https://bioimage.ucsb.edu/){target=_blank}. Later, it became [ViQi AI](https://viqiai.com/){target=_blank} CyVerse maintains a fork of the UCSB version of [BisQue](https://bisque.cyverse.org/client_service/){target=_blank} ## Deployment of CyVerse BisQue -[:simple-github: UCSB BisQue iRODS](https://github.com/cyverse/ucsb-bisque-irods){target=_blank} \ No newline at end of file +[:simple-github: UCSB BisQue iRODS](https://github.com/cyverse/ucsb-bisque-irods){target=_blank} diff --git a/docs/services/cloud.md b/docs/platform/cloud.md similarity index 84% rename from docs/services/cloud.md rename to docs/platform/cloud.md index 8a32fbb..9772714 100644 --- a/docs/services/cloud.md +++ b/docs/platform/cloud.md @@ -1,3 +1,11 @@ +--- +type: Service +title: "Cloud services" +description: "CACAO infrastructure-as-code for multi-cloud deployments, and the retired Atmosphere platform." +tags: [platform, cloud, cacao] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- CyVerse has a long history working with [:simple-openstack: OpenStack](https://www.openstack.org/) based clouds, including its deprecated "[Atmosphere](https://user.cyverse.org/services/1){target=_blank}" and "Jetstream" User Interfaces. CyVerse is partnered with [Jetstream2](https://jetstream-cloud.org/){target=_blank} to manage cloud-native applications such as Terraform, Argo Workflows, and Kubernetes. diff --git a/docs/platform/data-commons.md b/docs/platform/data-commons.md new file mode 100644 index 0000000..98d68f4 --- /dev/null +++ b/docs/platform/data-commons.md @@ -0,0 +1,45 @@ +--- +type: Service +title: "Data Commons" +description: "The publishing platform for curated and community-released datasets." +tags: [platform, data-commons, publishing, doi] +status: draft +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# What it is + +The [Data Commons](https://datacommons.cyverse.org) provides HTTP access to +datasets published from the CyVerse [Data Store](./data-store.md), together with +whatever metadata their owners attached. It is the read side of publishing: data +stays in iRODS, and the Data Commons presents it to people who do not have a +CyVerse account. + +Published data is read through the `anonymous` iRODS account, which is why +[zone initialization](../deployment/03-data-store/irods-provider.md#anonymous-access) +grants that account read permission on the zone and on `/home`. + +# Publishing paths + +| Path | What it produces | +|------|------------------| +| Community released | A folder visible to all CyVerse users, with no persistent identifier | +| Curated | A curated dataset with a DataCite DOI, after staff review | + +The DOI workflow is driven by permanent ID requests through Terrain — see +[permanent ID requests](../api/endpoints/permanent-id-requests.md) for the API, and +[DE administration](../operations/discovery-environment.md) for the review steps +curators follow. + +!!! note "This document is a stub" + + It records what the Data Commons is and how it connects to the rest of the + platform. The curation workflow lives in the operations guide, and there is no + Data Commons–specific deployment component beyond the Data Store's access + services. + +# Related + +* [Data Store](./data-store.md) +* [Permanent ID requests](../api/endpoints/permanent-id-requests.md) +* [Data Store administration](../operations/data-store.md) diff --git a/docs/services/ds.md b/docs/platform/data-store.md similarity index 90% rename from docs/services/ds.md rename to docs/platform/data-store.md index 8d6e8f5..49f37e9 100644 --- a/docs/services/ds.md +++ b/docs/platform/data-store.md @@ -1,4 +1,11 @@ - +--- +type: Service +title: "Data Store" +description: "The iRODS-backed storage platform and the access services layered on top of it." +tags: [platform, data-store, irods] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- Data Storage in CyVerse is managed both internally and through federation. ??? info "What is Federation?" diff --git a/docs/services/de.md b/docs/platform/discovery-environment.md similarity index 96% rename from docs/services/de.md rename to docs/platform/discovery-environment.md index 87d8b12..3a890ca 100644 --- a/docs/services/de.md +++ b/docs/platform/discovery-environment.md @@ -1,6 +1,14 @@ +--- +type: Service +title: "Discovery Environment" +description: "The web-based data science workbench: apps, analyses, VICE, and the services behind them." +tags: [platform, discovery-environment] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- [Discovery Environment User Guide](https://learning.cyverse.org/de/){target=_blank} -Deployment of the Discovery Environment (DE) is provided in the [Deployments section](../deployments/DiscoveryEnvironment.md) +Deployment of the Discovery Environment (DE) is provided in the [Deployments section](../deployment/06-applications/discovery-environment.md) ![de_condor](../assets/de/de_condor.svg) @@ -14,7 +22,7 @@ Deployment of the Discovery Environment (DE) is provided in the [Deployments sec The following sections describe the key components of the infrastructure upon which the DE operates. -## [Data Store](ds.md) +## [Data Store](./data-store.md) The DE provides access and management of data via the CyVerse [Data Store](http://www.cyverse.org/ci/data-store), which is built on top of diff --git a/docs/platform/dna-subway.md b/docs/platform/dna-subway.md new file mode 100644 index 0000000..cb36330 --- /dev/null +++ b/docs/platform/dna-subway.md @@ -0,0 +1,31 @@ +--- +type: Service +title: "DNA Subway" +description: "The educational genomics platform for classroom sequence analysis." +tags: [platform, dna-subway, education] +status: draft +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# What it is + +[DNA Subway](https://dnasubway.cyverse.org/) packages genomics workflows — +annotation, phylogenetics, RNA-Seq, and DNA barcoding — as guided "subway lines" +for classroom use. It hides the command line and the scheduler so that students +work with the biology rather than the infrastructure. + +It runs against CyVerse accounts and reads and writes the +[Data Store](./data-store.md), but it is a separate application from the +[Discovery Environment](./discovery-environment.md) and is not part of the +deployment procedures in this bundle. + +!!! note "This document is a stub" + + It exists so the service is discoverable and its relationship to the platform + is recorded. Educational documentation for DNA Subway is maintained with the + application itself. + +# Related + +* [Data Store](./data-store.md) +* [Discovery Environment](./discovery-environment.md) diff --git a/docs/platform/index.md b/docs/platform/index.md new file mode 100644 index 0000000..9714478 --- /dev/null +++ b/docs/platform/index.md @@ -0,0 +1,30 @@ +# Products and services + +The user-facing layer of CyVerse. These documents describe what each service *is*; +building them is covered in [deployment/](../deployment/). + +
+ ![layercake](../assets/layerCake.svg){width=800} +
The CyVerse layer cake: hardware, then services, then products
+
+ +# Data + +* [Data Store](data-store.md) - iRODS storage with WebDAV, SFTP, and API access +* [Data Commons](data-commons.md) - publishing curated and community-released datasets + +# Compute + +* [Discovery Environment](discovery-environment.md) - the web-based data science workbench +* [Cloud services](cloud.md) - CACAO infrastructure-as-code for multi-cloud deployments + +# Specialized applications + +* [BisQue](bisque.md) - bio-image semantic query and analysis +* [DNA Subway](dna-subway.md) - educational genomics workflows + +# Platform services + +* [Authentication](authentication.md) - Keycloak, CILogon, and OAuth 2.0 +* [Subscriptions](subscriptions.md) - resource allocations and how they are managed +* [Terrain API](../api/terrain.md) - the API every product is built on diff --git a/docs/services/subscriptions.md b/docs/platform/subscriptions.md similarity index 82% rename from docs/services/subscriptions.md rename to docs/platform/subscriptions.md index ff7ebd9..262db4a 100644 --- a/docs/services/subscriptions.md +++ b/docs/platform/subscriptions.md @@ -1,3 +1,11 @@ +--- +type: Service +title: "Subscriptions" +description: "Subscription tiers, what they allocate, and how administrators manage them." +tags: [platform, subscriptions, quotas] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- # Subscriptions CyVerse subscriptions provide enhanced resource allocations for computational analysis and data storage. @@ -32,4 +40,4 @@ Subscription status and resource allocations are managed through administrative - User subscription verification - Resource quota assignment based on subscription tier - Usage monitoring and reporting -- Subscription renewal notifications \ No newline at end of file +- Subscription renewal notifications diff --git a/docs/references/index.md b/docs/references/index.md new file mode 100644 index 0000000..258bcd4 --- /dev/null +++ b/docs/references/index.md @@ -0,0 +1,10 @@ +# References + +Provenance notes for material this bundle derives from, per the OKF `references/` +convention. + +* [Pilot CyVerse deployment record](pilot-deployment-record.md) - the internal pilot write-up behind the from-scratch runbook, and how it was anonymized + +# Related + +* [Deploying CyVerse from scratch](../deployment/from-scratch.md) diff --git a/docs/references/pilot-deployment-record.md b/docs/references/pilot-deployment-record.md new file mode 100644 index 0000000..d04c562 --- /dev/null +++ b/docs/references/pilot-deployment-record.md @@ -0,0 +1,57 @@ +--- +type: Reference +title: "Pilot CyVerse deployment record" +description: "Provenance note for the internal pilot deployment write-up that the from-scratch runbook and sizing tables derive from." +tags: [references, provenance, pilot] +status: stable +generated: { by: process:okf-migration, at: 2026-07-29T00:00:00Z } +--- + +# What this is + +Several documents in this bundle derive from an internal write-up of a +two-node CyVerse pilot deployment: a component and sizing list, a port list, an +ordered set of installation steps, and a short list of fixes found during the +install. + +That write-up is not reproduced here. It records one site's hostnames, zone +name, LDAP DNs, realm and client names, service account names, and the shape of +its secrets. Publishing it verbatim would leak site detail without helping +anyone deploy, so the material was rewritten with placeholders and generalized +one step away from the pilot. + +# What was derived from it + +| Document | What it took | +|----------|--------------| +| [Deploying CyVerse from scratch](../deployment/from-scratch.md) | The phase order and every installation step | +| [Component inventory and sizing](../architecture/component-inventory.md) | Component list, per-component requirements, node allocation, dependency graph | +| [Network requirements](../architecture/network-requirements.md) | The port list | +| [PostgreSQL](../deployment/01-foundation/postgresql.md) | Kernel and `postgresql.conf` tuning | +| [iRODS provider](../deployment/03-data-store/irods-provider.md) | Installer answers, policy installation, runtime initialization | +| [Keycloak](../deployment/05-core-services/keycloak.md) | Realm, LDAP federation, mapper, role, and client configuration | +| [Troubleshooting](../deployment/07-post-install/troubleshooting.md) | The fixes found during the install | + +# How it was transformed + +* **Hostnames and domains** became `core-1`, `analysis-1`, and + ``. +* **Realm, client, and account names** became ``-prefixed placeholders. +* **The iRODS zone name** became ``. +* **LDAP DNs** became ``. +* **Passwords, salts, zone keys, and negotiation keys** became + ``, with instructions to generate a fresh value per install + rather than any hint of the original. +* **A pointer to an example configuration file shared in a chat channel** + became a sanitized example checked in beside the procedure. +* **Individual names** in estimate annotations became the team. +* **Errors were corrected rather than copied.** The corrections are called out + where they appear — an rsyslog path, a shell command, a protocol number, and + a rounding inconsistency in the sizing totals. + +# Freshness + +The pilot deployment used iRODS 4.3.3, PostgreSQL 16, and k0s-managed +Kubernetes. Version-specific steps are marked where they matter. Treat the +upstream project documentation, not this bundle, as authoritative for anything +version-dependent. diff --git a/docs/services/api/endpoint-index.md b/docs/services/api/endpoint-index.md deleted file mode 100644 index 464a485..0000000 --- a/docs/services/api/endpoint-index.md +++ /dev/null @@ -1,294 +0,0 @@ -**Jump to:** - -[`/admin`](#admin) - -[`/apps`](#apps) - -[`/coge`](#coge) - -[`/favorites`](#favorites) - -[`/filesystem`](#filesystem) - -[`/permanent-id-requests`](#permanent-id-requests) - -[`/secured`](#secured) - -[`/send-notification`](#send-notification) - -[`/uuid`](#uuid) - -## get - -[`GET /`](endpoints/misc.md#verifying-that-terrain-is-running) - -## admin - -[`GET /admin/apps/categories`](endpoints/app-metadata.md#listing-app-categories) - -[`GET /admin/apps/categories/search`](endpoints/app-metadata.md#searching-for-categories-by-name) - -[`POST /admin/apps/categories/{system-id}`](endpoints/app-metadata.md#adding-categories) - -[`DELETE /admin/apps/categories/{system-id}/{category-id}`](endpoints/app-metadata.md#deleting-a-category-by-id) - -[`PATCH /admin/apps/categories/{system-id}/{category-id}`](endpoints/app-metadata.md#updating-an-app-category) - -[`DELETE /admin/apps/{app-id}/comments/{comment-id}`](endpoints/comments.md#administratively-deleting-a-comment) - -[`PATCH /admin/apps/{app-id}/comments/{comment-id}`](endpoints/comments.md#retractingreadmitting-a-comment) - -[`GET /admin/apps/{app-id}/metadata`](endpoints/app-metadata.md#managing-app-avu-metadata) - -[`POST /admin/apps/{app-id}/metadata`](endpoints/app-metadata.md#managing-app-avu-metadata) - -[`PUT /admin/apps/{app-id}/metadata`](endpoints/app-metadata.md#managing-app-avu-metadata) - -[`DELETE /admin/filesystem/entry/{entry-id}/comments/{comment-id}`](endpoints/comments.md#administratively-deleting-a-comment) - -[`PATCH /admin/filesystem/entry/{entry-id}/comments/{comment-id}`](endpoints/comments.md#retractingreadmitting-a-comment) - -[`GET /admin/filesystem/metadata/templates`](endpoints/filesystem/metadata.md#listing-metadata-templates) - -[`POST /admin/filesystem/metadata/templates`](endpoints/filesystem/metadata.md#adding-metadata-templates) - -[`DELETE /admin/filesystem/metadata/templates/{template-id}`](endpoints/filesystem/metadata.md#marking-a-metadata-template-as-deleted) - -[`POST /admin/filesystem/metadata/templates/{template-id}`](endpoints/filesystem/metadata.md#updating-metadata-templates) - -[`GET /admin/notifications/system`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`PUT /admin/notifications/system`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /admin/notifications/system-types`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`DELETE /admin/notifications/system/:uuid`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /admin/notifications/system/:uuid`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`POST /admin/notifications/system/:uuid`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /admin/ontologies`](endpoints/app-ontologies.md#listing-saved-ontology-details) - -[`POST /admin/ontologies`](endpoints/app-ontologies.md#save-an-ontology-xml-document) - -[`DELETE /admin/ontologies/{ontology-version}`](endpoints/app-ontologies.md#logically-deleting-an-ontology) - -[`GET /admin/ontologies/{ontology-version}`](endpoints/app-ontologies.md#listing-hierarchies-for-any-ontology) - -[`POST /admin/ontologies/{ontology-version}`](endpoints/app-ontologies.md#set-active-ontology-version) - -[`DELETE /admin/ontologies/{ontology-version}/{root-iri}`](endpoints/app-ontologies.md#deleting-an-ontology-hierarchy) - -[`GET /admin/ontologies/{ontology-version}/{root-iri}`](endpoints/app-ontologies.md#listing-filtered-hierarchies-for-any-ontology) - -[`PUT /admin/ontologies/{ontology-version}/{root-iri}`](endpoints/app-ontologies.md#save-an-ontology-hierarchy) - -[`GET /admin/ontologies/{ontology-version}/{root-iri}/apps`](endpoints/app-ontologies.md#listing-apps-in-hierarchies-for-any-ontology) - -[`GET /admin/ontologies/{ontology-version}/{root-iri}/unclassified`](endpoints/app-ontologies.md#listing-unclassified-apps-for-any-ontology) - -[`GET /admin/permanent-id-requests`](endpoints/permanent-id-requests.md#list-all-permanent-id-requests) - -[`GET /admin/permanent-id-requests/{request-id}`](endpoints/permanent-id-requests.md#get-any-permanent-id-request-details) - -[`POST /admin/permanent-id-requests/{request-id}/ezid`](endpoints/permanent-id-requests.md#create-a-permanent-id) - -[`POST /admin/permanent-id-requests/{request-id}/status`](endpoints/permanent-id-requests.md#update-the-status-of-a-permanent-id-request) - -[`DELETE /admin/workspaces`](endpoints/app-metadata.md#deleting-workspaces) - -[`GET /admin/workspaces`](endpoints/app-metadata.md#listing-workspaces) - -## apps - -[`GET /apps/{app-id}/comments`](endpoints/comments.md#listing-comments) - -[`POST /apps/{app-id}/comments`](endpoints/comments.md#creating-a-comment) - -[`PATCH /apps/{app-id}/comments/{comment-id}`](endpoints/comments.md#retractingreadmitting-a-comment) - -[`PATCH /apps/{app-id}/comments/{comment-id}`](endpoints/comments.md#retractingreadmitting-a-comment) - -## coge - -[`GET /coge/genomes`](endpoints/filesystem/coge.md#searching-for-genomes-in-coge) - -[`POST /coge/genomes/load`](endpoints/filesystem/coge.md#viewing-a-genome-file-in-coge) - -[`POST /coge/genomes/{genome-id}/export-fasta`](endpoints/filesystem/coge.md#exporting-coge-genome-data-to-irods) - -## favorites - -[`GET /favorites/filesystem`](endpoints/favorites.md#listing-stat-info-for-favorite-data) - -## filesystem - -[`PATCH /filesystem/entry/{entry-id}/comments/{comment-id}`](endpoints/comments.md#retractingreadmitting-a-comment) - -## permanent-id-requests - -[`GET /permanent-id-requests`](endpoints/permanent-id-requests.md#list-permanent-id-requests) - -[`POST /permanent-id-requests`](endpoints/permanent-id-requests.md#create-a-permanent-id-request) - -[`GET /permanent-id-requests/status-codes`](endpoints/permanent-id-requests.md#list-permanent-id-request-status-codes) - -[`GET /permanent-id-requests/types`](endpoints/permanent-id-requests.md#list-permanent-id-request-types) - -[`GET /permanent-id-requests/{request-id}`](endpoints/permanent-id-requests.md#list-permanent-id-request-details) - -## secured - -[`GET /secured/favorites/filesystem`](endpoints/favorites.md#listing-stat-info-for-favorite-data) - -[`DELETE /secured/favorites/filesystem/{favorite}`](endpoints/favorites.md#removing-a-data-resource-from-being-a-favorite) - -[`PUT /secured/favorites/filesystem/{favorite}`](endpoints/favorites.md#marking-a-data-resource-as-favorite) - -[`POST /secured/favorites/filter`](endpoints/favorites.md#filter-a-set-of-resources-for-favorites) - -[`GET /secured/fileio/download`](endpoints/fileio.md#downloading) - -[`POST /secured/fileio/save`](endpoints/fileio.md#save) - -[`POST /secured/fileio/saveas`](endpoints/fileio.md#save-as) - -[`POST /secured/fileio/upload`](endpoints/fileio.md#uploading) - -[`POST /secured/fileio/urlupload`](endpoints/fileio.md#url-uploads) - -[`POST /secured/filesystem/delete`](endpoints/filesystem/delete.md#deleting-files-andor-directories) - -[`POST /secured/filesystem/delete-contents`](endpoints/filesystem/delete.md#deleting-all-items-in-a-directory) - -[`POST /secured/filesystem/delete-tickets`](endpoints/filesystem/tickets.md#deleting-tickets) - -[`POST /secured/filesystem/directories`](endpoints/filesystem/directory-create.md#batch-directory-creation) - -[`GET /secured/filesystem/directory`](endpoints/filesystem/directory-listing.md#directory-list-non-recursive) - -[`POST /secured/filesystem/directory/create`](endpoints/filesystem/directory-create.md#directory-creation) - -[`GET /secured/filesystem/display-download`](endpoints/fileio.md#downloading) - -[`GET /secured/filesystem/entry/{entry-id}/comments`](endpoints/comments.md#listing-comments) - -[`POST /secured/filesystem/entry/{entry-id}/comments`](endpoints/comments.md#creating-a-comment) - -[`PATCH /secured/filesystem/entry/{entry-id}/comments/{comment-id}`](endpoints/comments.md#retractingreadmitting-a-comment) - -[`POST /secured/filesystem/exists`](endpoints/filesystem/existence.md#filedirectory-existence) - -[`GET /secured/filesystem/file/manifest`](endpoints/filesystem/manifest.md#file-manifest) - -[`GET /secured/filesystem/index`](endpoints/filesystem/search.md#endpoints) - -[`POST /secured/filesystem/list-tickets`](endpoints/filesystem/tickets.md#listing-tickets) - -[`POST /secured/filesystem/metadata/csv-parser`](endpoints/filesystem/metadata.md#adding-batch-metadata-to-multiple-paths-from-a-csv-file) - -[`GET /secured/filesystem/metadata/template/attr/{attribute-id}`](endpoints/filesystem/metadata.md#viewing-a-metadata-attribute) - -[`GET /secured/filesystem/metadata/template/{template-id}`](endpoints/filesystem/metadata.md#viewing-a-metadata-template) - -[`GET /secured/filesystem/metadata/template/{template-id}/blank-csv`](endpoints/filesystem/metadata.md#downloading-a-blank-template) - -[`GET /secured/filesystem/metadata/template/{template-id}/guide-csv`](endpoints/filesystem/metadata.md#downloading-a-template-guide) - -[`GET /secured/filesystem/metadata/templates`](endpoints/filesystem/metadata.md#listing-metadata-templates) - -[`POST /secured/filesystem/move`](endpoints/filesystem/move.md#moving-files-andor-directories) - -[`POST /secured/filesystem/move-contents`](endpoints/filesystem/move.md#moving-all-items-in-a-directory) - -[`GET /secured/filesystem/paged-directory`](endpoints/filesystem/directory-listing.md#paged-directory-listing) - -[`POST /secured/filesystem/path-list-creator`](endpoints/filesystem/path-lists.md#http-path-list-creator) - -[`POST /secured/filesystem/read-chunk`](endpoints/filesystem/read-chunk.md#reading-a-chunk-of-a-file) - -[`POST /secured/filesystem/read-csv-chunk`](endpoints/filesystem/csv-tsv-parsing.md#csvtsv-parsing) - -[`POST /secured/filesystem/rename`](endpoints/filesystem/rename.md#renaming-a-file-or-directory) - -[`POST /secured/filesystem/restore`](endpoints/filesystem/restore.md#restoring-a-file-or-directory-from-a-users-trash) - -[`POST /secured/filesystem/restore-all`](endpoints/filesystem/restore.md#restoring-all-items-in-a-users-trash) - -[`GET /secured/filesystem/root`](endpoints/filesystem/root-listing.md#top-level-root-listing) - -[`POST /secured/filesystem/stat`](endpoints/filesystem/stat.md#file-and-directory-status-information) - -[`POST /secured/filesystem/tickets`](endpoints/filesystem/tickets.md#creating-tickets) - -[`DELETE /secured/filesystem/trash`](endpoints/filesystem/empty-trash.md#emptying-a-users-trash-directory) - -[`POST /secured/filesystem/user-permissions`](endpoints/filesystem/permissions.md#listing-user-permissions) - -[`GET /secured/filesystem/{data-id}/metadata`](endpoints/filesystem/metadata.md#getting-metadata) - -[`POST /secured/filesystem/{data-id}/metadata`](endpoints/filesystem/metadata.md#setting-metadata) - -[`POST /secured/filesystem/{data-id}/metadata/copy`](endpoints/filesystem/metadata.md#copying-all-metadata-from-a-filefolder) - -[`POST /secured/filesystem/{data-id}/metadata/save`](endpoints/filesystem/metadata.md#exporting-metadata-to-a-file) - -[`POST /secured/filesystem/{data-id}/ore/save`](endpoints/filesystem/ore.md#generating-oai-ore-files-for-a-data-set) - -[`GET /secured/notifications/count-messages`](endpoints/notifications.md#obtaining-notification-counts) - -[`POST /secured/notifications/delete`](endpoints/notifications.md#marking-notifications-as-deleted) - -[`DELETE /secured/notifications/delete-all`](endpoints/notifications.md#marking-all-notifications-as-deleted) - -[`GET /secured/notifications/last-ten-messages`](endpoints/notifications.md#obtaining-the-ten-most-recent-notifications) - -[`POST /secured/notifications/mark-all-seen`](endpoints/notifications.md#marking-all-notifications-as-seen) - -[`GET /secured/notifications/messages`](endpoints/notifications.md#obtaining-notifications) - -[`POST /secured/notifications/seen`](endpoints/notifications.md#marking-notifications-as-seen) - -[`POST /secured/notifications/system/delete`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`DELETE /secured/notifications/system/delete-all`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`POST /secured/notifications/system/mark-all-received`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`POST /secured/notifications/system/mark-all-seen`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /secured/notifications/system/messages`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /secured/notifications/system/new-messages`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`POST /secured/notifications/system/received`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`POST /secured/notifications/system/seen`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /secured/notifications/system/unseen-messages`](endpoints/notifications.md#endpoints-for-system-messages-aka-system-notifications) - -[`GET /secured/notifications/unseen-messages`](endpoints/notifications.md#obtaining-unseen-notifications) - -[`GET /secured/oauth/access-code/{api-name}`](endpoints/callbacks.md#obtaining-oauth-authorization-codes) - -[`DELETE /secured/preferences`](endpoints/misc.md#removing-user-preferences) - -[`GET /secured/preferences`](endpoints/misc.md#retrieving-user-preferences) - -[`POST /secured/preferences`](endpoints/misc.md#saving-user-preferences) - -[`DELETE /secured/sessions`](endpoints/misc.md#removing-user-session-data) - -[`GET /secured/sessions`](endpoints/misc.md#retrieving-user-session-data) - -[`POST /secured/sessions`](endpoints/misc.md#saving-user-session-data) - -## send-notification - -[`POST /send-notification.`](endpoints/notifications.md#sending-an-arbitrary-notification) - -## uuid - -[`GET /uuid`](endpoints/misc.md#obtaining-identifiers) - diff --git a/docs/services/api/endpoints/endpoints.md b/docs/services/api/endpoints/endpoints.md deleted file mode 100644 index 86b8032..0000000 --- a/docs/services/api/endpoints/endpoints.md +++ /dev/null @@ -1,29 +0,0 @@ -# Terrain Endpoints - -All URLs referenced in endpoint documentation are listed as relative URLs with value names enclosed in braces. -For example, the service to get a list of workflow elements is accessed using the URL, `/get-workflow-elements/{element-type}`. -Where `{element-type}` refers to the type of workflow element that is being retrieved. -For example, to get a list of known property types, you can access the URL, `/get-workflow-elements/property-types`. -On the other hand, all examples use fully qualified URLs. - -Request and response bodies are in JSON format unless otherwise noted. - -* [App Metadata](app-metadata.md) - -* [Callback](callbacks.md) - -* [Categorizing Apps with Ontology Hierarchies](app-ontologies.md) - -* [Comments](comments.md) - -* [Data Search](filesystem/search.md) - -* [Favorites](favorites.md) - -* [Notification](notifications.md) - -* [Permanent ID Requests](permanent-id-requests.md) - -* [Quick Launches](quick-launches.md) - -* [Miscellaneous](misc.md) diff --git a/docs/services/dc.md b/docs/services/dc.md deleted file mode 100644 index 02944ba..0000000 --- a/docs/services/dc.md +++ /dev/null @@ -1,3 +0,0 @@ -[DataCommons](https://datacommons.cyverse.org) - -[Services] \ No newline at end of file diff --git a/docs/services/dnasubway.md b/docs/services/dnasubway.md deleted file mode 100644 index a06bbaa..0000000 --- a/docs/services/dnasubway.md +++ /dev/null @@ -1 +0,0 @@ -[DNA Subway](https://dnasubway.cyverse.org/) \ No newline at end of file diff --git a/docs/services/getting_started.md b/docs/services/getting_started.md deleted file mode 100644 index fa3f6be..0000000 --- a/docs/services/getting_started.md +++ /dev/null @@ -1,378 +0,0 @@ -# Getting Started with CyVerse Deployment - -This guide will help you get started with deploying and managing CyVerse infrastructure. Choose the path that matches your role and objectives. - -## Prerequisites - -!!! Success "Infrastructure Requirements" - - **Hardware/Cloud:** - - * Access to bare metal hardware, :simple-openstack: OpenStack cloud, or commercial provider (:simple-amazonaws: AWS, :simple-googlecloud: Google Cloud, :simple-azuredevops: Microsoft Azure) - * Minimum recommended: 8-node Kubernetes cluster with 32 CPU cores and 128GB RAM per node - * Persistent storage capability (NFS, iRODS, or cloud block storage) - * Public IP addresses and DNS management - - **Skills & Knowledge:** - - * Advanced understanding of :simple-linux: Linux system administration and file permissions - * Experience with :simple-kubernetes: Kubernetes (K8s) cluster management - * Familiarity with :simple-docker: Docker containerization - * Understanding of Infrastructure as Code (IaC) principles - * Experience with :simple-ansible: Ansible for configuration management - * Networking knowledge (DNS, load balancing, ingress controllers) - - **Tools & Access:** - - * :simple-github: GitHub or :simple-gitlab: GitLab with private repositories for sensitive credentials - * `kubectl`, `helm`, and other Kubernetes CLI tools - * Access via `ssh` to infrastructure nodes - * Web-enabled browser for administrative interfaces - - **Personal Qualities:** - - * Patience & Perseverance - complex distributed systems require methodical troubleshooting - -## Deployment Roadmap - -Deploying CyVerse follows a structured sequence. Each component builds upon the previous layers: - -```mermaid -graph TD - A[1. Kubernetes Cluster] --> B[2. Core Infrastructure] - B --> C[3. Core Services] - C --> D[4. Application Services] - D --> E[5. Databases] - E --> F[6. Discovery Environment] - - B --> B1[Storage: OpenEBS] - B --> B2[Networking: Ingress NGINX] - B --> B3[iRODS CSI Driver] - - C --> C1[KeyCloak: Authentication] - C --> C2[RabbitMQ: Message Queue] - C --> C3[Redis: Cache] - C --> C4[ElasticSearch: Search] - - D --> D1[User Portal] - D --> D2[VICE] - D --> D3[Exim4: Mail] - - E --> E1[PostgreSQL Databases] - E --> E2[Grouper] - E --> E3[Unleash] -``` - -### Component Dependencies - -| Component | Depends On | Purpose | -|-----------|------------|---------| -| Kubernetes Cluster | Hardware/Cloud | Container orchestration foundation | -| OpenEBS | Kubernetes | Persistent volume management | -| Ingress NGINX | Kubernetes | External traffic routing | -| KeyCloak | Kubernetes, PostgreSQL | User authentication and authorization | -| RabbitMQ | Kubernetes | Message passing between services | -| Redis HA | Kubernetes | Caching and session management | -| ElasticSearch | Kubernetes | Full-text search capabilities | -| Discovery Environment | All core services | Main user-facing application | -| VICE | DE, Kubernetes | Interactive computing environments | - -### Deployment Phases - -**Phase 1: Foundation (Weeks 1-2)** -- Set up Kubernetes cluster -- Deploy storage (OpenEBS) and networking (Ingress NGINX) -- Configure iRODS CSI driver for data storage -- Verify cluster health and resource allocation - -**Phase 2: Core Services (Week 3)** -- Deploy and configure KeyCloak for authentication -- Set up RabbitMQ message broker -- Deploy Redis HA for caching -- Configure ElasticSearch for search functionality -- Deploy monitoring (Jaeger) - -**Phase 3: Databases (Week 4)** -- Provision PostgreSQL databases for all services -- Initialize database schemas -- Configure backup and recovery procedures -- Set up Grouper for group management -- Deploy Unleash for feature flags - -**Phase 4: Applications (Weeks 5-6)** -- Deploy User Portal -- Configure Discovery Environment services -- Set up VICE for interactive applications -- Deploy mail services (Exim4) -- Configure external integrations - -**Phase 5: Verification & Tuning (Week 7)** -- End-to-end testing -- Performance optimization -- Security hardening -- Documentation of deployment specifics - ---- - -## Quick Start by Role - -### :material-server: For DevOps Engineers - -**Goal: Deploy CyVerse infrastructure from scratch** - -#### 1. Prepare Your Environment - -Start with the setup guides to configure your deployment tools: - -- [Ansible Setup](../guides/setup/ansible.md) - Configure Ansible for infrastructure automation -- [Docker Setup](../guides/setup/docker.md) - Set up Docker and container registry access -- [Database Setup](../guides/setup/database.md) - Prepare PostgreSQL deployment tools - -#### 2. Deploy Core Infrastructure - -Follow the deployment sequence: - -1. **[Kubernetes Cluster](../deployments/kubernetes-deploy.md)** - Deploy and configure your K8s cluster -2. **[Kubernetes Resources](../deployments/k8s-resources.md)** - Set up namespaces, resource quotas, and RBAC -3. **[Storage (OpenEBS)](../deployments/openebs.md)** - Deploy persistent volume management -4. **[iRODS CSI Driver](../deployments/irods-csi-driver.md)** - Connect to iRODS data storage -5. **[Networking](../deployments/ingress-nginx.md)** - Configure ingress for external access - -#### 3. Deploy Services - -With infrastructure in place, deploy the service layer: - -1. **[KeyCloak](../deployments/keycloak.md)** - Authentication and identity management -2. **[RabbitMQ](../deployments/RabbitMQ.md)** - Message broker for service communication -3. **[Redis HA](../deployments/redis-ha.md)** - High-availability caching -4. **[ElasticSearch](../deployments/elasticsearch.md)** - Search engine -5. **[Databases](../database/main.md)** - PostgreSQL for all services - -#### 4. Deploy Applications - -Finally, deploy the user-facing applications: - -1. **[Discovery Environment](../deployments/DiscoveryEnvironment.md)** - Main DE platform -2. **[User Portal](../deployments/userportal.md)** - User account management interface -3. **[VICE](../deployments/vice.md)** - Interactive computing - -#### 5. Verify Deployment - -- Check all pods are running: `kubectl get pods --all-namespaces` -- Verify services are accessible through ingress -- Test authentication flow through KeyCloak -- Review logs for errors: `kubectl logs -n ` - -**Next Steps:** -- Review [DevOps operational procedures](../guides/devops.md) -- Set up monitoring and alerting -- Configure backup procedures for databases -- Plan disaster recovery procedures - ---- - -### :octicons-people-24: For System Administrators - -**Goal: Manage users, apps, and daily operations** - -#### 1. Understand the Platform - -Before managing CyVerse, familiarize yourself with the architecture: - -- [System Overview](system_overview.md) - How CyVerse components work together -- [Discovery Environment](de.md) - The main user-facing platform -- [Data Store](ds.md) - iRODS-based data management - -#### 2. Learn Administrative Tools - -Review the admin guides for operational procedures: - -- **[DE Administration](../guides/de.md)** - User management, app publishing, VICE access grants -- **[Data Store Administration](../guides/ds.md)** - Data permissions, storage management -- **[User Portal Administration](../guides/user_portal.md)** - Account management - -#### 3. Common Administrative Tasks - -**User Management:** -- Grant VICE access to qualified users -- Manage user quotas and resource limits -- Process user support requests - -**App Publishing:** -- Review and approve tool integration requests -- Publish containerized apps to the Discovery Environment -- Test and validate app functionality - -**Data Management:** -- Process Permanent ID/DOI requests for data publishing -- Manage data sharing permissions -- Monitor storage usage and quotas - -#### 4. Resources - -- **[FAQ](../guides/faq.md)** - Common questions and troubleshooting -- **[Permanent ID Requests](../services/api/endpoints/permanent-id-requests.md)** - DOI workflow documentation -- **[Terrain API](../services/api_overview.md)** - Understanding the backend API - -**Next Steps:** -- Bookmark frequently used admin interfaces -- Join CyVerse staff communication channels -- Review common user support scenarios -- Familiarize yourself with escalation procedures - ---- - -### :material-code-braces: For Application Developers - -**Goal: Integrate with CyVerse APIs or contribute to the platform** - -#### 1. Understand the API Architecture - -Start with API fundamentals: - -- **[API Overview](api_overview.md)** - Introduction to Terrain API -- **[API Endpoint Index](../services/api/endpoint-index.md)** - Complete endpoint reference -- **[Developer Guide](../development/index.md)** - Development environment setup - -#### 2. Authentication & Access - -Learn how to authenticate with CyVerse services: - -- **[Authentication (KeyCloak)](keycloak.md)** - OAUTH 2.0 flow and token management -- **[Error Handling](../services/api/errors.md)** - API error codes and responses - -#### 3. Common Integration Patterns - -**Data Operations:** -- [Filesystem API](../services/api/endpoints/filesystem/directory-listing.md) - Browse and manage data in the Data Store -- [File I/O](../services/api/endpoints/fileio.md) - Upload and download files -- [Metadata](../services/api/endpoints/filesystem/metadata.md) - Attach metadata to data objects - -**Compute Operations:** -- [App Metadata](../services/api/endpoints/app-metadata.md) - Query available analysis tools -- [Job Submission](../services/api/endpoints/endpoints.md) - Launch computational analyses -- [Callbacks](../services/api/endpoints/callbacks.md) - Receive job status updates - -**User Interactions:** -- [Notifications](../services/api/endpoints/notifications.md) - Send messages to users -- [Comments](../services/api/endpoints/comments.md) - Enable collaborative annotations -- [Favorites](../services/api/endpoints/favorites.md) - Manage user bookmarks - -#### 4. Contributing to CyVerse - -If you're contributing code to CyVerse: - -- Review the **[Developer Guide](../development/index.md)** for contribution workflow -- Browse the [CyVerse-DE GitHub organization](https://github.com/cyverse-de){target=_blank} -- Test against the [live Terrain API](https://de.cyverse.org/terrain/docs/){target=_blank} - -**Migration Guides:** -- [Tapis v2 to v3 Migration](../services/api/tapis-v2-v3-migration.md) - Upgrade from legacy Tapis APIs - -**Next Steps:** -- Set up a CyVerse development account -- Review API rate limits and usage policies -- Explore example integrations in GitHub -- Join developer community channels - ---- - -## Installation Tools - -The [DevOps Guide](../guides/devops.md) provides a complete list of required software for managing a CyVerse deployment, including: - -- Kubernetes CLI tools (`kubectl`, `helm`) -- Ansible for configuration management -- Docker for container operations -- Database administration tools -- Monitoring and logging tools - ---- - -## Technology Stack Overview - -### Authentication - -CyVerse authentication relies upon: -- [LDAP](https://en.wikipedia.org/wiki/Lightweight_Directory_Access_Protocol) - Directory services for user accounts -- [OAUTH 2.0 protocol](https://www.rfc-editor.org/rfc/rfc6749) - Modern authentication standard -- [CILogon](https://www.cilogon.org/home) - Federated identity for research institutions - -See [KeyCloak](keycloak.md) for authentication deployment details. - -### :octicons-shield-24: Security Considerations - -Experience operating in a [Science DMZ](https://en.wikipedia.org/wiki/Science_DMZ_Network_Architecture) network architecture is beneficial for deploying CyVerse on university infrastructure. Key security topics include: - -- Firewall rules for high-performance data transfer -- Network segmentation between public and private services -- TLS certificate management for secure communications -- Secrets management for API keys and credentials - -### Core APIs - -[:material-terrain: Terrain API](api_overview.md) is the backbone service aggregating all Discovery Environment functionality. It provides a unified RESTful interface for: - -- Data management (Data Store operations) -- App execution (job submission and monitoring) -- User services (preferences, notifications, favorites) -- Administrative functions (user management, resource allocation) - -Detailed endpoint documentation is available in the [API endpoints section](api/endpoints/endpoints.md). - -### Platform Components - -**User-Facing Products:** - -- **[Discovery Environment](de.md)** - Web-based data science workbench with 1000+ pre-integrated tools -- **[Data Store](ds.md)** - 6+ PB iRODS-based storage with data management, hosting, and sharing -- **[Data Commons](dc.md)** - Community data repository with DataCite DOI publishing -- **[VICE](../deployments/vice.md)** - Visual Interactive Computing (Jupyter, RStudio, Shiny apps) -- **[BisQue](bisque.md)** - Browser-based large image analysis platform -- **[DNA Subway](dnasubway.md)** - Educational genomics software for students - -**Backend Services:** - -- **[Core Services](services_overview.md)** - Microservices architecture overview -- **[Authentication (KeyCloak)](keycloak.md)** - Identity and access management -- **[Cloud Services (CACAO)](cloud.md)** - Multi-cloud automation and orchestration - -### :simple-kubernetes: Deployment Platform - -All CyVerse services are deployed on [Kubernetes (K8s)](../deployments/deployment_overview.md) using: - -- Helm charts for package management -- Ansible playbooks for configuration -- GitOps practices for infrastructure as code -- Namespace isolation for service boundaries - -See the complete [Deployments overview](../deployments/deployment_overview.md) for details. - -### :octicons-database-24: Database Infrastructure - -CyVerse uses [:simple-postgresql: PostgreSQL](https://www.postgresql.org/) as its primary database platform. Each service maintains its own database for isolation and independent scaling: - -- [DE Database](../database/de-db.md) - Discovery Environment core data -- [Metadata Database](../database/metadata-db.md) - User-defined metadata -- [Notifications Database](../database/notifications-db.md) - User notification system -- [KeyCloak Database](../database/keycloak-db.md) - Authentication data -- [And more...](../database/main.md) - ---- - -## Next Steps - -After completing your getting started tasks: - -1. **Verify Deployment** - Test core functionality end-to-end -2. **Review Troubleshooting Resources** - Familiarize yourself with common issues in the [FAQ](../guides/faq.md) -3. **Set Up Monitoring** - Implement logging and alerting for production operations -4. **Join the Community** - Connect with other CyVerse operators and developers via [GitHub](https://github.com/cyverse-de){target=_blank} -5. **Plan for Scale** - Review performance tuning and capacity planning for your workload - -**Need Help?** - -- Check the [FAQ](../guides/faq.md) for common questions -- Review deployment-specific documentation in the [Deployment Guide](../deployments/deployment_overview.md) -- Explore the [API Reference](api_overview.md) for integration details -- Consult the [Developer Guide](../development/index.md) for contribution workflow diff --git a/docs/services/keycloak.md b/docs/services/keycloak.md deleted file mode 100644 index cea3ac4..0000000 --- a/docs/services/keycloak.md +++ /dev/null @@ -1,29 +0,0 @@ -CyVerse Authentication via Keycloak, CILogon, & OAUTH - -``` mermaid -sequenceDiagram - autonumber - User->>Browser: Click on external Auth - Browser-->>Keycloak: Authentication request (TOKEN) - loop - Keycloak-->>Browser: Browser opened with ../auth?=client_id=de-prod=TOKEN - end - Keycloak-->>CILogon: Auth Request - User->>CILogon: Enter Credentials - Keycloak-->>OAUTH: Auth Response - CILogon-->>Keycloak: Auth Response - Browser-->>OAUTH: Ask for Token - OAUTH-->>Browser: Retrieve Token -``` - -[comment]: <> (![keycloak](../assets/de/keycloak.svg)) - -**Mermaid Diagram** Users authenticate (starting on left side) via their browser, which passes through Keycloack to either CILogon or OAUTH. - -[:material-shield-key: Keycloak](https://kc.cyverse.org){target=_blank} service manages authenticaation via [CILogon](https://cilogon.org/){target=_blank} and [OAUTH](https://oauth.net/){target=_blank}. - -## Keycloak - -Keycloak is provisioned and deployed as part of the main K8s - -[:simple-github: Keycloak](https://github.com/cyverse/ansible-kubernetes-keycloak){target=_blank} provisioning with K8s \ No newline at end of file diff --git a/docs/services/services_overview.md b/docs/services/services_overview.md deleted file mode 100644 index ae3c115..0000000 --- a/docs/services/services_overview.md +++ /dev/null @@ -1,26 +0,0 @@ -# Products & Services Overview - -CyVerse provides multiple end-user services. - -
- ![layercake](../assets/layerCake.svg){width=800} -
The top of the CyVerse layer cake includes its Products & Services
-
- -Primarily the Products and Services layers are served directly over `https://` websites which then link to the CyVerse Terrain API, K8s clusters, Data Store, and Authentication layers. - -[API (Terrain)](api_overview.md) - -[Authentication](keycloak.md) - -[BisQue](bisque.md) - -[Cloud Services](cloud.md) - -[Data Publishing](dc.md) - -[Data Storage](ds.md) - -[Discovery Environment](de.md) - -[DNA Subway](dnasubway.md) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index f82245d..ab6fca2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,10 @@ site_url: https://cyverse.github.io/docs site_description: CyVerse USA Documentation site_author: 'CyVerse USA' +# NOTE: the published site is built with Zensical (see zensical.toml, which is +# what .github/workflows/gh-actions.yml runs). This file is kept in sync for +# local `mkdocs serve` previews; change both when the navigation changes. + # uncomment to show the github icon in the header with # number of forks & stars repo_url: https://github.com/cyverse/docs @@ -16,10 +20,6 @@ copyright: 'Copyright © 2016-2026 CyVerse' extra_javascript: - javascripts/intercom.js -# logos -#medunilogo: 'images/logo-meduni-graz.png' -#unigrazlogo: 'images/universitaet_graz_logo_signet.png' -#tugrazlogo: 'images/TUG.png' imgwidth: 70px; theme: highlightjs: true @@ -57,74 +57,147 @@ theme: icon: material/weather-sunny name: Switch to system preference -# Navigation Pane +# Navigation pane. Ordered to follow a CyVerse deployment: plan, build the +# foundation, then the layers above it, then operate what you built. nav: - - About: 'index.md' - - DevOps: - - Getting Started: 'services/getting_started.md' - - System Overview: 'services/system_overview.md' - - Subscriptions: 'services/subscriptions.md' - - API (Terrain): - - Overview: 'services/api_overview.md' - - Endpoints: - - Overview: 'services/api/endpoints/endpoints.md' - - App Metadata: 'services/api/endpoints/app-metadata.md' - - Callbacks: 'services/api/endpoints/callbacks.md' - - Comments: 'services/api/endpoints/comments.md' - - Data Search: 'services/api/endpoints/filesystem/search.md' - - Favorites: 'services/api/endpoints/favorites.md' - - Notification: 'services/api/endpoints/notifications.md' - - Ontology Hierarchies: 'services/api/endpoints/app-ontologies.md' - - Permanent ID Requests: 'services/api/endpoints/permanent-id-requests.md' - - Quick Launches: 'services/api/endpoints/quick-launches.md' - - Miscellaneous: 'services/api/endpoints/misc.md' - - Endpoints Index: 'services/api/endpoint-index.md' - - Errors: 'services/api/errors.md' - - Products & Services: - - Overview: 'services/services_overview.md' - - Authentication: 'services/keycloak.md' - - BisQue: [services/bisque.md] - - Cloud Services: "services/cloud.md" - - Data Publishing: 'services/dc.md' - - Data Storage: 'services/ds.md' - - Discovery Environment: 'services/de.md' - - DNA Subway: 'services/dnasubway.md' - - Deployments: - - Overview: 'deployments/deployment_overview.md' - - Kubernetes (K8s): 'deployments/kubernetes-deploy.md' - - User Portal: 'deployments/userportal.md' - - OpenEBS: 'deployments/openebs.md' - - KeyCloak: 'deployments/keycloak.md' - - Mail: 'deployments/exim4.md' - - Redis HA: 'deployments/redis-ha.md' - - K8s Resources: 'deployments/k8s-resources.md' - - K8s Name Spaces: 'deployments/k8s-namespace.md' - - ElasticSearch: 'deployments/elasticsearch.md' - - RabbitMQ: 'deployments/RabbitMQ.md' # move this since its not k8s - - Unleash: 'deployments/unleash.md' - - Grouper: 'deployments/grouper.md' - - iRODS CSI Driver: 'deployments/irods-csi-driver.md' - - Local Exim: 'deployments/local-exim.md' - - Discovery Environment: 'deployments/DiscoveryEnvironment.md' - - VICE: 'deployments/vice.md' - - Jaeger: 'deployments/jaeger.md' - - Databases: - - Overview: 'database/main.md' - - DE: 'database/de-db.md' - - Metadata: 'database/metadata-db.md' - - KeyCloak: 'database/keycloak-db.md' - - Notifications: 'database/notifications-db.md' - - Unleash: 'database/unleash-db.md' - - Grouper: 'database/grouper-db.md' - - QMS: 'database/qms-db.md' - - Portal: 'database/portal-db.md' - - Admin Guides: - - DevOps: 'guides/devops.md' - - Discovery Environment: 'guides/de.md' - - Data Store: 'guides/ds.md' - - User Portal: 'guides/user_portal.md' - - Frequently Asked Questions: 'guides/faq.md' + - Home: + - index.md + - About: 'about.md' + - Update Log: 'log.md' + - Architecture: + - architecture/index.md + - System Overview: 'architecture/system-overview.md' + - Component Inventory & Sizing: 'architecture/component-inventory.md' + - Network Requirements: 'architecture/network-requirements.md' + - Kubernetes Namespaces: 'architecture/namespaces.md' + - Deployment: + - deployment/index.md + - From Scratch: 'deployment/from-scratch.md' + - 'Phase 0: Planning': + - deployment/planning/index.md + - Prerequisites: 'deployment/planning/prerequisites.md' + - Ansible: 'deployment/planning/ansible.md' + - Docker: 'deployment/planning/docker.md' + - 'Phase 1: Foundation': + - deployment/01-foundation/index.md + - HAProxy: 'deployment/01-foundation/haproxy.md' + - PostgreSQL: 'deployment/01-foundation/postgresql.md' + - RabbitMQ: 'deployment/01-foundation/rabbitmq.md' + - 'Phase 2: Databases': + - deployment/02-databases/index.md + - Migrations: 'deployment/02-databases/migrations.md' + - iCAT: 'deployment/02-databases/icat.md' + - DE: 'deployment/02-databases/de.md' + - Metadata: 'deployment/02-databases/metadata.md' + - Notifications: 'deployment/02-databases/notifications.md' + - QMS: 'deployment/02-databases/qms.md' + - Keycloak: 'deployment/02-databases/keycloak.md' + - Grouper: 'deployment/02-databases/grouper.md' + - Unleash: 'deployment/02-databases/unleash.md' + - Portal: 'deployment/02-databases/portal.md' + - 'Phase 3: Data Store': + - deployment/03-data-store/index.md + - iRODS Catalog Provider: 'deployment/03-data-store/irods-provider.md' + - DE Integration: 'deployment/03-data-store/de-integration.md' + - 'Phase 4: Kubernetes': + - deployment/04-kubernetes/index.md + - Cluster: 'deployment/04-kubernetes/cluster.md' + - Cluster Resources: 'deployment/04-kubernetes/resources.md' + - cert-manager: 'deployment/04-kubernetes/cert-manager.md' + - Ingress: 'deployment/04-kubernetes/ingress.md' + - Storage: 'deployment/04-kubernetes/storage.md' + - Harbor: 'deployment/04-kubernetes/harbor.md' + - Argo Workflows: 'deployment/04-kubernetes/argo.md' + - 'Phase 5: Core Services': + - deployment/05-core-services/index.md + - OpenLDAP: 'deployment/05-core-services/openldap.md' + - Keycloak: 'deployment/05-core-services/keycloak.md' + - Grouper: 'deployment/05-core-services/grouper.md' + - OpenSearch: 'deployment/05-core-services/opensearch.md' + - Elasticsearch (legacy): 'deployment/05-core-services/elasticsearch.md' + - NATS: 'deployment/05-core-services/nats.md' + - Redis HA: 'deployment/05-core-services/redis-ha.md' + - Unleash: 'deployment/05-core-services/unleash.md' + - iRODS CSI Driver: 'deployment/05-core-services/irods-csi-driver.md' + - Mail: 'deployment/05-core-services/mail.md' + - Jaeger: 'deployment/05-core-services/jaeger.md' + - 'Phase 6: Applications': + - deployment/06-applications/index.md + - Discovery Environment: 'deployment/06-applications/discovery-environment.md' + - VICE: 'deployment/06-applications/vice.md' + - User Portal: 'deployment/06-applications/user-portal.md' + - 'Phase 7: Post-Install': + - deployment/07-post-install/index.md + - Bootstrap: 'deployment/07-post-install/bootstrap.md' + - Verification: 'deployment/07-post-install/verification.md' + - Troubleshooting: 'deployment/07-post-install/troubleshooting.md' + - Platform: + - platform/index.md + - Discovery Environment: 'platform/discovery-environment.md' + - Data Store: 'platform/data-store.md' + - Data Commons: 'platform/data-commons.md' + - Authentication: 'platform/authentication.md' + - Subscriptions: 'platform/subscriptions.md' + - Cloud Services: 'platform/cloud.md' + - BisQue: 'platform/bisque.md' + - DNA Subway: 'platform/dna-subway.md' + - Operations: + - operations/index.md + - Discovery Environment: 'operations/discovery-environment.md' + - Data Store: 'operations/data-store.md' + - User Portal: 'operations/user-portal.md' + - FAQ: 'operations/faq.md' + - API: + - api/index.md + - Terrain: 'api/terrain.md' + - Endpoints: + - api/endpoints/index.md + - Overview: 'api/endpoints/endpoints.md' + - App Metadata: 'api/endpoints/app-metadata.md' + - App Ontologies: 'api/endpoints/app-ontologies.md' + - Callbacks: 'api/endpoints/callbacks.md' + - Comments: 'api/endpoints/comments.md' + - Favorites: 'api/endpoints/favorites.md' + - File I/O: 'api/endpoints/fileio.md' + - Notifications: 'api/endpoints/notifications.md' + - Permanent ID Requests: 'api/endpoints/permanent-id-requests.md' + - Quick Launches: 'api/endpoints/quick-launches.md' + - Terrain & Apps: 'api/endpoints/terrain-v-apps.md' + - Miscellaneous: 'api/endpoints/misc.md' + - Filesystem: + - api/endpoints/filesystem/index.md + - Root Listing: 'api/endpoints/filesystem/root-listing.md' + - Directory Listing: 'api/endpoints/filesystem/directory-listing.md' + - Directory Create: 'api/endpoints/filesystem/directory-create.md' + - Stat: 'api/endpoints/filesystem/stat.md' + - Existence: 'api/endpoints/filesystem/existence.md' + - Manifest: 'api/endpoints/filesystem/manifest.md' + - Read Chunk: 'api/endpoints/filesystem/read-chunk.md' + - CSV/TSV Parsing: 'api/endpoints/filesystem/csv-tsv-parsing.md' + - Move: 'api/endpoints/filesystem/move.md' + - Rename: 'api/endpoints/filesystem/rename.md' + - Delete: 'api/endpoints/filesystem/delete.md' + - Restore: 'api/endpoints/filesystem/restore.md' + - Empty Trash: 'api/endpoints/filesystem/empty-trash.md' + - Metadata: 'api/endpoints/filesystem/metadata.md' + - Search: 'api/endpoints/filesystem/search.md' + - Permissions: 'api/endpoints/filesystem/permissions.md' + - Sharing: 'api/endpoints/filesystem/sharing.md' + - Tickets: 'api/endpoints/filesystem/tickets.md' + - CoGe: 'api/endpoints/filesystem/coge.md' + - OAI-ORE: 'api/endpoints/filesystem/ore.md' + - Path Lists: 'api/endpoints/filesystem/path-lists.md' + - Errors: 'api/endpoints/filesystem/errors.md' + - Endpoint Index: 'api/endpoint-index.md' + - Errors: 'api/errors.md' + - Tapis v2 to v3 Migration: 'api/tapis-v2-v3-migration.md' + - Development: + - development/index.md + - Developer Guide: 'development/developer-guide.md' + - References: + - references/index.md + - Pilot Deployment Record: 'references/pilot-deployment-record.md' # Customization extra: @@ -164,15 +237,15 @@ markdown_extensions: - pymdownx.critic - pymdownx.details - pymdownx.emoji: - emoji_index: !!python/name:materialx.emoji.twemoji - emoji_generator: !!python/name:materialx.emoji.to_svg + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg - pymdownx.highlight - pymdownx.inlinehilite - pymdownx.keys - pymdownx.magiclink: repo_url_shorthand: true - user: squidfunk - repo: mkdocs-material + user: cyverse + repo: docs - pymdownx.mark - pymdownx.smartsymbols - pymdownx.superfences: @@ -180,10 +253,11 @@ markdown_extensions: - name: mermaid class: mermaid format: !!python/name:pymdownx.superfences.fence_code_format - - pymdownx.tabbed + - pymdownx.tabbed: + alternate_style: true - pymdownx.tasklist: custom_checkbox: true - pymdownx.tilde extra_css: - - stylesheets/extra.css \ No newline at end of file + - stylesheets/custom.css diff --git a/zensical.toml b/zensical.toml index cd503a2..ad06ea5 100644 --- a/zensical.toml +++ b/zensical.toml @@ -15,7 +15,7 @@ copyright = "Copyright © 2016-2026 CyVerse" # Theme configuration [project.theme] -name = "classic" # Use classic theme for Material for MkDocs appearance +name = "material" logo = "assets/de/cyverse_ball_2022.png" favicon = "images/favicon.ico" @@ -45,137 +45,178 @@ text = "Open Sans" code = "Roboto Mono" # Navigation structure +# Ordered to follow a CyVerse deployment: plan, build the foundation, then the +# layers above it, then operate what you built. Keep mkdocs.yml in sync. + [[project.nav]] -About = "index.md" +Home = [ + "index.md", + { About = "about.md" }, + { "Update Log" = "log.md" } +] [[project.nav]] -"Getting Started" = [ - { "Overview & Prerequisites" = "services/getting_started.md" }, - { "System Overview" = "services/system_overview.md" }, - { "Subscriptions" = "services/subscriptions.md" } +Architecture = [ + "architecture/index.md", + { "System Overview" = "architecture/system-overview.md" }, + { "Component Inventory & Sizing" = "architecture/component-inventory.md" }, + { "Network Requirements" = "architecture/network-requirements.md" }, + { "Kubernetes Namespaces" = "architecture/namespaces.md" } ] [[project.nav]] -"Deployment Guide" = [ - { "Overview" = "deployments/deployment_overview.md" }, - { "Prerequisites & Setup" = [ - { "Ansible Setup" = "guides/setup/ansible.md" }, - { "Docker Setup" = "guides/setup/docker.md" }, - { "Database Setup" = "guides/setup/database.md" } +Deployment = [ + "deployment/index.md", + { "From Scratch" = "deployment/from-scratch.md" }, + { "Phase 0: Planning" = [ + "deployment/planning/index.md", + { Prerequisites = "deployment/planning/prerequisites.md" }, + { Ansible = "deployment/planning/ansible.md" }, + { Docker = "deployment/planning/docker.md" } + ]}, + { "Phase 1: Foundation" = [ + "deployment/01-foundation/index.md", + { HAProxy = "deployment/01-foundation/haproxy.md" }, + { PostgreSQL = "deployment/01-foundation/postgresql.md" }, + { RabbitMQ = "deployment/01-foundation/rabbitmq.md" } + ]}, + { "Phase 2: Databases" = [ + "deployment/02-databases/index.md", + { Migrations = "deployment/02-databases/migrations.md" }, + { iCAT = "deployment/02-databases/icat.md" }, + { DE = "deployment/02-databases/de.md" }, + { Metadata = "deployment/02-databases/metadata.md" }, + { Notifications = "deployment/02-databases/notifications.md" }, + { QMS = "deployment/02-databases/qms.md" }, + { Keycloak = "deployment/02-databases/keycloak.md" }, + { Grouper = "deployment/02-databases/grouper.md" }, + { Unleash = "deployment/02-databases/unleash.md" }, + { Portal = "deployment/02-databases/portal.md" } + ]}, + { "Phase 3: Data Store" = [ + "deployment/03-data-store/index.md", + { "iRODS Catalog Provider" = "deployment/03-data-store/irods-provider.md" }, + { "DE Integration" = "deployment/03-data-store/de-integration.md" } ]}, - { "Core Infrastructure" = [ - { "Kubernetes Cluster" = "deployments/kubernetes-deploy.md" }, - { "Kubernetes Resources" = "deployments/k8s-resources.md" }, - { "Kubernetes Namespaces" = "deployments/k8s-namespace.md" }, - { "Storage (OpenEBS)" = "deployments/openebs.md" }, - { "iRODS CSI Driver" = "deployments/irods-csi-driver.md" }, - { "Networking (Ingress NGINX)" = "deployments/ingress-nginx.md" } + { "Phase 4: Kubernetes" = [ + "deployment/04-kubernetes/index.md", + { Cluster = "deployment/04-kubernetes/cluster.md" }, + { "Cluster Resources" = "deployment/04-kubernetes/resources.md" }, + { "cert-manager" = "deployment/04-kubernetes/cert-manager.md" }, + { Ingress = "deployment/04-kubernetes/ingress.md" }, + { Storage = "deployment/04-kubernetes/storage.md" }, + { Harbor = "deployment/04-kubernetes/harbor.md" }, + { "Argo Workflows" = "deployment/04-kubernetes/argo.md" } ]}, - { "Core Services" = [ - { "Authentication (KeyCloak)" = "deployments/keycloak.md" }, - { "Message Queue (RabbitMQ)" = "deployments/RabbitMQ.md" }, - { "Cache (Redis HA)" = "deployments/redis-ha.md" }, - { "Search (ElasticSearch)" = "deployments/elasticsearch.md" }, - { "Monitoring (Jaeger)" = "deployments/jaeger.md" } + { "Phase 5: Core Services" = [ + "deployment/05-core-services/index.md", + { OpenLDAP = "deployment/05-core-services/openldap.md" }, + { Keycloak = "deployment/05-core-services/keycloak.md" }, + { Grouper = "deployment/05-core-services/grouper.md" }, + { OpenSearch = "deployment/05-core-services/opensearch.md" }, + { "Elasticsearch (legacy)" = "deployment/05-core-services/elasticsearch.md" }, + { NATS = "deployment/05-core-services/nats.md" }, + { "Redis HA" = "deployment/05-core-services/redis-ha.md" }, + { Unleash = "deployment/05-core-services/unleash.md" }, + { "iRODS CSI Driver" = "deployment/05-core-services/irods-csi-driver.md" }, + { Mail = "deployment/05-core-services/mail.md" }, + { Jaeger = "deployment/05-core-services/jaeger.md" } ]}, - { "Application Services" = [ - { "Discovery Environment" = "deployments/DiscoveryEnvironment.md" }, - { "User Portal" = "deployments/userportal.md" }, - { "VICE" = "deployments/vice.md" }, - { "Mail (Exim4)" = "deployments/exim4.md" }, - { "Local Exim" = "deployments/local-exim.md" } + { "Phase 6: Applications" = [ + "deployment/06-applications/index.md", + { "Discovery Environment" = "deployment/06-applications/discovery-environment.md" }, + { VICE = "deployment/06-applications/vice.md" }, + { "User Portal" = "deployment/06-applications/user-portal.md" } ]}, - { "Supporting Services" = [ - { "Grouper" = "deployments/grouper.md" }, - { "Unleash" = "deployments/unleash.md" } + { "Phase 7: Post-Install" = [ + "deployment/07-post-install/index.md", + { Bootstrap = "deployment/07-post-install/bootstrap.md" }, + { Verification = "deployment/07-post-install/verification.md" }, + { Troubleshooting = "deployment/07-post-install/troubleshooting.md" } ]} ] [[project.nav]] -"Administration Guide" = [ - { "DevOps Tools" = "guides/devops.md" }, - { "Discovery Environment" = "guides/de.md" }, - { "Data Store" = "guides/ds.md" }, - { "User Portal" = "guides/user_portal.md" }, - { "FAQ" = "guides/faq.md" } +Platform = [ + "platform/index.md", + { "Discovery Environment" = "platform/discovery-environment.md" }, + { "Data Store" = "platform/data-store.md" }, + { "Data Commons" = "platform/data-commons.md" }, + { Authentication = "platform/authentication.md" }, + { Subscriptions = "platform/subscriptions.md" }, + { "Cloud Services" = "platform/cloud.md" }, + { BisQue = "platform/bisque.md" }, + { "DNA Subway" = "platform/dna-subway.md" } ] [[project.nav]] -"API Reference" = [ - { "Overview" = "services/api_overview.md" }, - { "Endpoints" = [ - { "Overview" = "services/api/endpoints/endpoints.md" }, - { "App Metadata" = "services/api/endpoints/app-metadata.md" }, - { "App Ontologies" = "services/api/endpoints/app-ontologies.md" }, - { "Callbacks" = "services/api/endpoints/callbacks.md" }, - { "Comments" = "services/api/endpoints/comments.md" }, - { "Favorites" = "services/api/endpoints/favorites.md" }, - { "File I/O" = "services/api/endpoints/fileio.md" }, - { "Filesystem" = [ - { "Search" = "services/api/endpoints/filesystem/search.md" }, - { "Directory Listing" = "services/api/endpoints/filesystem/directory-listing.md" }, - { "Directory Create" = "services/api/endpoints/filesystem/directory-create.md" }, - { "Root Listing" = "services/api/endpoints/filesystem/root-listing.md" }, - { "File Operations" = "services/api/endpoints/filesystem/read-chunk.md" }, - { "CSV/TSV Parsing" = "services/api/endpoints/filesystem/csv-tsv-parsing.md" }, - { "Move" = "services/api/endpoints/filesystem/move.md" }, - { "Rename" = "services/api/endpoints/filesystem/rename.md" }, - { "Delete" = "services/api/endpoints/filesystem/delete.md" }, - { "Restore" = "services/api/endpoints/filesystem/restore.md" }, - { "Empty Trash" = "services/api/endpoints/filesystem/empty-trash.md" }, - { "Metadata" = "services/api/endpoints/filesystem/metadata.md" }, - { "Permissions" = "services/api/endpoints/filesystem/permissions.md" }, - { "Sharing" = "services/api/endpoints/filesystem/sharing.md" }, - { "Tickets" = "services/api/endpoints/filesystem/tickets.md" }, - { "CoGe" = "services/api/endpoints/filesystem/coge.md" }, - { "ORE" = "services/api/endpoints/filesystem/ore.md" }, - { "Manifest" = "services/api/endpoints/filesystem/manifest.md" }, - { "Path Lists" = "services/api/endpoints/filesystem/path-lists.md" }, - { "Stat" = "services/api/endpoints/filesystem/stat.md" }, - { "Existence" = "services/api/endpoints/filesystem/existence.md" }, - { "Errors" = "services/api/endpoints/filesystem/errors.md" } - ]}, - { "Notifications" = "services/api/endpoints/notifications.md" }, - { "Permanent ID Requests" = "services/api/endpoints/permanent-id-requests.md" }, - { "Quick Launches" = "services/api/endpoints/quick-launches.md" }, - { "Terrain vs Apps" = "services/api/endpoints/terrain-v-apps.md" }, - { "Miscellaneous" = "services/api/endpoints/misc.md" } - ]}, - { "Endpoint Index" = "services/api/endpoint-index.md" }, - { "Error Handling" = "services/api/errors.md" }, - { "Migration Guides" = [ - { "Tapis v2 to v3" = "services/api/tapis-v2-v3-migration.md" } - ]} +Operations = [ + "operations/index.md", + { "Discovery Environment" = "operations/discovery-environment.md" }, + { "Data Store" = "operations/data-store.md" }, + { "User Portal" = "operations/user-portal.md" }, + { FAQ = "operations/faq.md" } ] [[project.nav]] -"Developer Guide" = "development/index.md" +API = [ + "api/index.md", + { Terrain = "api/terrain.md" }, + { Endpoints = [ + "api/endpoints/index.md", + { Overview = "api/endpoints/endpoints.md" }, + { "App Metadata" = "api/endpoints/app-metadata.md" }, + { "App Ontologies" = "api/endpoints/app-ontologies.md" }, + { Callbacks = "api/endpoints/callbacks.md" }, + { Comments = "api/endpoints/comments.md" }, + { Favorites = "api/endpoints/favorites.md" }, + { "File I/O" = "api/endpoints/fileio.md" }, + { Notifications = "api/endpoints/notifications.md" }, + { "Permanent ID Requests" = "api/endpoints/permanent-id-requests.md" }, + { "Quick Launches" = "api/endpoints/quick-launches.md" }, + { "Terrain & Apps" = "api/endpoints/terrain-v-apps.md" }, + { Miscellaneous = "api/endpoints/misc.md" }, + { Filesystem = [ + "api/endpoints/filesystem/index.md", + { "Root Listing" = "api/endpoints/filesystem/root-listing.md" }, + { "Directory Listing" = "api/endpoints/filesystem/directory-listing.md" }, + { "Directory Create" = "api/endpoints/filesystem/directory-create.md" }, + { Stat = "api/endpoints/filesystem/stat.md" }, + { Existence = "api/endpoints/filesystem/existence.md" }, + { Manifest = "api/endpoints/filesystem/manifest.md" }, + { "Read Chunk" = "api/endpoints/filesystem/read-chunk.md" }, + { "CSV/TSV Parsing" = "api/endpoints/filesystem/csv-tsv-parsing.md" }, + { Move = "api/endpoints/filesystem/move.md" }, + { Rename = "api/endpoints/filesystem/rename.md" }, + { Delete = "api/endpoints/filesystem/delete.md" }, + { Restore = "api/endpoints/filesystem/restore.md" }, + { "Empty Trash" = "api/endpoints/filesystem/empty-trash.md" }, + { Metadata = "api/endpoints/filesystem/metadata.md" }, + { Search = "api/endpoints/filesystem/search.md" }, + { Permissions = "api/endpoints/filesystem/permissions.md" }, + { Sharing = "api/endpoints/filesystem/sharing.md" }, + { Tickets = "api/endpoints/filesystem/tickets.md" }, + { CoGe = "api/endpoints/filesystem/coge.md" }, + { "OAI-ORE" = "api/endpoints/filesystem/ore.md" }, + { "Path Lists" = "api/endpoints/filesystem/path-lists.md" }, + { Errors = "api/endpoints/filesystem/errors.md" } + ]} + ]}, + { "Endpoint Index" = "api/endpoint-index.md" }, + { Errors = "api/errors.md" }, + { "Tapis v2 to v3 Migration" = "api/tapis-v2-v3-migration.md" } +] [[project.nav]] -"System Architecture" = [ - { "Overview" = "services/services_overview.md" }, - { "Products & Services" = [ - { "Discovery Environment" = "services/de.md" }, - { "Data Store" = "services/ds.md" }, - { "Data Commons" = "services/dc.md" }, - { "BisQue" = "services/bisque.md" }, - { "DNA Subway" = "services/dnasubway.md" }, - { "Cloud Services (CACAO)" = "services/cloud.md" }, - { "Authentication (KeyCloak)" = "services/keycloak.md" } - ]} +Development = [ + "development/index.md", + { "Developer Guide" = "development/developer-guide.md" } ] [[project.nav]] -"Database Reference" = [ - { "Overview" = "database/main.md" }, - { "DE Database" = "database/de-db.md" }, - { "Metadata Database" = "database/metadata-db.md" }, - { "KeyCloak Database" = "database/keycloak-db.md" }, - { "Notifications Database" = "database/notifications-db.md" }, - { "Unleash Database" = "database/unleash-db.md" }, - { "Grouper Database" = "database/grouper-db.md" }, - { "QMS Database" = "database/qms-db.md" }, - { "Portal Database" = "database/portal-db.md" } +References = [ + "references/index.md", + { "Pilot Deployment Record" = "references/pilot-deployment-record.md" } ] # Extra configuration @@ -211,7 +252,7 @@ link = "https://www.youtube.com/c/CyverseOrgProject" extra_javascript = ["javascripts/intercom.js"] # CSS files -extra_css = ["stylesheets/extra.css"] +extra_css = ["stylesheets/custom.css"] # Markdown extensions configuration # Zensical enables a sensible default set of extensions automatically