diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c79fac76..8ba8d278 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,6 +11,7 @@ jobs: Docs: permissions: contents: read - uses: tskit-dev/.github/.github/workflows/docs.yml@v19 + uses: petrelharp/tskit.github/.github/workflows/docs.yml@a35059f7617ec56cb61be0538b4f5554c41f66a1 with: install-slim: true + install-slim-branch: "multitrait" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 8b29520b..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Lint - -on: - pull_request: - push: - branches: [main] - merge_group: - -jobs: - Lint: - permissions: - contents: read - uses: tskit-dev/.github/.github/workflows/lint.yml@v19 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index c41d6293..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: Tests - -on: - pull_request: - push: - branches: [main] - merge_group: - -permissions: - contents: read - -jobs: - packaging: - name: Python packaging - uses: tskit-dev/.github/.github/workflows/python-packaging.yml@v19 - - test: - name: Python - runs-on: ${{ matrix.os }} - strategy: - matrix: - python: ["3.11", "3.13"] - os: [macos-latest, ubuntu-24.04, windows-latest] - sys: [mingw64, ucrt64] - env: [x86_64, ucrt-x86_64] - exclude: - - os: macos-latest - sys: ucrt64 - - os: macos-latest - sys: mingw64 - env: ucrt-x86_64 - - os: ubuntu-24.04 - sys: ucrt64 - - os: ubuntu-24.04 - sys: mingw64 - env: ucrt-x86_64 - - os: windows-latest - sys: ucrt64 - env: x86_64 - - os: windows-latest - sys: mingw64 - env: ucrt-x86_64 - defaults: - run: - shell: bash -l {0} - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Install Conda - uses: mamba-org/setup-micromamba@d7c9bd84e824b79d2af72a2d4196c7f4300d3476 # v3.0.0 - with: - environment-name: anaconda-client-env - cache-environment: true - create-args: | - python=${{ matrix.python }} - - - name: Setup MSYS2 ${{matrix.sys}} - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 - if: matrix.os == 'windows-latest' - with: - msystem: ${{matrix.sys}} - release: false - install: >- - git - base-devel - msys2-devel - mingw-w64-${{matrix.env}}-zstd - mingw-w64-${{matrix.env}}-zlib - mingw-w64-${{matrix.env}}-toolchain - mingw-w64-${{matrix.env}}-cmake - mingw-w64-${{matrix.env}}-autotools - - - name: Cache SLiM build - if: matrix.os == 'windows-latest' - id: cache-slim - uses: actions/cache@2c8a9bd7457de244a408f35966fab2fb45fda9c8 # v6.0.0 - with: - path: D:\a\pyslim\pyslim\SLiM - key: ${{runner.os}}-${{matrix.sys}}-${{matrix.env}}-key - - - name: Build SLiM (Windows) - if: matrix.os == 'windows-latest' && steps.cache-slim.outputs.cache-hit != 'true' - shell: msys2 {0} - run: | - git clone https://github.com/messerlab/SLiM.git - mkdir -p SLiM/Release - cd SLiM/windows_compat/gnulib - touch --date="`date`" aclocal.m4 Makefile.am configure configure.ac config.h.in Makefile.in - cd ../.. - cd Release - cmake -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Release .. - make -j 2 - - - name: Install uv and dependencies - run: | - pip install uv - uv sync --locked --group test --no-default-groups - - - name: Install SLiM (macOS / Linux) - if: matrix.os == 'macos-latest' || matrix.os == 'ubuntu-24.04' - run: micromamba install slim -y - - - name: Run tests - run: | - export PATH=$PWD/SLiM/Release:$PATH - uv run --no-default-groups --group test pytest \ - -n 0 -v \ - --cov=pyslim --cov-branch \ - --cov-report=xml \ - tests - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - flags: python-tests - files: coverage.xml - disable_search: true - verbose: true diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml deleted file mode 100644 index 6bba338b..00000000 --- a/.github/workflows/wheels.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Publish Python release - -on: - push: - branches: [test-publish] - release: - types: [published] - -permissions: - contents: read - -jobs: - build: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Install uv - uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 - with: - version: "0.10.0" - enable-cache: false - - - name: Build - run: uv build - - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: dist - path: dist/ - - publish: - runs-on: ubuntu-24.04 - environment: release - needs: [build] - permissions: - id-token: write - steps: - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: dist - path: dist - - - name: Publish to TestPyPI - if: github.event_name == 'push' && github.ref_name == 'test-publish' - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 - with: - repository-url: https://test.pypi.org/legacy/ - verbose: true - - - name: Publish to PyPI - if: github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f61487d2..a31bb709 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,7 +2,52 @@ [1.1.2] - 2026-XX-XX ******************** -In development +**Breaking changes**: + +- The release of SLiM 6.0, changes to metadata (see below) mean that accessing + top-level metadata (e.g., `ts.metadata["SLiM"]`) more than a few times in + a script will take a long time. Scripts that previously ran quickly may take a + prohibitively long. See the documentation for simple changes that fix the problem: + https://tskit.dev/pyslim/docs/latest/previous_versions.html + +- The SLiM tree sequence file version number has changed to 1.0. Use `pyslim.update` + to convert your tree sequence file to this format. + +- Metadata for SLiM's mutations are no longer stored along with the tskit mutations, + because mutation stacking allows each tskit mutation to be associated with more + than one SLiM mutation. Now, metadata for each unique mutation is stored in + top-level metadata, under `ts.metadata["SLiM_mutation_list"]`. The recommended + way to access this information is by obtaining the SLiM ID-to-metadata dict + returned by `pyslim.mutation_metadata(ts)`. + +- Previously, `msprime.sim_mutations` with the `msprime.SLiMMutationModel` + would record SLiM metadata along with each new mutation. However, msprime + does not modify top-level metadata, and so the method `add_mutation_metadata` + should be used after adding SLiM mutations. + +- This is a SLiM change, but top-level metadata is now encoded using the `json+struct` + codec now provided by tskit (so that the mutation metadata is not too large/slow). + +- The top-level and individual metadata schemas now depend on the number of traits + in the model. The methods `slim_tree_sequence_metadata_schema` and + `slim_individual_metadata_schema` can be used to produce correct schema. + +**Bug fixes:** + +- In some previous versions, converting files produced by a yet-older version of SLiM + to the previously-current file version dropped some information from metadata: + nucleotide values for mutations, and pedigree parent IDs for individuals. This only + may have affected users using `pyslim.convert(ts)` in a previous version of pyslim + on a tree sequence `ts` with SLiM file version prior to 0.9. + +**New features**: + +- SLiM now includes in metadata information about the effects of mutations on + quantitative traits, the values of traits for individuals, and the values of + various "tags" defined in SLiM. + +- `default_slim_metadata` can now take additional arguments to modify the returned + values. ******************** [1.1.1] - 2026-03-06 diff --git a/docs/_toc.yml b/docs/_toc.yml index b2a83888..c47647d8 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -7,17 +7,4 @@ parts: - file: installation - caption: Using pyslim chapters: - - file: tutorial - - file: vignette_space - - file: vignette_continuing - - file: vignette_coalescent_diversity - - file: vignette_parallel_phylo - file: time_units - - file: metadata - - file: previous_versions -- caption: pyslim reference - chapters: - - file: python_api -- caption: Miscellaneous - chapters: - - file: development diff --git a/docs/development.md b/docs/development.md deleted file mode 100644 index 53b951a0..00000000 --- a/docs/development.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -(sec_development)= - -# Development - -All contributions, bug reports, documentation improvements and ideas are welcome. If you think -there is anything missing, please open an [issue](https://github.com/tskit-dev/pyslim/issues) -or [pull request](https://github.com/tskit-dev/pyslim/pulls) on GitHub. - -See the [tskit developer documentation](https://tskit.dev/tskit/docs/stable/development.html) -for the general development workflow (git, prek, testing, documentation). - -Install development dependencies with: - -```bash -uv sync -``` - -Run the tests with: - -```bash -uv run pytest -``` diff --git a/docs/metadata.md b/docs/metadata.md deleted file mode 100644 index e4255d51..00000000 --- a/docs/metadata.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{code-cell} -:tags: [remove-cell] -import pyslim, tskit, msprime -from IPython.display import SVG -import numpy as np -import random -random.seed(23) - -ts = tskit.load("example_sim.trees") -tables = ts.tables -``` - -```{eval-rst} -.. currentmodule:: pyslim -``` - - -(sec_metadata)= - -# Metadata - -(sec_metdata_overview)= - -## Overview - -SLiM puts SLiM-specific information into the *metadata* for the tree sequence, -as well as for each populations, individuals, nodes and mutations. -Here is a quick reference to what information is available: -see the SLiM manual for the more technical writeup. -A good way to get a generic metadata example is with {func}`.default_slim_metadata`. - -**Top-level:** -If `ts` is your tree sequence, then `ts.metadata` is a dict, -and `ts.metadata["SLiM"]` contains information about the simulation: - -- `file_version`: the version of the SLiM tree sequence file format -- `tick`: the value of `community.tick` within SLiM when the file was written out -- `cycle`: the value of `sim.cycle` within SLiM when the file was written out -- `model_type`: either `"WF"` or `"nonWF"` -- `nucleotide_based`: whether this is a nucleotide-based simulation -- `separate_sexes`: whether the simulation has separate sexes or not -- `spatial_dimensionality`: for instance, `""` or `"x"` or `"xy"` (etcetera) -- `spatial_periodicity`: whether space wraps around in some directions (same format as dimensionality) -- `stage`: the *stage* of the life cycle at which the file was written out (either `"first"`, `"early"`, or `"late"`) - -**Populations:** -Information about each SLiM-produced population is written to metatadata. -The format uses JSON and is extensible, so other keys may be present -and some keys may be missing (for instance, there are no spatial bounds -in a nonspatial simulation). The metadata may be `None` for populations -that SLiM did not use. The keys that SLiM uses are: - -- `slim_id`: the ID of this population in SLiM -- `name`: the name of the population (by default, `p0`, `p1`, etcetera) -- `description`: a string describing the population -- `selfing_fraction`, `female_cloning_fraction`, `male_cloning_fraction`, and `sex_ratio`: only present when applicable (e.g., in WF simulations) -- `bounds_x0`, `bounds_x1`, `bounds_y0`, `bounds_y1`, `bounds_z0`, and `bounds_z1`: the spatial bounds, when applicable -- `migration_records`: A *list* of entries decribing migration between populations in a WF model. - -**Individuals:** -Each individual produced by SLiM contains the following metadata: - -- `pedigree_id`: the "pedigree ID", unique within the SLiM simulation -- `pedigree_p1`, `pedigree_p2`: the pedigree IDs of the individuals' two - parents (they may be equal in the case of selfing, or `-1` to indicate no - parent, in the case of the initial generation or for cloning) -- `age`: the `.age` property within SLiM at the time the file was written out -- `subpopulation`: the subpopulation within SLiM the individual was in at the time the file was written out -- `sex`: the sex of the individual (either {data}`.INDIVIDUAL_TYPE_FEMALE`, {data}`.INDIVIDUAL_TYPE_MALE`, or {data}`.INDIVIDUAL_TYPE_HERMAPHRODITE`) -- `flags`: additional information; currently only recording whether the individual was a "migrant" or not (see the SLiM manual) - -**Nodes:** -Each "node" produced by SLiM (i.e., "genome" within SLiM) has: - -- 'slim_id': the unique ID associated with the genome by SLiM -- 'is_null': whether the genome is a "null" genome (in which case it isn't - really there, so shouldn't have any mutations or relationships in the tree - sequence!) -- 'genome_type': the 'type' of this genome (0 for autosome, 1 for X, 2 for Y) - -**Mutations:** -Each mutation's metadata is a dictionary with a single key, `"mutation_list"`, -whose entry is a *list* of metadata dictionaries corresponding to the mutations that are "stacked", -i.e., all present, in all genomes inheriting from this (tskit) mutation. -So, `ts.mutation(12).metadata["mutation_list"]` is a list, each of whose entries contains: - -- `mutation_type`: the numeric ID of the `MutationType` within SLiM -- `selection_coeff`: the selection coefficient -- `subpopulation`: the numeric ID of the subpopulation the mutation occurred in -- `slim_time`: the value of `community.tick` when the mutation occurred -- `nucleotide`: either `-1` if there is no associated nucleotide, or the numeric code for the nucleotide (see {data}`.NUCLEOTIDES`) - - -(sec_metadata_tools)= - -## Metadata tools - -The dictionaries describing the schema for these metadata entries -are available in `pyslim.slim_metadata_schemas`. -Furthermore, this method may be useful in working with metadata: - -```{eval-rst} -.. autofunction:: default_slim_metadata -``` - - -## Modifying SLiM metadata -For more on working with metadata, -see {ref}`tskit's metadata documentation `. - - -### Top-level metadata - -The entries of the top-level metadata dict are *read-only*. -So, you might think that -`tables.metadata["SLiM"]["model_type"] = "nonWF"` -would switch the model type, -but this in fact (silently) does nothing. To modify the top-level metadata, -we must (a) work with tables (as tree sequences are immutable, and (b) -extract the metadata dict, modify the dict, and copy it back in. -Instead, you should do -```{code-cell} -md = tables.metadata -md["SLiM"]["model_type"] = "nonWF" -tables.metadata = md -``` -Modifying the top-level metadata -could be used to set spatial bounds on an annotated msprime simulation, for instance. -(This is recorded in the population metadata.) - - -### Modifying SLiM metadata in tables - - -To modify the metadata that ``pyslim`` has introduced into -the tree sequence produced by a coalescent simulation, -or the metadata in a SLiM-produced tree sequence, -we need to edit the TableCollection that forms the editable data behind the tree sequence. -For instance, to set the ages of the individuals in the tree sequence to random numbers between 1 and 4, -we will extract a copy of the underlying tables, clear it, -and then iterate over the individuals in the tree sequence, -as we go re-inserting them into the tables -after replacing their metadata with a modified version: - -```{code-cell} -tables = ts.dump_tables() -tables.individuals.clear() -for ind in ts.individuals(): - md = ind.metadata - md["age"] = random.choice([1,2,3,4]) - _ = tables.individuals.append( - ind.replace(metadata=md) - ) - -mod_ts = tables.tree_sequence() - -# check that it worked: -print("First ten ages:", [mod_ts.individual(i).metadata["age"] for i in range(10)]) -for ind in mod_ts.individuals(): - assert ind.metadata['age'] in [1, 2, 3, 4] - -# save out the tree sequence -mod_ts.dump("modified_ts.trees") -``` - -## Technical details - -### Metadata entries - -SLiM records additional information in the metadata columns of Individual, Node, and Mutation tables, -in a binary format using the python ``struct`` module. -See {ref}`tskit's metadata documentation ` -for details on how this works. -Nothing besides this binary information can be stored in the metadata of these tables if the tree sequence is to be used by SLiM, -and so when ``pyslim`` annotates an existing tree sequence, anything in those columns is overwritten. -Population metadata is stored as JSON, however, which is more flexible. -For more detailed documentation on the contents and format of the metadata, see the SLiM manual. - -Of particular note is that *nodes* and *populations* may have empty metadata. -SLiM will not use the metadata of nodes that are not associated with alive individuals, -so this can safely be omitted (and makes recapitation easier). -And, populations not used by SLiM will have empty metadata. -All remaining metadata are required (besides edges and sites, whose metadata is not used at all). diff --git a/docs/previous_versions.md b/docs/previous_versions.md deleted file mode 100644 index 3c1f74da..00000000 --- a/docs/previous_versions.md +++ /dev/null @@ -1,258 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{code-cell} -:tags: [remove-cell] -import pyslim, tskit, msprime - -ts = tskit.load("example_sim.trees") -tables = ts.tables -``` - - -(sec_previous_versions)= - - -# Migrating from previous versions of pyslim - -## 1.1 - -Release 1.1 goes along with SLiM v5, which introduces multichromosome support. -See [](sec_overview_vacant_nodes) for a description of the possibility of "vacant" nodes. - -1. Most importantly, if your tree sequence contains vacant nodes, these must -be removed or (better) simply amended to be not marked as samples before certain -operations, including computing statistics or recapitation. -To do this, you might do -```{code-cell} -removed_vacant = pyslim.has_vacant_samples(ts) -if removed_vacant: - ts = pyslim.remove_vacant(ts) -``` -Note that this does not remove the vacant nodes from the tree sequence, it just -removes them from the *sample*, which will make them invisible to most operations. -However, if you use {func}`.recapitate` then this is unnecessary, because -**{func}`.recapitate` does this for you.** - -2. If you *have* removed vacant samples and you wish to reload the tree seqeuence -into SLiM, you'll have to reverse this, like -```{code-cell} -if removed_vacant: - ts = pyslim.restore_vacant(ts) -``` -Note that `remove_vacant` and `restore_vacant` are harmless on tree sequences -without vacant nodes; they're just wrapped in `if` statements to avoid the extra -overhead if not needed. - -3. Replace `node.metadata["is_null"]` with `node.metadata["is_vacant"][0] > 0`. -(Previously, `is_null` contained a boolean; now it contains a list of ints; -for a single-chromosome simulation this will be a single int that will be -either 0 (if vacant) or 1 (if not). - -4. Instead of checking `node.metadata["genome_type"]`, instead consult -`ts.metadata["SLiM"]["this_chromosome"]["type"]`. (It was previously redundant -to have a separate "genome type" entry for every node, anyhow.) - -## 1.0 - -The pyslim 1.0 release coincides with that of SLiM v4, -which introduced a number of changes to SLiM. -pyslim remains backwards compatible, in that pyslim 1.0 -will happily read tree sequences produced by previous versions of SLiM or pyslim, -and will convert them to the current version. -However, previous pyslim code may not work, due to two sets of changes: -(1) much of the functionality originally in pyslim has moved to tskit -(e.g., metadata processing), and (2) minor changes to terminology in SLiM v4 -("generation" is now "tick"). - -Converting previous code should be straightforward, as there are exact replacements. -The most important changes are to remove calls to `pyslim.load( )` or `SlimTreeSequence( )`, -and change "generation=" arguments to "tick=". - -In more detail, to upgrade code you should: - -1. Change `pyslim.load( )` to `tskit.load( )`. -2. Remove calls to `SlimTreeSequence( )`. They are not needed. -3. Change `generation` to `tick` in any arguments to functions, or in metadata. -4. Change `pyslim.annotate_defaults( )` to `pyslim.annotate( )`. - and `pyslim.annotate_defaults_tables( )` to `pyslim.annotate_tables( )`. -5. Change `pyslim.update_tables( )` to `pyslim.update( )`. - -Some methods of SlimTreeSequence are now methods of pyslim that take a tree sequence -as their first argument: - -6. Change `ts.recapitate(...)` to `pyslim.recapitate(ts, ...)`. -7. Change `ts.individuals_alive_at(t)` to `pyslim.individuals_alive_at(ts, t)`. -8. Change `ts.has_individual_parents()` to `pyslim.has_individual_parents(ts)`, - and `ts.individual_parents()` to `pyslim.individual_parents(ts)`. -9. Replace `ts.first_generation_individuals()` with - an appropriate call to `pyslim.individuals_alive_at( )`. -10. Change `ts.mutation_at(...)` to `pyslim.mutation_at(ts, ...)`. - and `ts.nucleotide_at(...)` to `pyslim.nucleotide_at(ts, ...)`. - -Several properties previously provided by SlimTreeSequence are now provided -by TreeSequence (e.g., `ts.individual_times`); so these need no change. -However, these were briefly available as pyslim methods, so would need changing: - -11. Change `pyslim.individual_times(ts)` to `ts.individuals_time`, - `pyslim.individual_populations(ts)` to `ts.individuals_population`, and - `pyslim.individual_locations(ts)` to `ts.individuals_location` - -The change from `pyslim.annotate_defaults( )` to `pyslim.annotate( )` -also entailed some small changes in behavior. Most notably, -since msprime.sim_ancestry() now simulates individuals -by default, annotation does not set up individuals: if you have a tree -sequence without individuals (e.g., produced by msprime.simulate()) then you -need to set up those individuals yourself. - -To update a tree sequence produced by an old version of SLiM to the current one, -use `pyslim.update( )`. (However, note that reading it in to SLiM and -writing it out again might be even easier.) - -Also see notes below for 0.700. - - -## 0.700 - -A number of features that were first introduced in pyslim have been made part of core -tskit functionality. For instance, reference sequence support was provided (although -loosely) inpyslim to support SLiM's nucleotide models, but is now part of a standard -tskit {class}`tskit.TreeSequence`. Similarly, metadata processing in tskit made -code to do this within pyslim obsolete; this "legacy metadata" code has been removed -and instructions for how to migrate your code are [below](sec_legacy_metadata). - -In fact, we are now at the (very good) place where we don't really need -the `pyslim.SlimTreeSequence` class any longer, -and it will soon be deprecated. -So, pyslim is migrating to be purely functional: instead of providing the SlimTreeSequence -class with specialized methods, all methods will be functions of TreeSequences, -that take in a tree sequence and return something -(a modified tree sequence or some summary of it). -Backwards compatibility will be maintained for some time, but we request that you -switch over sooner, as your code will be cleaner and faster. - -To migrate, you should: - - -1. Replace `ts.slim_generation` with `ts.metadata['SLiM']['generation']`, - and `ts.model_type` with `ts.metadata['SLiM']['model_type']`. -2. Replace `ts.reference_sequence` with `ts.reference_sequence.data`. -3. Replace calls to `ts.recapitate(...)` with `pyslim.recapitate(ts, ...)`, - and similarly with other SlimTreeSequence methods. - -If you encounter difficulties, please post an -[issue](https://github.com/tskit-dev/pyslim/issues) -or [discussion](https://github.com/tskit-dev/pyslim/discussions) on github. - - -(sec_legacy_metadata)= - -## Legacy metadata - -In previous versions of pyslim, -SLiM-specific metadata was provided as customized objects: -for instance, for a node ``n`` provided by a ``SlimTreeSequence``, -we'd have ``n.metadata`` as a ``NodeMetadata`` object, -with attributes ``n.metadata.slim_id`` and ``n.metadata.is_null`` and ``n.metadata.genome_type``. -However, with tskit 0.3, -the capacity to deal with structured metadata -was implemented in {ref}`tskit itself `, -and so pyslim shifted to using the tskit-native metadata tools. -As a result, parsed metadata is provided as a dictionary instead of an object, -so that now ``n.metadata`` would be a dict, -with entries ``n.metadata["slim_id"]`` and ``n.metadata["is_vacant"]`` -(previously, ``n.metadata["is_null"]`` and ``n.metadata["genome_type"]``). -Annotation should be done with tskit methods (e.g., ``packset_metadata``). - -.. note:: - - Until pyslim version 0.600, the old-style metadata was still available, - but this functionality has been removed. - -Here are more detailed notes on how to migrate a script from the legacy -metadata handling. If you run into issues, please ask (open a discussion on github). - -**1.** Use top-level metadata instead of ``slim_provenance``: -previously, information about the model type and the time counter (generation) -in SLiM was provided in the Provenances table, made available through -the ``ts.slim_provenance`` object. This is still available but deprecated, -and should be obtained from the *top-level* metadata object, ``ts.metadata["SLiM"]``. -So, in your scripts ``ts.slim_provenance.model_type`` should be replaced with -``ts.metadata["SLiM"]["model_type"]``, -and (although it's not deprecated), probably ``ts.slim_generation`` should -probably be replaced with -``ts.metadata["SLiM"]["generation"]``. - -**2.** Switch metadata objects to dicts: -if ``md`` is the ``metadata`` property of a population, individual, or node, -this means replacing ``md.X`` with ``md["X"]``. -The ``migration_records`` property of population metadata is similarly -a list of dicts rather than a list of objects, so instead of -``ts.population(1).metadata.migration_records[0].source_subpop`` -we would write -``ts.population(1).metadata["migration_records"][0]["source_subpop"]``. - -Mutations were previously a bit different - if ``mut`` is a mutation -(e.g., ``mut = ts.mutation(0)``) -then ``mut.metadata`` was previously a list of MutationMetadata objects. -Now, ``mut.metadata`` is a dict, with a single entry: -``mut.metadata["mutation_list"]`` is a list of dicts, each containing the information -that was previously in the MutationMetadata objects. -So, for instance, instead of ``mut.metadata[0].selection_coeff`` -we would write ``mut.metadata["mutation_list"][0]["selection_coeff"]``. - -**3.** The ``decode_X`` and ``encode_X`` methods are now deprecated, -as this is handled by tskit itself. -For instance, ``encode_node`` would take a NodeMetadata object -and produce the raw bytes necessary to encode it in a Node table, -and ``decode_node`` would do the inverse operation. -This is now handled by the relevant MetadataSchema object: -for nodes one can obtain this as ``nms = ts.tables.nodes.metadata_schema``, -which has the methods ``nms.validate_and_encode_row`` and ``nms.decode_row``. -Decoding is for the most part not necessary, -since the metadata is automatically decoded, -but ``pyslim.decode_node(raw_md)`` could be replaced by ``nms.decode_row(raw_md)``. -Encoding is necessary to modify tables, -and ``pyslim.encode_node(md)`` can be replaced by ``nms.validate_and_encode_row(md)`` -(where furthermore ``md`` should now be a dict rather than a NodeMetadata object). - -**4.** The ``annotate_X_metadata`` methods are deprecated, -as again tskit has tools to do this. -These methods would set the metadata column of a table - -for instance, if ``metadata`` is a list of NodeMetadata objects, then -``annotate_node_metadata(tables, metadata)`` would modify ``tables.nodes`` in place -to contain the (encoded) metadata in the list ``metadata``. -Now, this could be done as follows (where now ``metadata`` is a list of metadata dicts): - -```{code-cell} -metadata = [ {'slim_id': k, 'is_vacant': [0]} - for k in range(tables.nodes.num_rows) ] -nms = tables.nodes.metadata_schema -tables.nodes.packset_metadata( - [nms.validate_and_encode_row(r) for r in metadata] -) -``` - -If speed is an issue, then ``encode_row`` can be substituted for ``validate_and_encode_row``, -but at the risk of missing errors in metadata. - -**5.** the ``extract_X_metadata`` methods are not necessary, -since the metadata in the tables of a TableCollection are automatically decoded. -For instance, ``[ind.metadata["sex"] for ind in tables.individuals]`` will obtain -a list of sexes of the individuals in the IndividualTable. - -:::{warning} - It is our intention to remain backwards-compatible for a time. - However, the legacy code will disappear at some point in the future, - so please migrate over scripts you intend to rely on. -::: diff --git a/docs/python_api.md b/docs/python_api.md index b570cd69..51e96d23 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -18,7 +18,7 @@ from IPython.display import SVG import numpy as np ts = tskit.load("example_sim.trees") -tables = ts.tables +tables = ts.dump_tables() ``` ```{eval-rst} @@ -38,6 +38,7 @@ Here is a quick reference to some of the methods: .. autosummary:: recapitate + mutation_metadata annotate individuals_alive_at individual_ages @@ -47,6 +48,9 @@ Here is a quick reference to some of the methods: has_vacant_samples node_is_vacant slim_time + next_slim_mutation_id + add_mutation_metadata + add_mutation_metadata_tables convert_alleles generate_nucleotides population_size @@ -89,6 +93,11 @@ Here is a quick reference to some of the methods: .. autofunction:: set_slim_state ``` +```{eval-rst} +.. autofunction:: add_mutation_metadata +.. autofunction:: add_mutation_metadata_tables +``` + ## Summarizing tree sequences Additionally, ``pyslim`` contains the following methods: @@ -119,6 +128,10 @@ Additionally, ``pyslim`` contains the following methods: ## Utilities +```{eval-rst} +.. autofunction:: mutation_metadata +``` + ```{eval-rst} .. autofunction:: slim_time ``` @@ -131,24 +144,23 @@ Additionally, ``pyslim`` contains the following methods: .. autofunction:: has_vacant_samples ``` +```{eval-rst} +.. autofunction:: nodes_vacant +``` + ```{eval-rst} .. autofunction:: node_is_vacant ``` +```{eval-rst} +.. autofunction:: is_current_version +``` + ## Metadata -SLiM-specific metadata is made visible to the user by ``.metadata`` properties. -For instance: -```{code-cell} -ts.individual(4).metadata -``` -shows that the fifth individual in the tree sequence was given pedigree ID ``495999`` by SLiM, -had parents with pedigree IDs ``493739`` and ``494784``, -was age 10 at the time that they died (or the simulation ended), -lived in subpopulation 1, -was female (because ``sex`` matches ``pyslim.INDIVIDUAL_TYPE_FEMALE``, below), -and has no additional metadata flags. +SLiM-specific metadata is made visible to the user by ``.metadata`` properties, +described in [](sec_metadata). ### Annotation diff --git a/docs/rapid_adaptation.slim b/docs/rapid_adaptation.slim index ed2ed497..909eeecf 100644 --- a/docs/rapid_adaptation.slim +++ b/docs/rapid_adaptation.slim @@ -1,5 +1,5 @@ initialize() { - initializeTreeSeq(); + initializeTreeSeq(timeUnit="generations"); initializeMutationRate(1e-8); initializeMutationType("m1", 0.5, "e", 0.1); initializeGenomicElementType("g1", m1, 1.0); diff --git a/docs/selection.slim b/docs/selection.slim index 3c1860b4..b6173896 100644 --- a/docs/selection.slim +++ b/docs/selection.slim @@ -1,7 +1,7 @@ initialize() { initializeSLiMModelType("WF"); - initializeTreeSeq(); + initializeTreeSeq(timeUnit="generations"); initializeMutationRate(1e-6); initializeMutationType("m1", 0.5, "e", -0.1); initializeMutationType("m2", 0.5, "e", 0.5); diff --git a/docs/time_units.md b/docs/time_units.md index f997c49c..419651b1 100644 --- a/docs/time_units.md +++ b/docs/time_units.md @@ -226,7 +226,8 @@ so we expect generation time to go up at first. ```{code-cell} gts = tskit.load("generation_time.trees") -gentimes = gts.metadata["SLiM"]["user_metadata"]["generation_times"] +gts_metadata = gts.metadata +gentimes = gts_metadata["SLiM"]["user_metadata"]["generation_times"] fig, ax = plt.subplots(figsize=(12, 6), dpi=300) ax.set_xlabel("tick") @@ -266,7 +267,6 @@ Furthermore, since we already have mutations up until 100 time units ago, we need to put mutations on only previous to that time. ```{code-cell} -gentimes = gts.metadata["SLiM"]["user_metadata"]["generation_times"] gt = np.mean(gentimes[-50:]) recomb_rate = 1e-8 # per generation Ne = 1000 # generations diff --git a/docs/tutorial.md b/docs/tutorial.md deleted file mode 100644 index 35fc967e..00000000 --- a/docs/tutorial.md +++ /dev/null @@ -1,1334 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{code-cell} -:tags: [remove-cell] - -import warnings -import pyslim, tskit, msprime -from IPython.display import SVG -import numpy as np -import util - -np.random.seed(1234) -warnings.simplefilter('ignore', msprime.TimeUnitsMismatchWarning) -``` - -```{eval-rst} -.. currentmodule:: pyslim -``` - - -# Tutorial - -This tutorial covers the most common uses of tree sequences in SLiM/pyslim. - -## Recapitation, simplification, and mutation - -Perhaps the most common pyslim operations involve [](sec_tutorial_recapitation), -[](sec_tutorial_simplification), and/or [](sec_tutorial_adding_neutral_mutations). -Below we illustrate all three in the context of running a "hybrid" simulation, combining -both forwards and backwards (coalescent) methods. This hybrid approach is a popular -application of pyslim because coalescent algorithms, although more limited in the degree -of biological realism they can attain, can be much faster than the forwards algorithms -implemented in SLiM. - -A typical use-case is to take an existing SLiM simulation and endow -it with a history derived from a coalescent simulation: this is known as *recapitation*. -For instance, suppose we have a SLiM simulation of a population of 100,000 individuals -that we have run for 10,000 generations without neutral mutations. Now, we wish to -extract whole-genome genotype data for only 1,000 individuals. Here's one way to do it: - - -1. {func}`.recapitate` : - The simulation has likely not reached demographic equilibrium - it has not - *coalesced* entirely; recapitation uses coalescent simulation to provide - a "prior history" for the initial generation of the simulation. - -2. {meth}`simplify() ` : For efficiency, subset the tree - sequence to only the information relevant for those 1,000 individuals - we wish to sample. - -3. {func}`msprime.sim_mutations` : Add neutral mutations to the tree sequence. - -These steps are described below. First, to get something to work with, -you can run this simple SLiM script of a single population of sexual organisms, -fluctuating around 1000 individuals, for 1000 generations: - -```{literalinclude} example_sim.slim -``` - -You can run this in the shell, -setting the random seed so you get exactly the same results -as in the code below: -```{code-cell} -:tags: ["hide-output"] -%%bash -slim -s 23 example_sim.slim -``` - - -(sec_tutorial_recapitation)= - -### Recapitation - - -```{figure} _static/pedigree_recapitate.png ---- -scale: 42% -align: right -name: pedigree_recapitate ---- -Recapitation adds the green nodes by coalescent simulation. -(See [the introduction](sec_left_in_tree_sequence) -for a diagram of the previous state.) -``` - -Although we can initialize a SLiM simulation with the results of a coalescent simulation, -if during the simulation we don't actually use the genotypes for anything, it -can be much more efficient to do this afterwards, hence only doing a coalescent -simulation for the portions of the first-generation ancestors that have -not yet coalesced. (See the SLiM manual for more explanation.) -This is depicted in {numref}`figure {number} `: -imagine that at some sites, some of the samples -don't share a common ancestor within the SLiMulated portion of history (shown in blue). -Recapitation starts at the *top* of the genealogies, -and runs a coalescent simulation back through time -to fill out the rest of genealogical history relevant to the samples. -The green chromosomes are new ancestral nodes that have been added to the tree sequence. -This is important - if we did not do this, -then effectively we are assuming the initial population would be genetically homogeneous, -and so our simulation would have less genetic variation than it should have -(since the component of variation from the initial population would be omitted). - -Doing this is as simple as: - -```{code-cell} -orig_ts = tskit.load("example_sim.trees") -rts = pyslim.recapitate(orig_ts, - recombination_rate=1e-8, - ancestral_Ne=200, random_seed=5) -``` -The warning is harmless; it is reminding us to think about generation time -when recapitating a nonWF simulation (a topic we'll deal with later). - -We can check that this worked as expected, by verifying that after recapitation -all trees have only one root: - -```{code-cell} -orig_max_roots = max(t.num_roots for t in orig_ts.trees()) -recap_max_roots = max(t.num_roots for t in rts.trees()) -print(f"Maximum number of roots before recapitation: {orig_max_roots}\n" - f"After recapitation: {recap_max_roots}") -``` - -The {func}`.recapitate` method -is just a thin wrapper around {func}`msprime.sim_ancestry`, -and you need to set up demography explicitly - for instance, in the example above -we've simulated from an ancestral population of ``Ne=200`` diploids. -If you have more than one population, -you must set migration rates or else coalescence will never happen -(see [](sec_recapitate_with_migration) for an example, -and {func}`.recapitate` for more). - - -#### Recapitation with a nonuniform recombination map - -Above, we recapitated using a uniform genetic map. -But, msprime - like SLiM - can simulate with recombination drawn from an arbitrary genetic map. -Let's say we've already got a recombination map as specified by SLiM, -as a vector of "positions" and a vector of "rates". -msprime also needs vectors of positions and rates, but the format is slightly different. -To use the SLiM values for msprime, we need to do three things: - -1. Add a 0 at the beginning of the positions, -2. add 1 to the last position. - -The reason why msprime "positions" must start with 0 (step 1) is that in SLiM, -a position or "end" indicates the end of a recombination block such that its associated -"rate" applies to everything to the left of that end (see ``initializeRecombinationRate``). -In msprime, we will pass in a {class}`msprime.RateMap`, -which requires two things: - -- ``position``: A list of n+1 positions, starting at 0, and ending in the sequence length over which the RateMap will apply. -- ``rate``: A list of n positive rates that apply between each position. - -So, msprime needs a vector of positions that is 1 longer than what you give SLiM, -but one fewer rate values than positions. - -The reason for step 2 is that intervals for tskit (which msprime uses) -are "closed on the left and open on the right", -which means that the genomic interval from 0.0 to 100.0 includes 0.0 but does not include 100.0. -If SLiM has a final genomic position of 99, then it could have mutations occurring at position 99. -Such mutations would *not* be legal, on the other hand, if we set the tskit sequence length to 99, -since the position 99 would be outside of the interval from 0 to 99. -Said another way, if SLiM's final position is 99, the total sequence length is 100, -and so we need to set the end of the genome to 100. -The upshot is that we need to use SLiM's last position plus one - i.e., -the length of the genome - as the rightmost coordinate. - -For instance, suppose that we have a recombination map file in the following (tab-separated) format: - -```{literalinclude} _static/recomb_rates.tsv -``` - -This describes recombination rates across a 100Mb genome with higher rates on the ends -(for instance, 3.2 and 2.8 cM/Mb in the first and last 15Mb respectively) -and lower rates in the middle (0.25 cM/Mb between 50Mb and 85Mb). -The first column gives the starting position, in bp, -for the window whose recombination rate is given in the second column. -(*Note:* this is *not* a standard format for recombination maps - -it is more usual for the *starting* position to be listed!) - -Here is SLiM code to read this file and set the recombination rates: - -``` -lines = readFile("recomb_rates.tsv"); -header = strsplit(lines[0], "\t"); -if (header[0] != "end_position" - | header[1] != "rate(cM/Mb)") { - stop("Unexpected format!"); -} -rates = NULL; -ends = NULL; -nwindows = length(lines) - 1; -for (line in lines[1:nwindows]) { - components = strsplit(line, "\t"); - ends = c(ends, asInteger(components[0])); - rates = c(rates, asFloat(components[1])); -} -initializeRecombinationRate(rates * 1e-8, ends); -``` - -Now, here's code to take the same recombination map used in SLiM, -and use it for recapitation in msprime: - -```{code-cell} -positions = [] -rates = [] -with open('_static/recomb_rates.tsv', 'r') as file: - header = file.readline().strip().split("\t") - assert(header[0] == "end_position" and header[1] == "rate(cM/Mb)") - for line in file: - components = line.split("\t") - positions.append(float(components[0])) - rates.append(1e-8 * float(components[1])) - -# step 1 -positions.insert(0, 0) -# step 2 -positions[-1] += 1 -assert positions[-1] == orig_ts.sequence_length - -recomb_map = msprime.RateMap(position=positions, rate=rates) -rts = pyslim.recapitate(orig_ts, - recombination_rate=recomb_map, - ancestral_Ne=200, random_seed=7) -assert(max([t.num_roots for t in rts.trees()]) == 1) -``` -(As before, you should *not* usually explicitly set -the random seed in your scripts; we set it here so -the content of this document does not change.) - -:::{note} -Starting from msprime 1.0, the default model of recombination -in msprime is *discrete* - recombinations only occur at integer -locations - which matches SLiM's model of recombination. -::: - - -(sec_tutorial_simplification)= - -### Simplification - -```{figure} _static/pedigree_simplify.png ---- -scale: 42% -align: right -name: pedigree_simplify ---- -The result of simplifying the tree sequence -in figure {numref}`figure {number} ` -to only two of the three samples. -``` - -Probably, your simulations have produced many more fictitious genomes -than you will be lucky enough to have in real life, -so at some point you may want to reduce your dataset to a realistic sample size. -We can get rid of unneeded samples and any extra information from them by using -an operation called *simplification* (this is the same basic approach that SLiM -implements under the hood when outputting a tree sequence, as described in -[the introduction](sec_left_in_tree_sequence)). - -Depicted in the figure at the right is the result of applying an explicit call to -{meth}`tskit.TreeSequence.simplify` to our example tree sequence. -In the call we asked to keep only 4 -genomes (contained in 2 of the individuals in the current generation). This has -substantially simplified the tree sequence, because only information relevant to the -genealogies of the 4 sample nodes has been kept. (Precisely, simplification retains only -nodes of the tree sequence that are branching points of some marginal genealogy -- see -[Kelleher et al 2018](https://doi.org/10.1371/journal.pcbi.1006581) for details.) -While simplification sounds very appealing - it makes things simpler after all - -it is often not necessary in practice, because tree sequences are very compact, -and many operations with them are quite fast. -(It will, however, speed up many operations, so if you plan to do a large number of simulations, -your workflow could benefit from early simplification.) -So, you should probably not make simplification a standard step in your workflow, -only using it if necessary. - -It is important that simplification - if it happens at all - -either (a) comes after recapitation, or (b) is done with the -``keep_input_roots=True`` option (see {meth}`tskit.TreeSequence.simplify`). -This is because simplification removes some of the -ancestral genomes in the first generation, -which are necessary for recapitation, -unless it is asked to "keep the input roots". -If we simplify without this option before recapitating, -some of the first-generation blue chromosomes in the figure on the right -would not be present, so the coalescent simulation would start from a more recent point in time -than it really should. -As an extreme example, suppose our SLiM simulation has a single diploid who has reproduced -by clonal reproduction for 1,000 generations, -so that the final tree sequence is just two vertical lines of descent going back -to the two chromosomes in the initial individual alive 1,000 generations ago. -Recapitation would produce a shared history for these two chromosomes, -that would coalesce some time longer ago than 1,000 generations. -However, if we simplified first, then those two branches going back 1,000 generations would be removed, -since they don't convey any information about the shape of the tree; -and so recapitation might produce a common ancestor more recently than 1,000 generations, -which would be inconsistent with the SLiM simulation. - -After recapitation, -simplification to the history of 100 individuals alive today -can be done with the {meth}`tskit.TreeSequence.simplify` method: - -```{code-cell} -import numpy as np -rng = np.random.default_rng(seed=3) -alive_inds = pyslim.individuals_alive_at(rts, 0) -keep_indivs = rng.choice(alive_inds, 100, replace=False) -keep_nodes = [] -for i in keep_indivs: - keep_nodes.extend(rts.individual(i).nodes) - -sts = rts.simplify(keep_nodes, keep_input_roots=True) - -print(f"Before, there were {rts.num_samples} sample nodes (and {rts.num_individuals} individuals)\n" - f"in the tree sequence, and now there are {sts.num_samples} sample nodes\n" - f"(and {sts.num_individuals} individuals).") -``` - -**Note** that you must pass simplify a list of *node IDs*, not individual IDs. -Here, we used the {func}`.individuals_alive_at` method to obtain the list -of individuals alive today. -Also note that there are *still* more than 100 individuals remaining - 15 non-sample individuals -have not been simplified away, -because they have nodes that are required to describe the genealogies of the samples. -(Since this is a non-Wright-Fisher simulation, -parents and children can be both alive at the same time in the final generation.) - - - -(sec_tutorial_adding_neutral_mutations)= - -### Adding neutral mutations to a SLiM simulation - -```{figure} _static/pedigree_mutate.png ---- -scale: 42% -align: right -name: pedigree_mutate ---- -The tree sequence, with mutations added. -``` - -If you have recorded a tree sequence in SLiM, likely you have not included any neutral mutations, -since it is much more efficient to simply add these on afterwards. -To add these (in a completely equivalent way to having included them during the simulation), -you can use the {func}`msprime.sim_mutations` function, which returns a new tree sequence with additional mutations. -Continuing with the cartoons from above, these are added to each branch of the tree sequence -at the rate per unit time that you request. -We'll add these using the {class}`msprime.SLiMMutationModel`, so that the file can be read back into SLiM, -but any of the other mutation models in msprime could be used. -This works as follows: - -```{code-cell} -next_id = pyslim.next_slim_mutation_id(sts) -ts = msprime.sim_mutations( - sts, - rate=1e-8, - model=msprime.SLiMMutationModel(type=0, next_id=next_id), - keep=True, -) - -print(f"The tree sequence now has {ts.num_mutations} mutations,\n" - f"and mean pairwise nucleotide diversity is {ts.diversity():0.3e}.") -``` - - -What's going on here? Let's step through the code. - -1. The mutation ``rate = 1e-8``, which adds mutations at a rate of {math}`10^{-8}` per bp. - Unlike previous versions of msprime, this adds mutations using a discrete-sites model, - i.e., only at integer locations (like SLiM). - -2. We're passing ``type=0`` to the mutation model. - This is because SLiM mutations need a "mutation type", - and it makes the most sense if we add a type that was unused in the simulation. - In this example we don't have any existing mutation types, so we can safely use ``type=0``. - -3. We also add ``keep = True``, to keep any existing mutations. - In this example there aren't any, so this isn't strictly necessary, - but this is a good default. - -4. If there are existing SLiM mutations on the tree sequence we need to - make sure any newly added mutations have distinct SLiM IDs, - so we use {func}`.next_slim_mutation_id` to figure out - what the next available ID is, and pass it in. - - -(sec_output)= - -### Writing out genotypes to VCF - -Downstream applications often need input in VCF format, -which we can get with a call to {meth}`tskit.TreeSequence.write_vcf`. -However, if we do that with this tree sequence, we'll get a malformed VCF, -with empty strings in the REF column and a strange comma-separated list of integers -in the ALT column. The reason for this is because we added mutations -using the `SLiMMutationModel`, and has to do with how SLiM stores enough information -in the tree sequence to be able to load it back in. -So, to write out valid VCF with nucleotides for alleles, -we need to (1) if the SLiM simulation was not a nucleotide model, add nucleotides -to the SLiM mutations with {func}`generate_nucleotides`, -and (2) move those nucleotides over into the "ancestral state" -and "derived state" slots of the tree sequence with {func}`convert_alleles`. -If all your mutations in SLiM were nucleotide mutations, you only need to do (2). -And, beware that (2) is an irreversible step: if you write the tree sequence -produced by {func}`convert_alleles` to a file, you can't load that file into SLiM any more. -So, to do this we'll do: - -```{code-cell} -nts = pyslim.generate_nucleotides(ts) -nts = pyslim.convert_alleles(nts) -sample_indivs = np.unique([ts.node(n).individual for n in nts.samples()]) -with open("example_sim.vcf", "w") as vcffile: - nts.write_vcf(vcffile, individuals=sample_indivs[:5]) -``` - -Here we've just extracted genotypes for the first five individuals; -see below for what's going on in that code and what you probably -actually want to do; -see also {meth}`tskit.TreeSequence.write_vcf` for more options. - -For instance, if you want to use the SLiM pedigree IDs for the names in the VCF file, -we could do: - -```{code-cell} -pedigree_ids = [ - f"ind_{ts.individual(i).metadata['pedigree_id']}" for i in sample_indivs -] -with open("example_sim2.vcf", "w") as vcffile: - nts.write_vcf( - vcffile, - individuals=sample_indivs[:5], - individual_names=pedigree_ids[:5], - ) -``` - - -(sec_extracting_individuals)= - -## Extracting SLiM individuals - -Another important thing to be able to do is to extract -individuals from a simulation, -for analysis or for outputting their genotypes, for instance. -This section demonstrates some basic manipulations of individuals. - -### Extracting a sample of individuals - -The first, most common method to extract individuals is simply to get all -those that were alive at a particular time, -using {func}`.individuals_alive_at`. For instance, to get -the list of individual IDs of all those alive at the end of the -simulation (i.e., zero time units ago), we could do: - -```{code-cell} -orig_ts = tskit.load("example_sim.trees") -alive = pyslim.individuals_alive_at(orig_ts, 0) - -print(f"There are {len(alive)} individuals alive in the final generation.") -``` - -Here, ``alive`` is a vector of *individual* IDs, -so one way to take a sample of living individuals -and write their SNPs to a VCF is: - -```{code-cell} -rng = np.random.default_rng(seed=1) -keep_indivs = rng.choice(alive, 100, replace=False) -ts = msprime.sim_mutations(orig_ts, rate=1e-8, random_seed=1) -with open("example_snps.vcf", "w") as vcffile: - ts.write_vcf(vcffile, individuals=keep_indivs) -``` - -If you've done nothing else to the output from SLiM, -then this code will work, -but it does requires all alive individuals to be *samples*. -A situation in which this isn't the case is shown in the next section. - - -### Extracting individuals after simplification - -If the tree sequence has been simplified to retain only information -about a set of focal individuals, -then knowing an individual is alive at the end of the simulation -isn't enough to guarantee we have their entire genome sequence: -there are often individuals retained after simplification with -one or more non-sample nodes. -So, to output genotypes after simplification, we need to also check -that the individuals' nodes are also *samples*. -As mentioned earlier, {meth}`tskit.TreeSequence.simplify` takes a list -of nodes as input: - -```{code-cell} -keep_nodes = [] -for i in keep_indivs: - keep_nodes.extend(orig_ts.individual(i).nodes) -sts = rts.simplify(keep_nodes) -ts = msprime.sim_mutations(sts, rate=1e-8, random_seed=1) -``` -Individuals are retained by simplify if any of their nodes are, -so we would get an alive individual without sample nodes if, for instance, -a parent and two offspring are all alive, and we happen to keep the offspring -but not the parent. -For this reason, if at this point we try to extract genotypes for all of the -alive individuals, we encounter a (somewhat confusing) error: - -```{code-cell} -try: - alive = pyslim.individuals_alive_at(ts, 0) - with open("example_snps.vcf", "w") as vcffile: - ts.write_vcf(vcffile, individuals=alive) -except Exception as e: - print ("Error:") - print (e) -``` - -This is just telling us that some of the individuals we're trying -to write to the VCF have nodes that are not samples. -The reference to "missing" is a red herring: -see {ref}`tskit documentation ` -for what it's talking about. -So, instead of writing out genotypes of everyone alive, -we need to get the list of alive individuals *whose nodes are samples*, -using {meth}`is_sample() `: - -```{code-cell} -indivlist = [] -for i in pyslim.individuals_alive_at(ts, 0): - ind = ts.individual(i) - if ts.node(ind.nodes[0]).is_sample(): - indivlist.append(i) - # if one node is a sample, the other should be also: - assert ts.node(ind.nodes[1]).is_sample() -with open("example_snps.vcf", "w") as vcffile: - ts.write_vcf(vcffile, individuals=indivlist) -``` - - -### Extracting particular individuals - -Now let's see how to examine other attributes of individuals, -e.g., which subpopulation they're in. -To get another example with discrete subpopulations, -let's run another SLiM simulation, similar to the above -but with two populations exchanging migrants: - -```{literalinclude} migrants.slim -``` - -Let's run it: -```{code-cell} -:tags: ["hide-output"] -%%bash -slim -s 32 migrants.slim -``` - -To count up how many individuals are in each population, -we could do: - -```{code-cell} -orig_ts = tskit.load("migrants.trees") -alive = pyslim.individuals_alive_at(orig_ts, 0) -num_alive = [0 for _ in range(orig_ts.num_populations)] -for i in alive: - ind = orig_ts.individual(i) - ind_population = orig_ts.node(ind.nodes[0]).population - num_alive[ind_population] += 1 - -for pop, num in enumerate(num_alive): - print(f"Number of individuals in population {pop}: {num}") -``` - -:::{note} -Our SLiM script started numbering populations at 1, while tskit starts counting at 0, -so there is an empty "population 0" in a SLiM-produced tree sequence. -::: - - -(sec_recapitate_with_migration)= - -## Recapitation with migration between more than one population - -Following on the last example, -let's recapitate and mutate the tree sequence. -Recall that this recipe had two populations, ``p1`` and ``p2``, -each of size 1000. -Recapitation takes a bit more thought, because if the two populations stay separate, -it will run forever, unable to coalesce. -By default, {func}`.recapitate` *merges* the two populations into a single -one of size ``ancestral_Ne``. -But, if we'd like them to stay separate, we need to inclue migration between them. -Here's how we set up the demography using msprime's tools: - -```{code-cell} -demography = msprime.Demography.from_tree_sequence(orig_ts) -for pop in demography.populations: - # must set their effective population sizes - pop.initial_size = 1000 - -demography.add_migration_rate_change( - time=orig_ts.metadata['SLiM']['tick'], - rate=0.1, source="p1", dest="p2", -) -demography.add_migration_rate_change( - time=orig_ts.metadata['SLiM']['tick'], - rate=0.1, source="p2", dest="p1", -) -rts = pyslim.recapitate( - orig_ts, demography=demography, - recombination_rate=1e-8, - random_seed=4 -) -ts = msprime.sim_mutations( - rts, rate=1e-8, - model=msprime.SLiMMutationModel(type=0), - random_seed=7 -) -``` - -Again, there are *three* populations because SLiM starts counting at 1; -the first population is unused (no migrants can go to it). -Let's compute genetic diversity within and between each of the two populations -(we compute the mean density of pairwise nucleotide differences, -often denoted {math}`\pi` and {math}`d_{xy}`). -To do this, we need to extract the node IDs from the individuals of the two populations -that are alive at the end of the simulation. - -```{code-cell} -pop_nodes = [ts.samples(population=p, time=0) for p in range(ts.num_populations)] -diversity = ts.diversity(pop_nodes[1:]) -divergence = ts.divergence(pop_nodes[1:]) - -print(f"There are {ts.num_mutations} mutations across {ts.num_trees} distinct\n" - f"genealogical trees describing relationships among {ts.num_samples}\n" - f"sampled genomes, with a mean genetic diversity of {diversity[0]:0.3e}\n" - f"and {diversity[1]:0.3e} within the two populations,\n" - f"and a mean divergence of {divergence:0.3e} between them.") -``` - - -## Individual metadata - -Each ``Mutation``, ``Population``, ``Node``, and ``Individual``, as well as the tree -sequence as a whole, carries additional information stored by SLiM in its ``metadata`` -property. A fuller description of metadata in general is given in [](sec_metadata), -but as a quick introduction, here is the information available -about an individual in the previous example: - -```{code-cell} -:tags: ["remove-output"] -ind = ts.individual(0) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(ind) -``` - -Some information is generic to individuals in tree sequences of any format: -``id`` (the ID internal to the tree sequence), -``flags`` (described [below](sec_individual_flags)), -``location`` (the [x,y,z] coordinates of the individual), -``nodes`` (an array of the node IDs that represent the genomes of this individual), -and ``time`` (the time, in units of "time ago" that the individual was born). - -Other information, contained in the ``metadata`` field, is specific to tree sequences -produced by SLiM. This is described in more detail in the SLiM manual, but briefly: - -- the ``pedigree_id`` is SLiM's internal ID for the individual, -- ``age`` and ``subpopulation`` are their age and population at the time they - were recorded, or at the time - the simulation stopped if they were still alive (NB: SLiM uses the word - "subpopulation" for what is simply called a "population" in tree-sequence parlance) -- ``sex`` is their sex (as an integer, one of {data}`.INDIVIDUAL_TYPE_FEMALE`, - {data}`.INDIVIDUAL_TYPE_MALE`, or {data}`.INDIVIDUAL_TYPE_HERMAPHRODITE`). -- ``flags`` holds additional information about the individual recorded by SLiM - (currently, only whether the individual has migrated or not: - see [](sec_constants_and_flags)). - - -We can use this metadata in many ways, for example, to create an age distribution by sex: - -```{code-cell} -import numpy as np -max_age = max([ind.metadata["age"] for ind in ts.individuals()]) -age_table = np.zeros((max_age + 1, 2)) -age_labels = { pyslim.INDIVIDUAL_TYPE_FEMALE: 'females', - pyslim.INDIVIDUAL_TYPE_MALE: 'males' } -for i in pyslim.individuals_alive_at(ts, 0): - ind = ts.individual(i) - age_table[ind.metadata["age"], ind.metadata["sex"]] += 1 - -print(f"number\t{age_labels[0]}\t{age_labels[1]}") -for age, x in enumerate(age_table): - print(f"{age}\t{x[0]}\t{x[1]}") -``` - -We have looked up how to interpret the ``sex`` attribute -by using the values of {data}`.INDIVIDUAL_TYPE_FEMALE` (which is 0) -and {data}`.INDIVIDUAL_TYPE_MALE` (which is 1). -In a simulation without separate sexes, -all individuals would have sex equal to {data}`.INDIVIDUAL_TYPE_HERMAPHRODITE` -(which is -1). - -Several fields associated with individuals are also available as numpy arrays, -across all individuals at once: -{attr}`tskit.TreeSequence.individuals_location`, -{attr}`tskit.TreeSequence.individuals_population`, -{attr}`tskit.TreeSequence.individuals_time` (also see -{func}`.individual_ages` and {func}`.individual_ages_at`). -Using these can sometimes be easier than -iterating over individuals as above. For example, -suppose that we want to randomly sample 10 individuals alive and older than 2 time steps -from each of the populations at the end of the simulation, -and simplify the tree sequence to retain only those individuals. -This can be done using the numpy arrays returned by {func}`.individual_ages` -and `.individuals_population` as follows: - -```{code-cell} -alive = pyslim.individuals_alive_at(ts, 0) -adults = alive[pyslim.individual_ages(ts)[alive] > 2] -pops = [ - [i for i in adults if ts.individual(i).metadata['subpopulation'] == k] - for k in [1, 2] -] -sample_inds = [np.random.choice(pop, 10, replace=False) for pop in pops] -sample_nodes = [] -for samp in sample_inds: - for i in samp: - sample_nodes.extend(ts.individual(i).nodes) -sub_ts = ts.simplify(sample_nodes) -``` - -Note that here we have used the *subpopulation* attribute that SLiM places in metadata -to find out where each individual lives at the end of the simulation. -We might alternatively have used the *population* attribute of Nodes - -but, this would give each individual's *birth* location. - -The resulting tree sequence does indeed have fewer individuals and fewer trees: - -```{code-cell} -print(f"There are {sub_ts.num_mutations} mutations across {sub_ts.num_trees} distinct\n" - f"genealogical trees describing relationships among {sub_ts.num_samples} sampled genomes,\n" - f"with a mean overall genetic diversity of {sub_ts.diversity()}.") -``` - - -## Vacant nodes - -As discussed in [the Overview](sec_overview_vacant_nodes), -if not all individuals have two copies of the chromosome stored in the tree sequence, -then some nodes will be *vacant*, -which means they are merely a placeholder and don't represent actual genetic material. -The presence of these nodes can cause problems. -For instance, running an msprime simulation backwards from -a tree sequence with vacant sample nodes -(as in {numref}`figure {number} ` of the Overview) -would also simulate ancestry of the vacant nodes. -For this reason, {func}`.recapitate` removes these nodes -from the sample before running msprime, -which makes it so their ancestry will not be simulated. -Similarly, at present {ref}`statistics in tskit` -do not account for missing data, so will return incorrect results -if these vacant nodes are not removed from the sample. - -To be clear, the vacant nodes will still be present, -just not marked as samples (i.e., with the ``tskit.NODE_IS_SAMPLE`` -flag removed from their node flags). -Once they are not part of the sample, -they are essentially invisible to most operations. -However, it is helpful to know that they are there. -Why not remove them entirely, e.g., with ``simplify()``? -They are kept because if you wish to read the tree sequence back into SLiM -then you'll need them; -they can put them back in the sample after being removed -with {func}`.restore_vacant`. -If you would like to remove the vacant nodes from the sample for -other reasons, you can use {func}`.remove_vacant`. - - -## Historical individuals - -As we've seen, a basic tree sequence output by SLiM only contains the currently alive -individuals and the ancestral nodes (genomes) required to reconstruct their genetic -relationships. But you might want more than that. For example, there may be individuals -who are not alive any more, but whose complete ancestry you would like to know. Or -perhaps you'd like to know how the final generation relates to particular individuals in -the past. Or it may be that you want to access the spatial location of historical genomes -(which, for technical reasons is linked to individuals, not to genomes). The solution is -to *remember* an individual during the simulation, using the SLiM function -``treeSeqRememberIndividuals()``. Individuals can be Remembered in two ways, as -described below. - - - -```{figure} _static/pedigree_remember.png ---- -scale: 40% -align: right -name: pedigree_remember ---- -Individuals not alive in the last generation may still be present in the tree sequence -if they are either remembered permanently (purple), -or simply retained with ``permanent=F`` (dotted circle). -``` - - - -(sec_remembering_individuals)= - -### Permanently remembering individuals - -By default, a call to ``treeSeqRememberIndividuals()`` will permanently remember one or -more individuals, by marking their nodes as actual samples: the simulated equivalent of -ancient DNA dug out of permafrost, or stored -in an old collecting tube. This means any tree sequence subsequently recorded will always -contain this individual, its nodes (now marked as samples), and its full ancestry. As -with any other sample nodes, any permanently remembered individuals can be removed from -the tree sequence by [](sec_tutorial_simplification). The result of remembering an -individual in the [introductory example](sec_left_in_tree_sequence) is pictured on the right. - - -(sec_retaining_individuals)= - -### Retaining individuals - -Alternatively, you may want to avoid treating historical individuals and their genomes as -actual samples, but temporarily *retain* them as long as they are still relevant to -reconstructing the genetic ancestry of the sample nodes. This can save some computational -burden, as not only will nodes and individuals be removed once they are no longer -ancestral, but also the full ancestry of the retained individuals does not need to be -kept. You can retain individuals in this way by using -``treeSeqRememberIndividuals(..., permanent=F)``. - -Since a retained individual's nodes are not marked as samples, they are subject to the -[normal removal process](sec_left_in_tree_sequence), and it is possible to end up -with an individual containing only one genome, as shown in the diagram. However, as soon -as *both* nodes of a retained individual have been lost, the individual itself is deleted -too. - -Note that by default, nodes are only kept if they mark a coalescent point (MRCA or branch -point) in one or more of the trees in a tree sequence. This can be changed by -initialising tree sequence recording in SLiM using -``treeSeqInitialize(retainCoalescentOnly=F)``. SLiM will then -preserve all retained individuals while they remain in the genealogy, even if their nodes -are not coalescent points in a tree (so-called "unary nodes"). Similarly, if you later -decide to reduce the number of samples via [](sec_tutorial_simplification), -retained individuals will be kept only if they are still MRCAs in the ancestry of the -selected samples. To preserve them even if their nodes are not coalescent points, you -can specify ``ts.simplify(selected_samples, keep_unary_in_individuals=True)``. - -:::{todo} -Add SLiM code which includes retaining and remembering, and perhaps some python code -to show them. -::: - -(sec_remembering_everyone)= - - -### Remembering everyone - -Although not needed to reconstruct full genomic history, it is perfectly possible to -apply ``treeSeqRememberIndividuals()`` to every individual in every generation of a -simulation (i.e. everyone who has ever lived). If you simply mark everyone for temporary -retention, it should not increase the memory burden of your simulation much: most -individuals will be removed as the simulation progresses, since they will not contain -coalescent nodes. However, if you use ``treeSeqInitialize(retainCoalescentOnly=F)``, -the number of individuals in the resulting tree sequence is likely to become very large, -and the efficiencies provided by tree sequence recording will be substantially reduced. -Indeed in this case, retaining will be much the same as permanently remembering everyone -who has ever lived. Nevertheless, if you are willing to sacrifice enough computer memory, -either of these is (perhaps surprisingly) possible, even for medium-sized simulations. - - - -(sec_individual_flags)= - -### Individual flags - -We have seen that an individual can appear in the tree sequence because it was -Remembered, Retained, or alive at the end of the simulation (note these -are not mutually exclusive). The ``Individual.flags`` value stores this information. -For example, to count up the different individual types, we could do this: - -:::{todo} -Update this code with the simulation above so that we have some remembered and -retained individuals present -::: - -```{code-cell} -indiv_types = {"remembered" : 0, - "retained" : 0, - "alive" : 0} -for ind in ts.individuals(): - if ind.flags & pyslim.INDIVIDUAL_REMEMBERED: - indiv_types['remembered'] += 1 - if ind.flags & pyslim.INDIVIDUAL_RETAINED: - indiv_types['retained'] += 1 - if ind.flags & pyslim.INDIVIDUAL_ALIVE: - indiv_types['alive'] += 1 - -for k in indiv_types: - print(f"Number of individuals that are {k}: {indiv_types[k]}") -``` - -:::{note} -In previous versions of SLiM/pyslim, the first generation of individuals were -kept in the tree sequence, to allow [](sec_tutorial_recapitation). With the -addition of the ``keep_input_roots=True`` option to the -[](sec_tutorial_simplification) process, this is no longer necessary, -so these are no longer present, unless you specifically Remember them. -::: - - -## Generating intial diversity with msprime - -Suppose now that we'd like to *start* a SLiM simulation -with the result of a coalescent simulation. -For instance, we might want to do this instead of recapitating -if we wanted to use msprime to generate genetic diversity that -would then be selected on during the SLiM simulation. -To do this, we'll: -1. simulate a tree sequence with msprime, -2. add SLiM information to the nodes and individuals, -3. add SLiM mutations, and -4. write it out to a ``.trees`` file. - -First, we'll (1) run a simulation of 1 Mb of genome sampled in 200 diploids -in a population of 1000 diploids, -and (2) use the {func}`.annotate` function to add default SLiM metadata to the result: -```{code-cell} -demog = msprime.Demography() -demog.add_population(initial_size=1000) -ts = msprime.sim_ancestry( - samples=200, - demography=demog, - recombination_rate=1e-8, - sequence_length=1e6, - random_seed=5) -ts = pyslim.annotate(ts, model_type="nonWF", tick=1) -assert ts.num_individuals == 200 -assert ts.num_samples == 400 -``` -We have set ``tick`` to 1; -this means that as soon as we load the tree sequence into SLiM, -SLiM will set the current time counter to 1. -(If we set ``tick`` to 100, then any script blocks scheduled to happen before 100 -would not execute after loading the tree sequence.) - -We now have 200 diploids (so, 400 sampled nodes). -Here's individual 199, which hsa SLiM metadata: -```{code-cell} -:tags: ["remove-output"] -ind = ts.individual(199) -print(ind) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(ind) -``` -Looking at the ``metadata`` above, we see the default values are ``age=0`` -hermaphrodites (``sex=-1``), for instance. - -Now let's add SLiM mutations. -These will be neutral, as {func}`msprime.sim_mutations` -doesn't have the ability to dynamically modify the selection coefficients -stored in the mutation metadata. -To modify the mutations to be under selection, -see [](sec_vignette_coalescent_diversity). -```{code-cell} -ts = msprime.sim_mutations( - ts, rate=1e-8, - model=msprime.SLiMMutationModel(type=0), - random_seed=9 -) -``` -Now the mutations have SLiM metadata. -For instance, here's the first mutation: -```{code-cell} -:tags: ["remove-output"] -ts.mutation(0) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(ts.mutation(0)) -``` - -Finally, we write this out to a file that can be loaded in to SLiM: -```{code-cell} -ts.dump("initialize_nonWF.trees") -``` - -Here's a minimal SLiM script that reads in the tree sequence file -and runs it for a bit longer. - -```{literalinclude} neutral_restart.slim -``` - -```{code-cell} -:tags: ["hide-output"] -%%bash -slim -s 123 neutral_restart.slim -``` - -A more in-depth example is provided at [](sec_vignette_coalescent_diversity). -See the SLiM manual for more about this operation. - - -## Nucleotide-based models - -By default, {func}`.annotate` produces standard SLiM mutations, not "nucleotide-based" mutations. -To demonstrate how to further adjust the starting state of the simulation, -we'll further adjust the tree sequence `ts` from the previous section -to add in information about nucleotides. - -First, we need to set the ``nucleotide_based`` property in top-level metadata. -To do this, there are two possibly unfamiliar things: -first, we need to modify the underlying {class}`tskit.TableCollection` -(since tree sequences are immutable); -and second, we have to extract the metadata, modify it, and put it back in -(modifying it in-place will silently do nothing): - -```{code-cell} -tables = ts.dump_tables() -md = tables.metadata -md['SLiM']['nucleotide_based'] = True -tables.metadata = md -ts = tables.tree_sequence() -``` - -Next, we need to generate a reference sequence -and nucleotides for each mutation. -This is easy with {func}`.generate_nucleotides`: - -```{code-cell} -ts = pyslim.generate_nucleotides(ts) -ts.dump("initialize_nonWF_nuc.trees") -ts.reference_sequence.data[:20] -``` - -Now, mutations have a ``nucleotide`` property in metadata that is not ``-1``: - -```{code-cell} -:tags: ["remove-output"] -m = ts.mutation(0) -print(m) -``` - -```{code-cell} -:tags: ["remove-input"] -util.pp(m) -``` - -We can see which nucleotide is the derived state produced by each mutation - by indexing the {data}`.NUCLEOTIDES` object: - -```{code-cell} -for k in range(3): - m = ts.mutation(k) - print(f"Mutation {k}: position {ts.site(m.site).position}, time {m.time}") - for ml in m.metadata['mutation_list']: - print(f" nucleotide: {pyslim.NUCLEOTIDES[ml['nucleotide']]}") -``` - -Here's a script minimally modified from the above to be nucleotide-based: - -```{literalinclude} neutral_restart.slim -``` - -```{code-cell} -:tags: ["hide-output"] -%%bash -slim -s 123 neutral_nucleotide_restart.slim -``` - - -## Extracting information about selected mutations - -Here is a simple SLiM simulation with two types of mutation: -`m1` are deleterious, and `m2` are beneficial. -Let's see how to extract information about these mutations. - -```{literalinclude} selection.slim -``` -```{code-cell} -:tags: ["hide-output"] -%%bash -slim -s 23 selection.slim -``` - -First, let's see how many mutations there are: - -```{code-cell} -ts = tskit.load("selection.trees") -print(f"Number of sites: {ts.num_sites}\n" - f"Number of mutations: {ts.num_mutations}") -``` - -Note that there are more mutations than sites; -that's because some sites (looks like 24 of them) have multiple mutations. -The information about the mutation is put in the mutation's metadata. -Here's the first mutation: - -```{code-cell} -:tags: ["remove-output"] -m = ts.mutation(0) -print(m) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(m) -``` -Here, `m.site` tells us the ID of the *site* on the genome that the mutation occurred at, -and we can pull up information about that with the `ts.site( )` method: -```{code-cell} -:tags: ["remove-output"] -ts.site(m.site) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(ts.site(m.site)) -``` -This mutation occurred at position 54 along the genome (from `site.position`) -which previously had no mutations (since `site.ancestral_state` is the empty string, `''`) -and was given SLiM mutation ID 1653896 (`m.derived_state`). -The metadata (`m.metadata`, a dict) tells us that -the mutation has selection coefficient 1.5597 and occurred in population 1 in generation 827, -which was 172 generations ago. -This is not a nucleotide model, so the nucleotide entry is `-1`. -Note that `m.time` and `m.metadata['mutation_list'][0]['slim_time']` are in this case redundant: -they contain the same information, but the first is in tskit time -(i.e., number of steps before the tree sequence was written out) -and the second is using SLiM's internal "tick" counter. - -Also note that the mutation's metadata is a *list* of metadata entries. -That's because of SLiM's mutation stacking feature. -We know that some sites have more than one mutation, -so to get an example let's pull out one such mutation. -In this case, -`m.metadata['mutation_list']` is a list of length one, -so the mutation was not stacked on top of previous ones. - -Let's pull out a mutation that was stacked on top of another one: -```{code-cell} -:tags: ["remove-output"] -for m in ts.mutations(): - if m.parent != tskit.NULL: - break - -print(m) -print(ts.mutation(m.parent)) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(m) -util.pp(ts.mutation(m.parent)) -``` - -This mutation (which is `ts.mutation(1020)` in the tree sequence) -was the result of SLiM adding a new mutation of type `m1` and selection coefficient -0.0032 -on top of an existing mutation, also of type `m1` and with selection coefficient 0.3086. -This happened at generation 999 (i.e., at tskit time 0.0 time units ago), -and the older mutation occurred at generation 274 (at tskit time 725 time units ago). -The older mutation has SLiM mutation ID 547531, -and the newer mutation had SLiM mutation ID 1998096, -so the resulting "derived state" is `'1998096,547531'`. - -Now that we understand how SLiM mutations are stored in a tree sequence, -let's look at the allele frequencies. -The allele frequency spectrum for *all* mutations can be obtained using the -{meth}`tskit.TreeSequence.allele_frequency_spectrum` method, -shown here for a sample of size 10 to make the output easy to see: - -```{code-cell} -samps = np.random.choice(ts.samples(), 10, replace=False) -afs = ts.allele_frequency_spectrum([samps], span_normalise=False, polarised=True) -print(afs.astype('int')) -``` - -(The `span_normalise=False` argument gives us counts rather than a density per unit length.) -This shows us that there are 4169 alleles that are found among the tree sequence's samples -that are not present in any of our 10 samples, 96 that are present in just one, etcetera. -The surprisingly large number that are near 50% frequency are perhaps positively selected -and on their way to fixation: we can check if that's true next. -You may have noticed that the sum of the allele frequency spectrum is 5243, -which is not obviously related to the number of mutations (6044) *or* the number of sites (6020). -That's because each derived allele that is inherited by some but not all of the samples -in the tree sequence is counted in the polarised allele frequency spectrum: -Fixed mutations, or mutations that were entirely "overwritten" by subsequent mutations, -do not contribute. -Here's how we can check this: - -```{code-cell} -afs_total = 0 -for v in ts.variants(): - if len(set(v.genotypes)) > 1: - afs_total += len(set(v.genotypes) - set([0])) -print(afs_total) -``` - -At time of writing, we don't have a built-in ``allele_frequency`` method, -so we'll use the following snippet: - -```{code-cell} -def allele_counts(ts, sample_sets=None): - if sample_sets is None: - sample_sets = [ts.samples()] - def f(x): - return x - return ts.sample_count_stat(sample_sets, f, len(sample_sets), - span_normalise=False, windows='sites', - polarised=True, mode='site', strict=False) -``` - -This will return an array of counts, one for each site in the tree sequence, -giving the number of *all* nonancestral alleles at that site found in the sample set -(so, lumping together any of the various derived alleles we were looking at above). -Then, we'll separate out the counts in this array to get the derived frequency spectra -separately for sites with (a) only `m1` mutations, (b) only `m2` mutations, -and (c) both (for completeness, if there are any). -First, we need to know which site has which of these three mutation types (m1, m2, or both): - -```{code-cell} -mut_type = np.zeros(ts.num_sites) -for j, s in enumerate(ts.sites()): - mt = [] - for m in s.mutations: - for md in m.metadata["mutation_list"]: - mt.append(md["mutation_type"]) - if len(set(mt)) > 1: - mut_type[j] = 3 - else: - mut_type[j] = mt[0] -``` - -Now, we compute the frequency spectrum, and aggregate it -to produce the allele frequency spectrum separately by mutation type. -We'll use the function `np.bincount` to do this efficiently: - -```{code-cell} -freqs = allele_counts(ts, [samps]) -# convert the n x 1 array of floats to a vector of integers -freqs = freqs.flatten().astype(int) -mut_afs = np.zeros((len(samps)+1, 3), dtype='int64') -for k in range(3): - mut_afs[:, k] = np.bincount(freqs[mut_type == k+1], minlength=len(samps) + 1) - -print(mut_afs) -``` - -The first column gives the AFS among these 10 samples for the deleterious alleles, -the second for the beneficial mutations; -the third column for the seven sites that had both types of mutation. -Interestingly, there are similar numbers of both types of mutation at intermediate frequency: -perhaps because beneficial mutations are sweeping linked deleterious alleles along with them. -Many fewer benefical alleles are at low frequency: -3,666 deleterious alleles are not found in our sample of 10 genomes, -while only 486 beneficial alleles are. - -Finally, let's pull out information on the allele with the largest selection coefficient. - -```{code-cell} -:tags: ["remove-output"] -sel_coeffs = np.array([ - sum(md["selection_coeff"] for md in m.metadata["mutation_list"]) - for m in ts.mutations() -]) -which_max = np.argmax(sel_coeffs) -m = ts.mutation(which_max) -ts.site(m.site) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(ts.site(m.site)) -``` - -This allele had a whopping selection coefficient of 4.94 -and appeared about halfway through the simulation. -Let's find its frequency in the full population: - -```{code-cell} -full_freqs = allele_counts(ts) -print(f"The allele is found in {full_freqs[m.site][0]} copies\n" - f"out of {ts.num_nodes} genomes.") -``` - -The allele is above 50% in the population, so it is probably on its way to fixation. -Using its SLiM ID (which is shown in its derived state, ``1616148``), -we could reload the tree sequence into SLiM, -restart the simulation, and use its ID to track its subsequent progression. - - -## Possibly important technical notes - -Also known as "gotchas". - -1. If you use msprime to simulate a tree sequence, and then use that to initialize a SLiM simulation, - you have to specify the same sequence length in both: as in the examples above, - the ``sequence_length`` argument to {func}`msprime.sim_ancestry` should be equal to the SLiM sequence length - *plus 1.0* (e.g., if the base positions in SLiM are 0 to 99, then there are 100 bases in all, - so the sequence length should be 100). - -2. Make sure to distinguish *individuals* and *nodes*! - ``tskit`` "nodes" correspond to SLiM "genomes". - Individuals in SLiM are diploid, so normally, each has two nodes (but retained - individuals may have nodes removed by simplification: see below). - -3. As described above, the Individual table contains entries for - - 1. the currently alive individuals, - 2. any individuals that have been permanently remembered with - ``treeSeqRememberIndividuals()``, and - 3. any individuals that have been temporarily retained with - ``treeSeqRememberIndividuals(permanent=F)``. Importantly, the nodes in these - individuals are *not* marked as sample nodes, so they can be lost during - simplification. This means that a retained individual may only have one node (but - if both nodes are lost due to simplification, the individual is removed too, and - will not appear in the Individual table). - -4. SLiM requires that the two nodes corresponding to the haplosomes of each individual - are adjacent in the node table, and are sorted by haplosome ID. - SLiM always writes out tree sequences like this, but it is possible to make - tree sequences in python that are leval otherwise but don't satisfy this requirement. diff --git a/docs/vignette_coalescent_diversity.md b/docs/vignette_coalescent_diversity.md deleted file mode 100644 index 589d1bd1..00000000 --- a/docs/vignette_coalescent_diversity.md +++ /dev/null @@ -1,544 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{code-cell} -:tags: [remove-cell] - -import pyslim, tskit, msprime -from IPython.display import SVG -import numpy as np -import matplotlib -import matplotlib.pyplot as plt - -import util -``` - -```{eval-rst} -.. currentmodule:: pyslim -``` - -(sec_vignette_coalescent_diversity)= - -# Vignette: Starting with diversity generated by coalescent simulation - -This vignette shows how to simulate history with msprime, -add SLiM mutations to it and assign them selection coefficients, -then run a SLiM simulation using this as a starting point. - -Simulations of large populations with selection can be costly, -especially if we need to run a lengthy "burn-in" period to get the -genetic diversity for selection to act on. -Sometimes, the precise form of the burn-in is not important, -and so a *neutral* burn-in is acceptable - allowing us to use msprime. -For instance, suppose we'd like to simulate a lab experiment -in which we take high-diversity organisms from the wild and subject them -to selection for a few dozen generations. -Genetic diversity in the wild is certainly not neutral, but then again, -we don't quite know what it *does* look like, so a coalescent simulation -would be better than nothing. The key attribute of reality we'd like to -approximate is the joint distribution of allele frequencies and effect -sizes. If the alleles affect a trait under stabilizing selection, we'd -expect a negative correlation between the two. On the other hand, -if the trait we're selecting on in the lab is not under strong selection -in the wild, there might not be much of a relationship. -This is a simple example, to show how to do this: -the trait under selection is just fitness, -and there is no relationship between allele frequency and effect size. - -The steps will be: - -1. Run a coalescent simulation with msprime. -2. Add SLiM metadata to the nodes, individuals, and populations. -3. Add SLiM mutations with msprime, - and edit the mutation metadata to assign selection coefficients. -4. Run the SLiM portion of the simulation. -5. Do some descriptive analysis of the results of selection. -6. Add neutral mutations to the tree sequence. -7. Do some descriptive analysis of genetic diversity along the genome. - - -## Mutation and recombination maps - -In this model, -we'll also demonstrate how to modulate the mutation and recombination rate -along the genome. We'll do a simple example: a 9MB genome -with three equally-sized domains. -The first and last third of the genome will have high recombination -(5e-8 per generation per bp), -and the middle third will have low recombination -(0.5e-8 per generation per bp). -The mutation rate will be constant (3e-8 per generation per bp), -but a lower proportion of mutations in the middle region are under selection: -on the outside thirds, 1% of mutations are beneficial with a selection coefficient -drawn from an Exponential distribution, -and in the middle third, only 0.1% are (but with the same distribution of selection coefficients). -This implies that the mutation rate *of beneficial mutations* on the ends -is {math}`0.01 \times 30^{-8}` = 3e-10, and in the middle is 3e-11. -We'll simulate these first, and only add the neutral mutations after everything else, -at rates 2.97e-8 on the ends, and 2.997e-8 in the middle. - - -## The coalescent simulation - -First, we'll use msprime -to simulate the demographic history of 2,000 smallish chromosomes -({math}`N_e = 1,0000` diploids) -from a population of 10,000 diploids total. - -```{code-cell} -breaks = [0, 3000000, 6000000, 9000000] -recomb_map = msprime.RateMap( - position = breaks, - rate = [5e-8, 0.5e-8, 5e-8]) -demog_model = msprime.Demography() -demog_model.add_population(initial_size=10000) -ots = msprime.sim_ancestry( - samples=1000, - demography=demog_model, - random_seed=5, - recombination_rate=recomb_map) -``` - -## Annotate everyone - -At this point, we have genealogical information: individuals, -nodes (chromosomes), and relationships between them, -but no genetic diversity; no mutations. -First, we'll add SLiM metadata to all of these things, -a procedure we call "annotating". - -```{code-cell} -ots = pyslim.annotate(ots, model_type="WF", tick=1, stage="late") -``` - -This method adds default metadata to everything that needs it: -in this case, all individuals, all nodes that are part of alive individuals, -and all populations referenced by nodes. -These default values are returned by {func}`.default_slim_metadata` -(e.g., all individuals are hermaphrodite, all chromosomes are autosomal); -see {func}`.annotate` for more information. - -## Add SLiM mutations - -Next, we're going to use the {class}`msprime.SLiMMutationModel` to add mutations -to the tree sequence. These will carry SLiM metadata, but this metadata -will say that the mutations are neutral. So, we'll then need to modify their metadata -after the fact to have selection coefficients drawn from some distribution. -(Remember, our motivation here is that we are using msprime to obtain -a plausible level of functional standing genetic variation -on which selection can act: -imagine that the mutations were (nearly) neutral in the wild, -but then became subject to strong selection in the lab, for some reason.) -We'll want this to be as if we'd done it in a burn-in script in SLiM: - -``` - initialize() { - ... - initializeMutationType("m2", 0.5, "e", 0.04); - initializeMutationRate(3e-10); - } - - fitness(m2) { - return 1.0; - } -``` - -In other words, we'd like to pull selection coefficients from an exponential distribution -with mean 0.04 (but, of course, this is a coalescent simulation, so the -dynamics of the mutations up until this point have been neutral). -Note that the dominance coefficient is *not* stored in the tree sequence: -it gets set in the SLiM recipe -because it's a property of the mutation type, not of individual mutations, in SLiM. -Here's how to add SLiM mutations with msprime: - -```{code-cell} -mut_map = msprime.RateMap( - position=breaks, - rate=[0.03e-8, 0.003e-8, 0.03e-8]) -mut_model = msprime.SLiMMutationModel(type=2) -ots = msprime.sim_mutations( - ots, - rate=mut_map, - model=mut_model, - keep=True, - random_seed=12) -print(f"The tree sequence now has {ots.num_mutations} mutations, at " - f"{ots.num_sites} distinct sites.") -``` - -Note the ``type=2`` argument to {class}`msprime.SLiMMutationModel`: -this means the mutations will be of type "m2" in SLiM (and, so you must -initialize that mutation type in the recipe that loads this tree sequence in). - -Now, we'll assign selection coefficients. -Recall that to accomodate mutation stacking in SLiM, -a mutation metadata entry is in fact a *list* of metadata entries, -one for each of the SLiM mutations that are stacked at this position. -The SLiM IDs of these mutations are available (in the same order) -as a comma-separated list of integers in the derived state of the mutation. -So, in case some SLiM mutations appear in more than one mutation -in the tree sequence, we will build a map from SLiM ID to selection coefficient: -``mut_map[k]`` will give the selection coefficient of the SLiM mutation with -SLiM mutation ID ``k``. - -```{code-cell} -rng = np.random.default_rng(seed=1234) -tables = ots.tables -tables.mutations.clear() -mut_map = {} -for m in ots.mutations(): - md_list = m.metadata["mutation_list"] - slim_ids = m.derived_state.split(",") - assert len(slim_ids) == len(md_list) - for sid, md in zip(slim_ids, md_list): - if sid not in mut_map: - mut_map[sid] = rng.exponential(scale=0.04) - md["selection_coeff"] = mut_map[sid] - _ = tables.mutations.append( - m.replace(metadata={"mutation_list": md_list}) - ) - -# check we didn't mess anything up -assert tables.mutations.num_rows == ots.num_mutations -print(f"The selection coefficients range from {min(mut_map.values()):0.2e}") -print(f"to {max(mut_map.values()):0.2e}.") -``` - - -## Load into SLiM - -Before loading the tree sequence into SLiM, we should check the top-level metadata. -We can see this with ``tables.metadata``: - -```{code-cell} -:tags: ['remove-output'] -tables.metadata -``` -```{code-cell} -:tags: ['remove-input'] -util.pp(tables.metadata) -``` - -We should edit this to match our planned slimulation -- particularly the ``model_type`` (WF or nonWF) and the ``tick``. -The ``tick`` tells SLiM what value to set the tick counter to -once this tree sequence is loaded. In principle, it can be set to anything, -independently of the times in the tree sequence, -because the times in the tree sequence are measured in units of -"time before the end"; and the ``tick`` that gets -passed to SLiM sets what that "end time" is, in SLiM's time. -However, if you change this, the ``slim_time`` attributes in mutation metadata -will not be accurate. This is harmless, unless you do something with mutations' -times yourself. - -The ``model_type`` is already Wright-Fisher, but just to demonstrate how to -edit the metadata, let's make sure, -and then we'll write the tree sequence to a file. - -```{code-cell} -ts_metadata = tables.metadata -ts_metadata["SLiM"]["model_type"] = "WF" -tables.metadata = ts_metadata -ots = tables.tree_sequence() -ots.dump("vignette_annotated.init.trees") -``` - -Now for the SLiM recipe. -This simply continues selected mutations as before -(with mutation rate 1e-10 per bp per generation -and the same distribution of fitness effects). -The population size is determined by the number of individuals that were -read in from the tree sequence. -We need to make sure that the genome lengths match, -so we provide that as a constant ``L``, that will be provided at run time. -To facilitate later analysis, we'll also "Remember" the individuals -present at the *start* of the simulation, -so that they will remain in the tree sequence. - -```{literalinclude} reload_annotated.slim -``` - -Note that the simulation only has selected mutations (of type ``m2``), -but as we'll add in type ``m1`` mutations later, -we've declared them in the recipe as a placeholder. - -We could run this on the command line as -``slim -d L=100000000 reload_annotated.slim``, -but this time we'll stay within python, -and obtain the sequence length programatically: -```{code-cell} -import subprocess -msg = subprocess.check_output( - ["slim", "-d", f"L={int(ots.sequence_length - 1)}", - "-s", "5", "reload_annotated.slim"]) -print(msg.decode()) -``` -This runs quickly, since it's only 100 generations. - -## Analyze results - -First, let's look at what mutations are present. -```{code-cell} -ts = tskit.load("vignette_annotated.trees") -num_stacked = np.array([len(m.metadata["mutation_list"]) for m in ts.mutations()]) -init_time = ts.metadata['SLiM']['tick'] -old_mut = np.array([m.time > init_time - 1 - 1e-12 for m in ts.mutations()]) -assert sum(old_mut) == ots.num_mutations -print(f"There are {ts.num_mutations} present at {ts.num_sites} distinct sites.") -print(f"Of these, {np.sum(num_stacked > 1)} have more than one stacked mutation,") -print(f"and {np.sum(old_mut)} were produced by msprime.") -``` - -Most of the mutations were present as initial diversity, -but a few were added during the course of the simulation. -Along the way we did a consistency check, that the number of "old" mutations -matches the number of mutations we had in the tree sequence we loaded into SLiM. -Since we ran SLiM for 100 time steps, but loaded the tree sequence in during the ``late()`` -stage of time step 1, the "old" mutations are those -from at least 99 units of time ago -(and the 1e-12 is necessary for floating-point error). - -A simple thing to look at next is: how did the selected mutations -change in frequency? We can do this thanks to our having -Remembered the first generation. -First, we'll compute all allele frequencies -among both the first generation and the final generation: - -```{code-cell} -times = list(set(ts.individuals_time)) -times.sort() -print("The times ago at which individuals in the tree sequence were born:", times) -# The times ago at which individuals in the tree sequence were born: [0.0, 100.0] -nodes_by_time = [ts.samples(time=t) for t in times] - -num_nodes = np.array([len(x) for x in nodes_by_time]) -p = ts.sample_count_stat(nodes_by_time, lambda x: x/num_nodes, 2, windows='sites', - strict=False, span_normalise=False, polarised=True) -s = np.array([sum([sum([md["selection_coeff"] for md in m.metadata["mutation_list"]]) - for m in site.mutations]) for site in ts.sites()]) -``` - -To do this, we used the `time=t` argument to {meth}`tskit.TreeSequence.samples` -to find the nodes alive at each of the two times (0 and 100 generations ago); -then computed an array ``p`` of allele frequencies, with one row per site, -the first column giving the frequency among the initial generation, -and the second giving the frequency at the end. -We also pull out ``s``, the selection coefficients. -This last bit is a bit complex because each site can have more than one mutation, -and each tree sequence mutation can represent more than one SLiM mutation. -And, the way we've dealt with this is a bit of a hack, -so let's look at that site with multiple mutations: - -```{code-cell} -for j, v in enumerate(ts.variants()): - if len(v.site.mutations) > 1: - print(f"Site {j} has {num_stacked[j]} stacked mutations, " - f"with total derived allele frequency {p[j]} " - f"and sum of selection coefficients {s[j]}.") - print(f"The allele frequencies are:") - for k, a in enumerate(v.alleles): - print(f" '{a}': {sum(v.genotypes == k)}") - print(v.site) -``` - -There were two mutations at this site, both before the SLiM portion of the simulation -started. One happened on the background of the other, -and no genomes either today or in the initial generation carry the first allele in isolation. -Their effects combine in SLiM, so treating this as a single allele is correct. - -Now, we'll plot the initial and final allele frequencies, -with point size and color determined by the selection coefficient: - -```{code-cell} -fig, ax1 = plt.subplots(figsize=(5, 4)) -dp = ax1.scatter(p[:, 1], p[:,0], c=s, s=s*800, label='frequencies') -ax1.set_xlabel("initial allele frequency") -ax1.set_ylabel("final allele frequency") -fig.colorbar(dp, ax=ax1, label='selection coefficient'); -``` - - -Unsurprisingly, mutations that had a large change in allele frequency seem -to be biased towards ones with higher selection coefficients, -and those that were initially present at moderate frequency but were lost -are biased towards smaller selection coefficients. - - -## Add neutral mutations - -In real data, of course, we don't get to observe selection coefficients. -We haven't added in neutral mutations until this point for efficiency - -they are just bookkeeping, and do not affect the course of the simulation -in any way. For this reason, we can add them in after the fact, in a way -that is exactly equivalent to having kept track of them as we went along. - -Recall that out of an overall mutation rate of 3e-8, -we wanted 99% of the mutations to be neutral on the ends of the chromosome, -and 99.9% to be neutral in the middle. -So, we'll now add mutations at these rates, -using the same model of mutation as before. -The code is nearly the same as before, -with a few changes. -We've changed the ``type`` of the mutations -(so that neutral mutations will show up in SLiM as m1, -while selected mutations above were m2), -and we've asked these mutations to have SLiM mutation IDs -beginning at the ID where the previous mutations left off. -(This would be important were we to read this tree sequence -back in to SLiM; mutation IDs must be unique.) -And, importantly, we've added ``keep=True`` so that existing mutations -are not discarded. - -```{code-cell} -neutral_mut_map = msprime.RateMap( - position=breaks, - rate=[2.97e-8, 2.997e-8, 2.97e-8]) -next_id = pyslim.next_slim_mutation_id(ts) -neutral_mut_model = msprime.SLiMMutationModel( - type=1, - next_id=next_id) -mts = msprime.sim_mutations( - ts, - rate=neutral_mut_map, - model=neutral_mut_model, - keep=True, - random_seed=35) -print(f"The tree sequence now has {mts.num_mutations} mutations,") -print(f"at {mts.num_sites} distinct sites.") -``` - -We've now got a lot more mutations! -And, we've got a lot more sites with multiple mutations: - -```{code-cell} -num_alleles = np.array([len(s.mutations) for s in mts.sites()]) -for k in range(1, max(num_alleles)+1): - print(f"There are {sum(num_alleles == k)} sites with {k} distinct alleles.") -``` - -To get a nice a picture of what's happened, -we'll pull out a tree that had a lot of mutations on it, -and print a picture of it, with mutations labeled by their type: - -```{code-cell} -for t in mts.trees(): - mt = [max([u['mutation_type'] for u in m.metadata['mutation_list']]) for m in t.mutations()] - if t.num_mutations > 12: - break - -ml = {m.id: str(mtype) for mtype, m in zip(mt, t.mutations())} -SVG( - t.draw_svg(mutation_labels=ml, - node_labels={}, - size=(400, 300)) -) -``` - - -On this tree each mutation is marked by a red "x", and labeled with its mutation type: -either "1", for newly added mutations, or "2", for selected mutations present during the SLiM portion. -(Note: this is a large tree, with 68,211 nodes! -But as usual, the main structure is visible -because most nodes coalesce very recently.) - -OK, but how exactly is this working? -Can a neutral mutation be added to a site that previously had a selected mutation? -The short answer is: yes, and new alleles stack on top of -existing alleles, but existing alleles replace new alleles. -This is equivalent to including them as the simulation went along, -by the additivity property of Poisson mutations: -it turns out that the following two ways of generating mutations along the genome -are equivalent: either -(a) placing a random Poisson number with mean {math}`\mu`, -and randomly choosing each one to be non-neutral with probability 0.01, or -(b) placing random, independent Poisson numbers of neutral and non-neutral mutations -with means {math}`0.99\mu` and {math}`0.01\mu` respectively. -Since the neutral ones don't affect the simulation otherwise, -we can add them in afterwards. -Now, when the mutation algorithm in msprime puts down a new mutation -at a site with mutations already existing, -it appends the newly generated SLiM mutation ID to the previous derived state, -and adds the metadata for the new SLiM mutation to the list of metadata -from the previous mutation. -However, it doesn't modify any existing mutations, -so their derived states (and metadata) are unchanged. -The result is that, from the point of view of SLiM, -neutral ("m1") mutations "stack" on top of any other mutations (neutral or selected), -while selected ("m2") mutations stack with each other, but replace any neutral mutations. -This "stacking policy" is not actually exactly implementable in SLiM, -but given that our newly added mutations are meant to be entirely neutral, -seems like a reasonable policy. -If you wanted some other arrangement (e.g., to have m1 stack on top of m2), -you could go through and modify derived states and metadata appropriately. - -Let's check there are any sites with stacked mutations of different types in the simulation. -There is indeed one such site: - -```{code-cell} -:tags: ['remove-output'] -for site in mts.sites(): - if len(site.mutations) > 1: - types = [set([md["mutation_type"] for md in mut.metadata["mutation_list"]]) - for mut in site.mutations] - if max(map(len, types)) > 1: - print(site) -``` -```{code-cell} -:tags: ['remove-input'] -for site in mts.sites(): - if len(site.mutations) > 1: - types = [set([md["mutation_type"] for md in mut.metadata["mutation_list"]]) - for mut in site.mutations] - if max(map(len, types)) > 1: - util.pp(site) -``` - -Here, a neutral mutation has been put down on top of a selected mutation, -but stacked, so that any samples inheriting either of these mutations carries -the selected mutation. -For more discussion of how this works, see {class}`msprime.SLiMMutationModel`. - - -## Diversity along the genome - -Now that we've correctly added neutral mutations to the tree sequence, -and lengthily digested what exactly happened, -let's have a look at the result. -To do this, we'll compute two standard measures of genetic diversity -in windows along the genome: -nucleotide diverstiy (also called "Tajima's {math}`\pi`" or "mean density of pairwise differences"), -and Tajima's {math}`D` (with no known aliases). -This is easy to do thanks to the (statistics methods in tskit)[https://tskit.dev/tskit/docs/stable/stats.html]. - -```{code-cell} -windows = np.linspace(0, mts.sequence_length, 21) -pi = mts.diversity(mts.samples(), windows=windows) -taj_d = mts.Tajimas_D(mts.samples(), windows=windows) - - -fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(6,3), dpi=300) -mids = windows[1:] - np.diff(windows)/2 -ax1.set_xlabel("chromosome position (bp)") -ax1.set_ylabel("pairwise diversity") -ax1.plot(mids, pi, label="pairwise diversity") -ax2.set_xlabel("chromosome position (bp)") -ax2.set_ylabel("Tajima's D") -ax2.plot(mids, taj_d, label="Tajima's D"); -``` - - -The two statistics are very similar - perhaps unsurprisingly, because Tajima's D is calculated using -pairwise diversity, and we have a very large sample size (here, the entire population). -Tajima's D is negative across the entire genome, as the result of selection. -However, we don't see a strong difference between the three regions, -despite the stronger action of linked selection on the ends. diff --git a/docs/vignette_continuing.md b/docs/vignette_continuing.md deleted file mode 100644 index 3aff6616..00000000 --- a/docs/vignette_continuing.md +++ /dev/null @@ -1,245 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{code-cell} -:tags: [remove-cell] - -import pyslim, tskit, msprime -from IPython.display import SVG -import numpy as np -import util - -np.random.seed(1234) -``` - - -(sec_vignette_continuing)= - - -# Vignette: Following up with more coalescent simulation - -Previously, we saw how to use recapitation -to simulate the period *before* a SLiM simulation -with the coalescent simulator, msprime. -We can do the same thing *after* a period of SLiM simulation. -To demonstrate this, -below we'll run a simulation in which - -1. A population evolves neutrally for a long time, but then -2. it experiences strong positive selection on new mutations for 100 generations, and -3. evolves neutrally for another 1000 generations. - -To do this, we'll simulate step (2) first, with SLiM, -then recapitate to add step (1), and then "continue" the simulation using msprime to add in (3). - - -## Positive selection - -Here's a SLiM script that has rapid, strong selection acting genome-wide for 20 generations. -It is perhaps not very realistic, but it's dramatic. - -```{literalinclude} rapid_adaptation.slim -``` -```{code-cell} -%%bash -slim -s 5 rapid_adaptation.slim -``` - -We can see what happened in the GUI, -but let's pull some more statistics out of the tree sequence: -```{code-cell} -ts = tskit.load("rapid_adaptation.trees") - -# allele frequencies -p = ts.sample_count_stat( - [ts.samples()], lambda x: x/20000, 1, windows='sites', - span_normalise=False, polarised=True, strict=False) -print(f"There are {ts.num_sites} segregating sites, of which {np.sum(p > 0.25)}") -print(f"are at frequency above 25%, and {np.sum(p > 0.05)} are above 5%.") -``` - -The selection was, indeed, strong. - -## Recapitation - -Ok, now let's do phase (1), recapitating and mutating the result. -We'll add SLiM mutations with "mutation type" 0 -(so in SLiM these would be called type `m0`), -so first we check that all the existing mutations are of a different type. - -```{code-cell} -rts = pyslim.recapitate(ts, ancestral_Ne=1000, recombination_rate=1e-8, random_seed=6) - -# check type m0 is not used: -mut_types = set([md['mutation_type'] - for mut in ts.mutations() - for md in mut.metadata['mutation_list']]) -print(f"Keeping {rts.num_mutations} existing mutations of type(s) {mut_types}.") -assert 0 not in mut_types - -# add type m0 mutations -next_id = pyslim.next_slim_mutation_id(rts) -rts = msprime.sim_mutations( - rts, rate=1e-8, random_seed=7, keep=True, - model=msprime.SLiMMutationModel(type=0, next_id=next_id) -) - -p = rts.sample_count_stat( - [rts.samples()], lambda x: x/20000, 1, windows='sites', - span_normalise=False, polarised=True, strict=False) -print(f"After mutation, there are {rts.num_sites} segregating sites, of which {np.sum(p > 0.25)}") -print(f"are at frequency above 25%, and {np.sum(p > 0.05)} are above 5%.") -``` - -Now, there are more segregating sites - neutral ones. - - -## Continuing the simulation - -To "continue" the simulation neutrally, we'll - -1. simulate the desired period of time in msprime -2. randomly match the initial ancestors in the msprime simulation - with the final individuals of the SLim simulation, and -3. merge the two together, using the {meth}`tskit.TreeSequence.union` method. - - -**(1)** Simulating for a given period of time in msprime requires the ``end_time`` argument -(remembering that this is *time ago*); -we'll do this to simulate an additional 1000 generations. - -This is almost what we need, but there is one more detail: -if complete coalescence occurs on any region of the genome, -msprime will stop simulating the history of that region. -This is a problem, since we need all lineages to extend back to ``end_time``. -To make sure all lineages trace back to ``end_time``, -we'll add one "fake" sample from a separate population, that *can't* coalesce with the rest, -then remove it before the next step, using the ``keep_input_roots=True`` argument to ``simplify()``. - - -```{code-cell} -new_time = 1000 -demog_model = msprime.Demography() -demog_model.add_population(initial_size=10000, name='real') -demog_model.add_population(initial_size=10000, name='fake') -new_ts = msprime.sim_ancestry( - samples={'real' : 10000, 'fake' : 1}, - demography=demog_model, - end_time=new_time, - sequence_length=rts.sequence_length, - recombination_rate=1e-8, - random_seed=9) -new_ts = msprime.sim_mutations( - new_ts, rate=1e-8, random_seed=10, keep=True, - model=msprime.SLiMMutationModel(type=0) - ) -new_tables = new_ts.tables -# check that the spurious samples are 20000 and 20001 -for n in (20000, 20001): - assert n in new_ts.samples() - assert new_ts.node(n).population == 1 -new_tables.simplify(samples=np.arange(20000), keep_input_roots=True) -print(f"Remaining number of populations: {new_tables.populations.num_rows}") -``` - -**(2)** Now we'll pull out the IDs of the nodes from 1000 generations ago, -shift the times in the SLiM tree sequence back 1000 generations, -randomly assign each to a node at the end of the SLiM simulation, -and merge them. - -```{code-cell} - -new_nodes = np.where(new_tables.nodes.time == new_time)[0] -print(f"There are {len(new_nodes)} nodes from the start of the new simulation.") -# There are 4425 nodes from the start of the new simulation. - -slim_nodes = rts.samples(time=0) -assert(len(slim_nodes) == 20000) - -# randomly give new_nodes IDs in rts -node_map = np.repeat(tskit.NULL, new_tables.nodes.num_rows) -node_map[new_nodes] = np.random.choice(slim_nodes, len(new_nodes), replace=False) - -# shift times: in nodes and mutations -# since tree sequences are not mutable, we do this in the tables directly -# also, unmark the nodes at the end of the SLiM simulation as samples -tables = rts.tables -tables.nodes.flags = tables.nodes.flags & ~np.uint32(tskit.NODE_IS_SAMPLE) -tables.nodes.time = tables.nodes.time + new_time -tables.mutations.time = tables.mutations.time + new_time - -# merge the two sets of tables -tables.union(new_tables, node_map, - add_populations=False, - check_shared_equality=False) - -# get back the tree sequence -full_ts = tables.tree_sequence() - -p = full_ts.sample_count_stat( - [full_ts.samples()], lambda x: x/20000, 1, - windows='sites', span_normalise=False, - polarised=True, strict=False) -print(f"There are {full_ts.num_sites} segregating sites, of which {np.sum(p > 0.25)}") -print(f"are at frequency above 25%, and {np.sum(p > 0.05)} are above 5%.") -``` - -Well, allele frequencies have drifted. -Don't worry, we'll explain what happened there in a minute. - -Let's do a consistency check. -First, here's the root of the first tree in the recapitated SLiM simulation: -```{code-cell} -:tags: ["remove-output"] -t = rts.first() -assert(t.num_roots == 1) -r = rts.node(t.root) -print(r) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(r) -``` -Now, here's the root of the first tree *after* continuing -for 1000 generations, which should be the same: -```{code-cell} -:tags: ["remove-output"] -ft = full_ts.first() -assert(ft.num_roots == 1) -fr = full_ts.node(ft.root) -print(fr) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(fr) -``` -That matches up - the time of what should be the same node in the "continued" tree sequence -is 1000 generations earlier. - -So, what happened with ``union`` back there? -Well, the basic usage is ``tables.union(other, node_map)``, -where ``node_map`` is an array of length equal to the number of nodes in ``other``, -whose entries are either ``tskit.NULL`` or the ID of a node in ``tables``. -The entries that *aren't* NULL indicate that -``union`` should glue together ``tables`` and ``other`` by saying that that pair of nodes are the same. -(So, e.g., if ``node_map[3]`` is equal to ``25``, then it says that node 25 in ``tables`` -is actually the same, really, as node 3 in ``other``.) -We then asked ``union`` to please not create new populations, -since otherwise it would have assigned all the new nodes to a new population. -We also asked it to not "check for overlap equality": -sometimes, when unioning together two tree sequences, -we really expect everything having to do with the set of nodes we're saying are identical -to be identical in the two tree sequences, so ``union`` by default throws an error if it's not. -We don't expect that in this case, because, for instance, -there could be a mutation above one of the terminal nodes in the SLiM tree sequence; -this would clearly not be present in the new tree sequence. diff --git a/docs/vignette_parallel_phylo.md b/docs/vignette_parallel_phylo.md deleted file mode 100644 index efa70f2f..00000000 --- a/docs/vignette_parallel_phylo.md +++ /dev/null @@ -1,344 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 ---- - -```{code-cell} -:tags: [remove-cell] - -import pyslim, tskit, msprime -from IPython.display import SVG -import numpy as np -import pandas as pd -import os -``` - - -(sec_vignette_parallel)= - - -# Vignette: Parallelizing SLiM simulations in a phylogenetic tree - -Imagine you want to simulate the evolutionary history of a group. If there is -no migration between any of the branches in your tree, any branches stemming -from the same node can be simulated in parallel (see {numref}`phylo`). - -```{figure} _static/phylo.png -:height: 200px -:name: phylo - -Example of phylogeny we might want to simulate. Note how branches with the same color can be simulated in parallel when there is no migration. -``` - -To do this, we'll need to do two things: -(1) be able to *simulate* branches in parallel, and -(2) glue the resulting simulations (one per tip) back together. - - -## Simulating the branches - -First, we need to write a SLiM script that will be used for simulating the -history of each branch in our phylogeny. -We will perform a simple simulation, in which each branch can have a different -(but fixed) population size and length (number of ticks). -Also, we will allow deleterious mutations to happen across the entire chromosome -at a fixed rate. - -Here is a SLiM script that would do this: - -```{literalinclude} phylo_bgs.slim -``` - -For each branch, the presence or absence of ``infile`` tells SLiM -whether we want to start it from a previous branch or not. -If so, SLiM will read the previous tree sequence and change the -population size accordingly. -Note that when you read a tree sequence into SLiM, the tick counter will -be updated with the time encoded in the tree sequence, so we need to set the end -of the simulation as the length of the branch (`num_gens`) plus the current -"time" at the end of the loaded tree sequence. -At the end of the simulation, we call `sim.treeSeqRememberIndividuals` right -before saving the resulting tree sequence. This is necessary because we need to -ensure the individuals in the final generation are never dropped from the tree -sequence in future runs of SLiM which are started from the output of the -simulation, as they will later be used to glue the tree sequences together. - -I encoded the phylogeny we will simulate in a simple table, -which we'll use as ``df`` in the code below: - -```{code-cell} -:tags: ["hide-input"] -df = pd.read_csv("_static/phylo.tsv", sep="\t") -df = df.fillna('') -df["infile"] = df.parent + ".trees" -df["outfile"] = df.child + ".trees" -df.loc[df["infile"]==".trees", "infile"] = "" -df["is_leaf"] = ~df.child.isin(df.parent) -df -``` - -With our phylogeny and the simulation parameters, we are ready to run our -simulations. -One way to parallelize the simulation of sister branches is to use `make`. -You do not need to know much about this tool (though it is totally worthwhile -to check it out). -The main idea here is that you can specify dependency between files and `make` -works its magic to run the simulations in the right order. -Here is python code that will write out a makefile from the information in ``df``: - -```{code-cell} -f = open("sims.make", "w") -print(f"all: {' '.join(df.outfile.to_list())}\n", file=f) -for i, row in df.iterrows(): - print(f"{row.outfile}: {row.infile} phylo_bgs.slim", file=f) - print(f"\tslim -d \"infile='{row.infile}'\" -d popsize={row.popsize} " - f"-d \"popname=\'{row.child}\'\" " - f"-d num_gens={row.edgelen} " f"-d \"outfile='{row.child}.trees'\" " - "phylo_bgs.slim\n", - file=f) -f.close() -``` - -Here's the result. Again, don't worry about the details, -but you can see that the file encodes the phylogeny -through a bunch of ``child : parent`` "rules": -```{code-cell} -:tags: ["hide-input"] -%%bash -cat sims.make -``` - -With the makefile in hand, -we can now run make, specifying the maximum number of simulations -to be run simultaneously the ``-j``. -(Click on the "+" icon to see SLiM's output.) -```{code-cell} -:tags: ["hide-output"] -%%bash -make -f sims.make -j 3 -``` - - -```{dropdown} Click here for how to use python instead of make - -You would have to write a recursion over the branches in your tree (starting -from the root) and then parallelize the runs of sister branches somehow. - -```python -def phylo_recursion(parent, df): - print(parent) - childs = df[df.parent==parent] - print(childs) - if len(childs) == 0: - return - # you could parallelize this loop over childs with same parent - for i, row in childs.iterrows(): - if not os.path.exists(row.outfile): - os.system(f"slim -d \"infile='{row.infile}'\" -d popsize={row.popsize} -d num_gens={row.edgelen} -d \"outfile='{row.child}.trees'\" phylo_bgs.slim") - phylo_recursion(row.child, df) - -phylo_recursion("", df) -``` - -## Putting it all together: unioning the tree sequences - -With the tree sequences in hand, we now need to glue them together. -This can be done using -[**union**](https://tskit.dev/tskit/docs/stable/python-api.html#tskit.TreeSequence.union) -from tskit. -For two tree sequences which share some of its past history is shared, **union** -works by copying the non-shared parts of one of the tree sequence onto the other. -The trickiest part of this operation is defining the parts that are equivalent -in the two tree sequences. For that, you will have to create an array that serves -as a map of node IDs between the two tree sequences. - -Here is a function that will construct a map of the node IDs of two SLiM tree sequences -that correspond to the same chromosomes in SLiM -at any time older than the given time ago at which the two populations split. -Given two tree sequences ``other`` and ``ts``, -the goal here is to find, -for each node born before ``split_time`` ago in ``other``, -the matching node in ``ts``, where we can identify matching using the SLiM ID in metadata. -The code could be made easier to read by iterating over nodes, -but the following numpy-based version is much faster: - -```{code-cell} -def match_nodes(other, ts, split_time): - """ - Given SLiM tree sequences `other` and `ts`, builds a numpy array with length - `other.num_nodes` in which the indexes represent the node id in `other` and the - entries represent the equivalent node id in `ts`. If a node in `other` has no - equivalent in `ts`, then the entry takes the value `tskit.NULL` (-1). The - matching is done by comparing the IDs assigned by SLiM which are kept in - node metadata. This matching of SLiM IDs is *only* done for nodes with time - older than the specified `split_time`. - """ - node_mapping = np.full(other.num_nodes, tskit.NULL) - sids0 = np.array([n.metadata["slim_id"] for n in ts.nodes()]) - sids1 = np.array([n.metadata["slim_id"] for n in other.nodes()]) - alive_before_split1 = (other.tables.nodes.time >= split_time) - is_1in0 = np.isin(sids1, sids0) - both = np.logical_and(alive_before_split1, is_1in0) - sorted_ids0 = np.argsort(sids0) - matches = np.searchsorted( - sids0, - sids1[both], - side='left', - sorter=sorted_ids0 - ) - node_mapping[both] = sorted_ids0[matches] - return node_mapping -``` - -Now we are finally ready to **union** our tree sequences. For that, I wrote a -recursive function that goes through our data frame with the phylogeny and -returns a dictionary with the merged tree sequences from the tip to the root. - -```{code-cell} -merged = { - row.child : { - "ts": tskit.load(row.outfile), - "depth": row.edgelen, - "children": [row.child] - } - for i, row in df[df.is_leaf].iterrows() -} - -def union_children(parent, df, merged): - print(f"Going in: {parent}") - child_rows = df[df.parent == parent] - assert (len(child_rows) == 2) or (len(childs) == 0) - if len(child_rows) == 2: - children = [row.child for _, row in child_rows.iterrows()] - for child in children: - if child not in merged: - union_children(child, df, merged) - split_time = merged[children[0]]["depth"] - assert split_time == merged[children[1]]["depth"] # ultrametric - print(f'Unioning: {children}, Split time: {split_time}') - ts0 = merged[children[0]]["ts"] - ts1 = merged[children[1]]["ts"] - node_map = match_nodes(ts1, ts0, split_time) - tsu = ts0.union(ts1, node_map, check_shared_equality=True) - # the time from tip to start of simulation is split_time plus the - # length of the edge - parent_edgelength = df[df.child==parent].edgelen.item() - merged[parent] = { - "ts": tsu, - "depth": split_time + parent_edgelength, - "children": merged[children[0]]["children"] + merged[children[1]]["children"] - } - -union_children("root", df, merged) -# union of all three species tree sequences is in the root. -tsu = merged["root"]["ts"] -pops = merged["root"]["children"] -``` - -A slightly tricky thing we had to do there was to make sure we kept track of -which population in the union'ed tree sequence corresponds to -which population in our phylogeny. -Happily, we've stored each population's name in its metadata field, -so it's easy to match populations in the tree sequence up to what they're supposed to be. - -Let's make sure we have the right number of present-day samples -in each of the populations. To do this we need to make sure to get -"alive" samples, because recall that we have saved the state of the -population at each species split time. - -```{code-cell} -alive = np.where(np.isclose(tsu.tables.nodes.time, 0))[0] -pop_ids = {} -for pop in tsu.populations(): - if pop.metadata is not None: - pop_ids[pop.metadata['name']] = pop.id - -for name in pops: - pop_samples = tsu.samples(pop_ids[name]) - n_samples = sum(np.isin(pop_samples, alive)) // 2 - print(f"Union-ed tree sequence has {n_samples} samples in population {name},\n" - f"\tand we specified {df[df.child==name].popsize.item()} individuals in our simulations.") - assert n_samples == df[df.child==name].popsize.item() -``` - -Let's do an additional consistency check now, to see if we need to recapitate -(i.e., if some trees haven't coalesced), -and to make sure that all roots are in the root population, -as they should be: -```{code-cell} -# TODO: fix up -# for t in tsu.trees(): -# for r in t.roots: -# assert tsu.node(r).population == pop_ids["root"] - -print(f"Max number of roots: {max([t.num_roots for t in tsu.trees()])}.") -``` - -Finally, we will recapitate the result with a small population size of 100, -in case some trees on the root branch haven't coalesced, -and write out the result: - -```{code-cell} -tsu = pyslim.recapitate(tsu, recombination_rate=1e-8, ancestral_Ne=100) -tsu.dump("final.trees") -``` - -Now we're done, and can analyse the final tree sequence! -Just for fun, I'll look at the trees produced by the simulation. -For instance, we might be curious how often there are -disagreements between the species tree and the simulated gene trees -(also called incomplete lineage sorting, or ILS). - -To make it possible to look at the trees, -I will first simplify the union-ed tree sequence to keep only two diploid -samples per population. - -```{code-cell} -rng = np.random.default_rng(seed=123) -ind_alive = pyslim.individuals_alive_at(tsu, 0) -# TODO: this will work in the next tskit -# ind_pops = tsu.individuals_population[ind_alive] -ind_pops = np.array([tsu.node(tsu.individual(i).nodes[0]).population for i in ind_alive]) -subsample_indivs = [ - rng.choice(ind_alive[ind_pops == pop_ids[name]], 2) - for name in pops -] -subsample_nodes = [ - np.concatenate([tsu.individual(i).nodes for i in x]) - for x in subsample_indivs -] -tsus = tsu.simplify( - np.concatenate(subsample_nodes), - filter_populations=False, -) -pop_labels = {v: k for k, v in pop_ids.items()} -SVG(tsus.draw_svg( - node_labels={ - node.id: pop_labels[node.population] - for node in tsus.nodes() - if not node.time > 0.0 - }, - x_lim=[0,2200], - size=(800, 300), -)) -``` - -:::{note} -A possible gotcha in the code above lies in getting the time units to work out. -Note that in the SLiM script we both save and reload .trees files in the -``late()`` stage of the SLiM life cycle. This is important: if we had reloaded the -files in ``early()``, then each time we did so the "tskit time" and "SLiM time" -would become one step out of sync. This leads to errors either in union (since -if the time units in the two tree sequences do not match, union will raise an error) -or in recapitate (since recapitate assumes that the "top" of the trees are at -the number of generations ago recorded by SLiM in metadata). -::: - diff --git a/docs/vignette_space.md b/docs/vignette_space.md deleted file mode 100644 index 3c1d3ea2..00000000 --- a/docs/vignette_space.md +++ /dev/null @@ -1,464 +0,0 @@ ---- -jupytext: - text_representation: - extension: .md - format_name: myst - format_version: 0.12 - jupytext_version: 1.9.1 -kernelspec: - display_name: Python 3 - language: python - name: python3 -execution: - timeout: 90 ---- - -```{code-cell} -:tags: [remove-cell] - -import pyslim, tskit, msprime -from IPython.display import SVG -import numpy as np -import util - -np.random.seed(1234) -``` - - -(sec_vignette_space)= - - -# Vignette: A spatial simulation - -Here we'll talk through a typical workflow with pyslim, -which will: - -1. Simulate data with SLiM, remembering some ancestral individuals. -2. Recapitate and mutate. -3. Take a subsample of the modern and ancestral individuals. -4. Get these individual locations and make a map. -5. Compute divergences between individuals, and plot against geographic distance. -6. Write out a VCF file of these individuals' genotypes and other data for use by other programs. - - -## Simulation - -Here is a simple spatial SLiM recipe that simulates 1000 individuals on a spatial landscape. -The focus of this vignette is not on SLiM, so we won't go into detail here. -Here are notes: - -1. It does not have *any* mutations: we'll add these on afterwards. -2. There is local fecundity regulation of population density: individuals with more neighbors - have fewer offspring. -3. We run the simulation for 2000 time steps, and "remember" everyone who is alive at time step 1000. - -```{literalinclude} vignette_space.slim -``` -```{code-cell} -%%bash -slim -s 23 vignette_space.slim -``` - -Ok, now let's have a quick look at the output: - -```{code-cell} -import tskit -slim_ts = tskit.load("spatial_sim.trees") -print(f"The tree sequence has {slim_ts.num_trees} trees\n" - f"on a genome of length {slim_ts.sequence_length},\n" - f"{slim_ts.num_individuals} individuals, {slim_ts.num_samples} 'sample' genomes,\n" - f"and {slim_ts.num_mutations} mutations.") -``` - -It makes sense we have no mutations: we haven't added any yet. -The tree sequence is recording the relationship between 5,424 genomes (the "samples"), -which requires 37,095 distinct trees along the genome. -Individuals are diploid, which explains why the number of individuals -is equal to half the number of samples. -Let's have a look at how old those individuals are, -by tabulating when they were born: - -```{code-cell} -import numpy as np -individual_times = slim_ts.individuals_time -for t in np.unique(individual_times): - print(f"There are {np.sum(individual_times == t)} individuals from time {t}.") -``` - -These "times" record the birth times of each individual. -These are *tskit* times, which are in units of "time ago", -so for instance, there are 343 individuals born one time unit before the end of the simulation -and 167 born two time units before the end of the simulation. -(This confusing choice of units is because tskit was developed for msprime, a coalescent simulator.) -This also tells us that there's a bunch of individuals born around 1000 time steps ago, -when we asked SLiM to Remember everyone alive at the time, -and some more in the past few time steps, i.e., the present. -This is a non-Wright-Fisher simulation, -and so individuals may live for more than one time step (even up to age 10, it seems). -Let's check that all these individuals are alive at either (a) today or (b) 1000 time steps ago. - -```{code-cell} -for t in [0, 1000]: - alive = pyslim.individuals_alive_at(slim_ts, t) - print(f"There were {len(alive)} individuals alive {t} time steps in the past.") -``` - -And, 1242 + 1255 is 2497, the total number of individuals. -So, this all checks out. - - -## Recapitation and mutation - -Next, we want to (a) simulate some ancestral diversity and (b) add in neutral mutations. -Please see [Haller et al (2019)](https://onlinelibrary.wiley.com/doi/abs/10.1111/1755-0998.12968>) -for the why and how of these steps. -But, first let's see if recapitation is necessary: -on how much of the genome is the tree sequence not coalesced? -In other words, recapitation adds diversity present in the initial generation; -will it make a difference? -In fact, *no* segments of the genome have coalesced: - -```{code-cell} -print(f"Number of trees with only one root: {sum([t.num_roots == 1 for t in slim_ts.trees()])}\n" - f"Number with more than one root: {sum([t.num_roots > 0 for t in slim_ts.trees()])}") -``` - -Next, we will: - -1. Recapitate, running a coalescent simulation to build ancestral trees. -2. Mutate, adding neutral variation. -3. Save the resulting tree sequence to disk for future use. - -We *won't* simplify, since we may as well keep around all the information. -But, if we did (e.g., if we were running a large number of simulations), -we would need to pass ``keep_input_roots=True`` to allow recapitation. - -:::{note} - The units of time in the tree sequence are SLiM's "time steps", and - so are not necessarily equal to the mean generation time in a - non-Wright-Fisher model. Per-generation rates need to be divided by the - mean generation time, which can be measured in SLiM. -::: - -```{code-cell} -recap_ts = pyslim.recapitate(slim_ts, recombination_rate=1e-8, ancestral_Ne=1000) -ts = msprime.sim_mutations( - recap_ts, - rate=1e-8, - model=msprime.SLiMMutationModel(type=0), - keep=True, -) -ts.dump("spatial_sim.recap.trees") - -print(f"The tree sequence now has {ts.num_trees} trees,\n" - f" and {ts.num_mutations} mutations.") -``` -See [](sec_tutorial_adding_neutral_mutations) for discussion of the options to -{func}`msprime.sim_mutations`. - - -We will have no further use for ``slim_ts`` or for ``recap_ts``; -we've just given them separate names for tidyness. -And, since the original SLiM mutation had no mutations, we didn't need to specify ``keep=True`` -in {func}`sim_mutations `, but if we *had* put down selected mutations with SLiM -we'd probably want to keep them around. - - -## Take a sample of individuals - -Now it's time to compute some things. -In real life we don't get to work with *everyone* usually, -so we'll take a subset of individuals. -The range we have simulated has width and height 35 units, -with a population density of around 1 per unit area. -We'll get genomes to work with by pulling out - -1. All the modern individuals in the five squares of width 5 in the corners of the range - and the center, and -2. Five individuals sampled randomly from everyone alive 1000 time steps ago. - -```{code-cell} - -np.random.seed(23) - -alive = pyslim.individuals_alive_at(ts, 0) -locs = ts.individuals_location[alive, :] - -W = 35 -w = 5 -groups = { - 'topleft' : alive[np.logical_and(locs[:, 0] < w, locs[:, 1] < w)], - 'topright' : alive[np.logical_and(locs[:, 0] < w, locs[:, 1] > W - w)], - 'bottomleft' : alive[np.logical_and(locs[:, 0] > W - w, locs[:, 1] < w)], - 'bottomright' : alive[np.logical_and(locs[:, 0] > W - w, locs[:, 1] > W - w)], - 'center' : alive[np.logical_and(np.abs(locs[:, 0] - W/2) < w/2, - np.abs(locs[:, 1] - W/2) < w/2)] - } - -old_ones = pyslim.individuals_alive_at(ts, 1000) -groups['ancient'] = np.random.choice(old_ones, size=5) - -for k in groups: - print(f"We have {len(groups[k])} individuals in the {k} group.") -``` - -To keep names associated with each subset of individuals, -we've kept the individuals in a dict, so that for instance -``groups["topleft"]`` is an array of all the individual IDs that are in the top left corner. -The IDs of the ancient individuals we will work with are kept in the array ``ancient``. - -Let's do a quick consistency check, that everyone in ``ancient`` was actually born around 1000 time steps ago: - -```{code-cell} -for i in groups["ancient"]: - ind = ts.individual(i) - # TODO: will work on next tskit release - # assert(ind.time >= 1000 and ind.time < 1020) - time = ts.node(ind.nodes[0]).time - assert(time >= 1000 and time < 1020) -``` -No errors occurred, so that checks out. - -## Plotting locations - -We should check this: plot where these individuals lie -relative to everyone else. -The individuals locations are available as a property of individuals, -but to make things easier, it's also present in a `num_individuals x 3` -numpy array as ``ts.individuals_location``. -(There are three columns because SLiM allows for -`(x, y, z)` coordinates, but we'll just use the first two.) -Since ``groups["topleft"]`` is an array of individual IDs, -we can pull out the locations of the "topleft" individuals -by indexing the rows of the individual location array: -```{code-cell} -print("Locations:") -all_locs = ts.individuals_location -print(all_locs) -print("shape:") -all_locs.shape -print("topleft locations shape:") -all_locs[groups["topleft"], :].shape -``` - -Using this, we can easily plot the locations of all the individuals from today -(on the left) and 1000 time steps ago (on the right). -We have to do a bit of mucking around to set the colors so that they reflect -which group each individual is in. - -```{code-cell} -import matplotlib -import matplotlib.pyplot as plt - -group_order = ['topleft', 'topright', 'bottomleft', 'bottomright', 'center', 'ancient'] -ind_colors = np.repeat(0, ts.num_individuals) -for j, k in enumerate(group_order): - ind_colors[groups[k]] = 1 + j - -old_locs = ts.individuals_location[old_ones, :] - -fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 6), dpi=300) -ax1.set_title("today") -ax1.scatter(locs[:,0], locs[:,1], s=20, c=ind_colors[alive]) -ax2.set_title("long ago") -ax2.scatter(old_locs[:, 0], old_locs[:, 1], s=20, c=ind_colors[old_ones]); -``` - - -## Isolation by distance - -Now, let's look at *isolation by distance*, i.e., -let's compare geographic and genetic distances. -Here, "genetic distance" will be mean pairwise sequence divergence. -First, we'll compute mean genetic distance between each of our five groups. - -The first thing we need to do is some bookkeeping. -So far, we've just worked with *individuals*, -but tree sequence tools, in particular the statistics computation methods from tskit, -are designed to work with *genomes*, also known as "nodes". -So, first we need to pull out the *node IDs* corresponding to the individuals we want. -The things that make up a tree sequence - individuals, nodes, mutations, etcetera - -can generally be examined individually. -For instance, here's what we have for the the first "ancient" individual: - -```{code-cell} -:tags: ["remove-output"] -print(ts.individual(groups['ancient'][0])) -``` -```{code-cell} -:tags: ["remove-input"] -util.pp(ts.individual(groups['ancient'][0])) -``` - -Notice that among other things, each individual carries around a list of their node IDs, -i.e., their genomes. -We need to put these all in a list of lists, -so that, for instance, the first element of the list will have the node IDs of all the genomes -of the individuals in the "topleft" group. -And, since we kept the individual IDs in a dict, which are unordered, -we'll have to do some extra work to make sure we keep track of order. - -```{code-cell} -sampled_nodes = [[] for _ in groups] -for j, k in enumerate(group_order): - for ind in groups[k]: - sampled_nodes[j].extend(ts.individual(ind).nodes) -``` - -Let's do a consistency check: the number of nodes in each element of this list -should be twice the number of individuals in the corresponding list. -```{code-cell} -print([len(groups[k]) for k in groups]) -print([len(u) for u in sampled_nodes]) -``` -For instance, in the 'topleft' corner there are 12 diploids, -with 24 nodes. That checks out. - -Now, we can compute the matrix of pairwise mean sequence divergences -between and within these sets. -This is done using the {meth}`ts.divergence ` method. - -```{code-cell} - -pairs = [(i, j) for i in range(6) for j in range(6)] -group_div = ts.divergence(sampled_nodes, indexes=pairs).reshape((6, 6)) - -print("\t" + "\t".join(group_order)) -for i, group in enumerate(group_order): - print(f"{group_order[i]}:\t" + "\t".join(map(str, np.round(group_div[i], 7)))) -``` - - -That's nice, but to look at isolation by distance, -we should actually separate out the individuals. -To do that, we need to create a list of lists of nodes -whose j-th entry is the nodes belonging to the j-th individual, -and to keep track of which group each one belongs to. - -```{code-cell} -ind_nodes = [] -ind_group = [] -ind_ids = [] -for j, group in enumerate(group_order): - for ind in groups[group]: - ind_ids.append(ind) - ind_nodes.append(ts.individual(ind).nodes) - ind_group.append(group_order[j]) - -nind = len(ind_ids) -pairs = [(i, j) for i in range(nind) for j in range(i, nind)] -ind_div = ts.divergence(ind_nodes, indexes=pairs) -``` - -Here we've only computed divergences in the *upper triangle* of the pairwise divergence matrix, -with heterozygosities on the diagonal. -We'll also need pairwise geographic distances: - -```{code-cell} -geog_dist = np.repeat(0.0, len(pairs)) -locs = ts.individuals_location -for k, (i, j) in enumerate(pairs): - geog_dist[k] = np.sqrt(np.sum( - (locs[ind_ids[i], :2] - - locs[ind_ids[j], :2])**2 - )) -``` - -Let's check that makes sense: distances of individuals from themselves should be zero. - -```{code-cell} -for (i, j), x in zip(pairs, geog_dist): - if i == j: - assert(x == 0) -``` - -Python does not complain, which is good. -Now let's plot genetic distance against geographic distance. - -```{code-cell} -pair_colors = np.repeat(0, len(pairs)) -for k, (i, j) in enumerate(pairs): - if ind_group[i] == "ancient" or ind_group[j] == "ancient": - pair_colors[k] = 1 - -fig = plt.figure(figsize=(6, 6), dpi=300) -ax = fig.add_subplot(111) -ax.scatter(geog_dist, 1e3 * ind_div, s=20, alpha=0.5, - c=pair_colors) -ax.set_xlabel("geographic distance") -ax.set_ylabel("genetic distance (diffs/Kb)"); -``` - - -Since we multiplied ``ind_div`` by 1,000, -the units of genetic distance are in mean number of nucleotide differences per kilobase. -It is clear that closer samples are more closely related, -and the distinct clusters corresponding to the five sampled boxes are visible. -Furthermore, ancient samples are generally more distantly diverged. - - -## VCF output - -Now we want to write out these data for analysis with other programs. -To do this, and make sure that everything stays nicely cross-referenced, -we're going to loop through the sampled individuals, writing their information to a file, -while at the same time constructing a list of individual IDs, -whose genomes we will write out to a VCF file. - -```{code-cell} - -indivlist = [] -indivnames = [] -with open("spatial_sim_individuals.txt", "w") as indfile: - indfile.writelines("\t".join(["vcf_label", "tskit_id", "slim_id"] - + ["birth_time_ago", "age", "x", "y"]) + "\n") - for group in group_order: - for i in groups[group]: - indivlist.append(i) - ind = ts.individual(i) - vcf_label = f"tsk_{ind.id}" - indivnames.append(vcf_label) - time = ts.node(ind.nodes[0]).time - data = [vcf_label, str(ind.id), str(ind.metadata["pedigree_id"]), str(time), - str(ind.metadata["age"]), str(ind.location[0]), str(ind.location[1])] - indfile.writelines("\t".join(data) + "\n") - -with open("spatial_sim_genotypes.vcf", "w") as vcffile: - ts.write_vcf(vcffile, individuals=indivlist, individual_names=indivnames) -``` - - -## More information - -1. The distinction between "nodes" (i.e., genomes) and "individuals" can be confusing, - as well as the idea of "samples". - Please see the - {ref}`the tskit data model` - for more explanation about these concepts. - -2. The general interface for computing statistics (explaining, for instance, the "indexes" - argument above) is described in {ref}`the tskit documentation` also. - - -## What about simplification? - -The tree sequence we worked with here contains more information than we need, -including the first generation individuals. -If we wanted to remove this, we could have used the -{meth}`simplify ` method, -which reduced the tree sequence to the minimal required to record the information -about a provided set of nodes. -In the workflow above we didn't ever *simplify* the tree sequence, -because we didn't need to. -Because simplify reorders nodes and removes unused individuals and populations, -it requires an extra layer of bookkeeping. -Such relabeling also makes it harder to compare results across different analyses -of the same data. - -Simplifying the tree sequence down to the nodes of the individuals -in our "groups" would not change any subsequent analysis (except perhaps -removing monomorphic sites in the VCF output), -and would speed up computation of diversity. -Since the calculation was fast already, it wasn't worth it in this case, -but for much larger tree sequences it could be worth the extra code complexity. - diff --git a/pyproject.toml b/pyproject.toml index e86faeab..0a43898e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ keywords = ["tree sequences", "tskit"] requires-python = ">=3.11" dependencies = [ "msprime>=1.0.1", - "tskit", + "tskit>=1.0.3", "numpy", ] @@ -56,6 +56,7 @@ test = [ "tskit", "msprime", "pandas", + "frozendict", ] docs = [ diff --git a/pyslim/_version.py b/pyslim/_version.py index 3b480d79..8eea699a 100644 --- a/pyslim/_version.py +++ b/pyslim/_version.py @@ -5,6 +5,6 @@ except Exception: pyslim_version = "unknown" -slim_file_version = "0.9" +slim_file_version = "1.0" # other file versions that require no modification -compatible_slim_file_versions = ["0.9"] +compatible_slim_file_versions = ["1.0"] diff --git a/pyslim/methods.py b/pyslim/methods.py index 19c12206..cf84634c 100644 --- a/pyslim/methods.py +++ b/pyslim/methods.py @@ -12,6 +12,7 @@ set_metadata_schemas, set_tree_sequence_metadata, ) +from .slim_tree_sequence import mutation_metadata from .util import unique_labels_by_group @@ -43,7 +44,7 @@ def _mark_not_samples(tables, nodes): ) -def _chromosome_index(ts): +def _chromosome_index(ts_metadata): """ For a tree sequence produced by a multichromosome simulation, returns the index of the chromosome whose information is stored in this tree sequence @@ -52,18 +53,19 @@ def _chromosome_index(ts): ``ts.metadata['SLiM']['this_chromosome']``, and provides the index of this chromosome into `ts.metadata['SLiM']['chromosomes']``, if present. - :param tskit.TreeSequence ts: The tree sequence or table collection. + :param dict ts_metadata: The top-level metadata from a tree sequence + or table collection. """ if not ( - isinstance(ts.metadata, dict) - and "SLiM" in ts.metadata - and "this_chromosome" in ts.metadata["SLiM"] + isinstance(ts_metadata, dict) + and "SLiM" in ts_metadata + and "this_chromosome" in ts_metadata["SLiM"] ): raise ValueError( "The tree sequence does not have the necessary " "information in top-level metadata." ) - k = ts.metadata["SLiM"]["this_chromosome"]["index"] + k = ts_metadata["SLiM"]["this_chromosome"]["index"] return k @@ -88,15 +90,17 @@ def _is_chrom_vacant(k, b): return (b >> i & 1) > 0 -def has_vacant_samples(ts): +def has_vacant_samples(ts, _ts_metadata=None): """ Returns whether the tree sequence has vacant sample nodes. See :meth:`remove_vacant`. :param tskit.TreeSequence ts: The tree sequence. """ + if _ts_metadata is None: + _ts_metadata = ts.metadata out = False - k = _chromosome_index(ts) + k = _chromosome_index(_ts_metadata) for n in ts.samples(): md = ts.node(n).metadata if md is not None: @@ -106,8 +110,45 @@ def has_vacant_samples(ts): return out +def nodes_vacant(ts): + """ + Evaluates which nodes in the tree sequence are vacant: returns a boolean + vector whose k-th element is True if the k-th node is labelled as *vacant* + in the node's metadata recorded by SLiM. A vacant node represents a blank + placeholder in SLiM: either a "null haplosome" (used as placeholders for + sex chromosomes and other chromosome types not of consistent ploidy in all + individuals) or simply an unused node for haploid chromosome types. See + :meth:`remove_vacant`. + + + :param tskit.TreeSequence ts: The tree sequence. + :return boolean ndarray: + """ + # not using chrom_index here because we expect people to call this on lots of nodes + k = ts.metadata["SLiM"]["this_chromosome"]["index"] + out = np.array( + [ + node.metadata is not None and _is_chrom_vacant(k, node.metadata["is_vacant"]) + for node in ts.nodes() + ], + dtype="bool", + ) + return out + + def node_is_vacant(ts, node): """ + **DEPRECATED:** use :func:`.nodes_vacant` instead. This function requires + top-level metadata access, which can be costly, so it is much better to do, + for instance: + + .. code-block:: python + + vacant = nodes_vacant(ts) + for node in ts.nodes(): + # instead of node_is_vacant(ts, node), use: + vacant[node.id] + Returns True if the node is labelled as *vacant* in the node's metadata recorded by SLiM. A vacant node represents a blank placeholder in SLiM: either a "null haplosome" (used as placeholders for sex chromosomes and other @@ -117,12 +158,19 @@ def node_is_vacant(ts, node): :param tskit.TreeSequence ts: The tree sequence. :param tskit.Node node: The node object. """ + warnings.warn( + "The node_is_vacant method is deprecated: changes in SLiM v6 " + " means that repeated use of this method will be unreasonably slow, " + "so it will be removed in a future version of pyslim: " + "obtain this information from pyslim.vacant_nodes( ) instead.", + FutureWarning, + ) # not using chrom_index here because we expect people to call this on lots of nodes k = ts.metadata["SLiM"]["this_chromosome"]["index"] return node.metadata is not None and _is_chrom_vacant(k, node.metadata["is_vacant"]) -def _record_vacant_tables(tables): +def _record_vacant_tables(tables, ts_metadata): """ Sets the NODE_IS_VACANT_SAMPLE flag for all vacant, sample nodes. See :meth:`remove_vacant`. @@ -135,7 +183,7 @@ def _record_vacant_tables(tables): "flags are being overwritten; this may mean you've already run " "remove_vacant and so don't need to run it again." ) - k = _chromosome_index(tables) + k = _chromosome_index(ts_metadata) dn = tables.nodes.asdict() dn["flags"] &= ~NODE_IS_VACANT_SAMPLE @@ -156,7 +204,7 @@ def _remove_vacant_sample_flags(tables): tables.nodes.set_columns(**dn) -def remove_vacant(ts): +def remove_vacant(ts, _ts_metadata=None): """ Remove sample flags from all vacant nodes. @@ -177,23 +225,27 @@ def remove_vacant(ts): :param tskit.TreeSequence ts: The tree sequence. """ + if _ts_metadata is None: + _ts_metadata = ts.metadata tables = ts.dump_tables() - remove_vacant_tables(tables) + remove_vacant_tables(tables, _ts_metadata) return tables.tree_sequence() -def remove_vacant_tables(tables): +def remove_vacant_tables(tables, _ts_metadata=None): """ Does the work of :meth:`remove_vacant`, modifying ``tables`` in place. :param tskit.TableCollection tables: The tables underlying a tree sequence. """ - _record_vacant_tables(tables) + if _ts_metadata is None: + _ts_metadata = tables.metadata + _record_vacant_tables(tables, _ts_metadata) is_vacant = np.where(tables.nodes.flags & NODE_IS_VACANT_SAMPLE > 0)[0] _mark_not_samples(tables, is_vacant) -def restore_vacant(ts): +def restore_vacant(ts, _ts_metadata=None): """ The inverse of :meth:`remove_vacant`. @@ -204,19 +256,23 @@ def restore_vacant(ts): :param tskit.TreeSequence ts: The tree sequence. """ + if _ts_metadata is None: + _ts_metadata = ts.metadata tables = ts.dump_tables() - restore_vacant_tables(tables) + restore_vacant_tables(tables, _ts_metadata) return tables.tree_sequence() -def restore_vacant_tables(tables): +def restore_vacant_tables(tables, _ts_metadata=None): """ Does the work of :meth:`restore_vacant`, modifying ``tables`` in place. :param tskit.TableCollection tables: The tables underlying a tree sequence. """ + if _ts_metadata is None: + _ts_metadata = tables.metadata is_vacant = np.where(tables.nodes.flags & NODE_IS_VACANT_SAMPLE > 0)[0] - k = _chromosome_index(tables) + k = _chromosome_index(_ts_metadata) for j in is_vacant: n = tables.nodes[j] if n.metadata is None: @@ -273,14 +329,15 @@ def recapitate(ts, ancestral_Ne=None, *, keep_vacant=False, **kwargs): vacant sample nodes. Default: False. :param dict kwargs: Any other arguments to :func:`msprime.sim_ancestry`. """ - is_current_version(ts, _warn=True) + ts_metadata = ts.metadata + is_current_version(ts_metadata, _warn=True) # we need to ask msprime to *not* simulate from any 'vacant' haplosomes; # which we do by marking these as not samples; note that `initial_state` # can take a TableCollection, not just a TreeSequence - has_vacant = has_vacant_samples(ts) + has_vacant = has_vacant_samples(ts, ts_metadata) if has_vacant: - ts = remove_vacant(ts) + ts = remove_vacant(ts, ts_metadata) if ancestral_Ne is not None: if "demography" in kwargs: @@ -338,11 +395,90 @@ def recapitate(ts, ancestral_Ne=None, *, keep_vacant=False, **kwargs): recap = msprime.sim_ancestry(initial_state=ts, **kwargs) if has_vacant and keep_vacant: - recap = restore_vacant(recap) + recap = restore_vacant(recap, ts_metadata) return recap +def add_mutation_metadata(ts, mutation_type=0, remove_unused=False): + """ + Returns a new tree sequence with default information added to the top-level metadata + for each mutation in the tree sequence for which that information is not already present. + To do this, this method looks for all SLiM IDs that are found in the derived + state of some mutation but are not represented in the top-level metadata + (see :func:`.mutation_metadata`). This function then adds entries to that top-level + metadata with default values (see :func:`.default_slim_metadata`), + except that (a) the ``mutation_type`` can be specified; + and (b) the ``slim_time`` is set using the ``tick`` value in top-level metadata + and the ``time`` of the oldest tskit mutation in which the SLiM mutation occurs. + + :param tskit.TreeSequence ts: The tree sequence to transform. + :param int mutation_type: The numeric ID of the mutation type in SLiM. + :param bool remove_unused: Whether to also remove from metadata information about any + mutations not seen in the derived states of the tree sequence. + """ + tables = ts.dump_tables() + add_mutation_metadata_tables( + tables, mutation_type=mutation_type, remove_unused=remove_unused + ) + return tables.tree_sequence() + + +def add_mutation_metadata_tables(tables, mutation_type=0, remove_unused=False): + """ + Modifies the tables in place to add metadata for any mutations for which it is missing; + see :func:`.add_mutation_metadata`. + + :param tskit.TableCollection tables: The table collection to be modified. + :param int mutation_type: The numeric ID of the mutation type in SLiM. + :param bool remove_unused: Whether to also remove from metadata information about any + mutations not seen in the derived states of the tree sequence. + """ + ts_metadata = tables.metadata + if ( + not isinstance(ts_metadata, dict) + or "SLiM" not in ts_metadata + or "SLiM_mutation_list" not in ts_metadata + ): + raise ValueError( + "Top-level metadata schema is not correct: " + "do you need to run pyslim.annotate()?" + ) + existing_muts = {x["mutation_id"] for x in ts_metadata["SLiM_mutation_list"]} + mut_ids = [ + (int(j), mut.time) + for mut in tables.mutations + for j in mut.derived_state.split(",") + ] + mut_ids.sort() + mut_ids = np.array(mut_ids, dtype="int") # floors times + # remove duplicate IDs, keeping the last (most recent) + keep = np.full(len(mut_ids), True, dtype="bool") + keep[np.where(np.diff(mut_ids[:, 0]) == 0)[0]] = False + mut_ids = mut_ids[keep, :] + mut_ids[:, 1] = slim_time( + tables, mut_ids[:, 1], stage="late", ts_metadata=ts_metadata + ) + # this assumes mutations were added in late(), which is what SLiM does + ts_metadata["SLiM_mutation_list"].extend( + [ + default_slim_metadata( + "mutation_list_entry", + mutation_id=int(j), + mutation_type=mutation_type, + slim_time=int(t), + ) + for j, t in mut_ids + if j not in existing_muts + ] + ) + if remove_unused and len(mut_ids) < len(ts_metadata["SLiM_mutation_list"]): + ts_metadata["SLiM_mutation_list"] = [ + x for x in ts_metadata["SLiM_mutation_list"] if x["mutation_id"] in mut_ids + ] + tables.metadata = ts_metadata + + def convert_alleles(ts): """ Returns a modified tree sequence in which alleles have been replaced by @@ -379,25 +515,28 @@ def convert_alleles(ts): # so we must guess which is the most recent, by choosing the one that # has the largest SLiM time, doesn't appear in the parent list, or has # the lagest SLiM ID. - nuc_inds = tables.mutations.metadata_vector( - ["mutation_list", 0, "nucleotide"], dtype="int" - ) - num_stacked = np.array([len(m.metadata["mutation_list"]) for m in ts.mutations()]) - for k in np.where(num_stacked > 1)[0]: + mut_metadata = mutation_metadata(ts) + mut_ids = np.array([x["mutation_id"] for x in mut_metadata.values()], dtype="int") + alleles = np.array([x["nucleotide"] for x in mut_metadata.values()], dtype="int") + # mut_inds will map from tskit-mutations to slim-mutations + mut_inds = ts.mutations_derived_state.copy() + num_stacked = np.strings.count(mut_inds, ",") + mut_inds[num_stacked > 0] = "-1" + mut_inds = mut_inds.astype("int", copy=False) + for k in np.where(num_stacked > 0)[0]: mut = ts.mutation(k) if mut.parent == tskit.NULL: pids = [] else: pids = ts.mutation(mut.parent).derived_state.split(",") x = [ - (md["slim_time"], i not in pids, int(i), j) - for j, (i, md) in enumerate( - zip(mut.derived_state.split(","), mut.metadata["mutation_list"]) - ) + (mut_metadata[int(i)]["slim_time"], i not in pids, int(i), j) + for j, i in enumerate(mut.derived_state.split(",")) ] x.sort() - j = x[-1][3] - nuc_inds[k] = mut.metadata["mutation_list"][j]["nucleotide"] + mut_inds[k] = x[-1][2] + assert np.all(mut_inds >= 0), "This should not occur: please file a bug report." + nuc_inds = alleles[np.searchsorted(mut_ids, mut_inds)] if np.any(nuc_inds == -1): raise ValueError("All mutations must be nucleotide mutations.") da = np.array(NUCLEOTIDES)[nuc_inds] @@ -405,7 +544,6 @@ def convert_alleles(ts): k = tables.sites.position.astype("int") aa = np.frombuffer(ts.reference_sequence.data.encode("utf-8"), dtype="S1")[k] tables.sites.packset_ancestral_state(aa.tobytes().decode("utf-8")) - return tables.tree_sequence() @@ -461,11 +599,11 @@ def generate_nucleotides(ts, reference_sequence=None, keep=True, seed=None): raise ValueError( "Reference sequence must be a string of A, C, G, and T only." ) - + ts_metadata = ts.metadata + mut_info = mutation_metadata(ts, _ts_metadata=ts_metadata) tables = ts.dump_tables() if reference_sequence is not None: tables.reference_sequence.data = reference_sequence - tables.mutations.clear() sets = [[k for k in range(4) if k != i] for i in range(4)] states = np.full((ts.num_mutations,), -1) k = tables.sites.position.astype("int") @@ -484,9 +622,9 @@ def generate_nucleotides(ts, reference_sequence=None, keep=True, seed=None): pa = states[mut.parent] pds = ts.mutation(mut.parent).derived_state.split(",") this_da = pa - ml = mut.metadata max_time = -np.inf - for i, md in zip(mut.derived_state.split(","), ml["mutation_list"]): + for i in mut.derived_state.split(","): + md = mut_info[int(i)] da = md["nucleotide"] if da == -1 or not keep: if i in muts: @@ -502,22 +640,26 @@ def generate_nucleotides(ts, reference_sequence=None, keep=True, seed=None): this_da = da max_time = md["slim_time"] states[mut.id] = this_da - tables.mutations.append(mut.replace(metadata=ml)) - md = tables.metadata - md["SLiM"]["nucleotide_based"] = True - tables.metadata = md + ts_metadata["SLiM"]["nucleotide_based"] = True + ts_metadata["SLiM_mutation_list"] = list(mut_info.values()) + tables.metadata = ts_metadata return tables.tree_sequence() -def individual_ages(ts): +def individual_ages(ts, ts_metadata=None): """ Returns the ages of all individuals in the tree sequence, extracted from metadata. The result is a array of length equal to the number of individuals, with k-th entry equal to ``ts.individual(k).metadata["age"]``. + :param tskit.TreeSequence ts: The tree sequence. + :param dict ts_metadata: Optionally, the top-level metadata for ``ts``. If + this does not match the actual top-level metadata, incorrect values may result. :return: An array of ages of individuals. """ - if ts.metadata["SLiM"]["model_type"] != "WF": + if ts_metadata is None: + ts_metadata = ts.metadata + if ts_metadata["SLiM"]["model_type"] != "WF": ages = ts.tables.individuals.metadata_vector("age") else: ages = np.zeros(ts.num_individuals, dtype="int") @@ -525,7 +667,13 @@ def individual_ages(ts): def individuals_alive_at( - ts, time, stage="late", remembered_stage=None, population=None, samples_only=False + ts, + time, + stage="late", + remembered_stage=None, + population=None, + samples_only=False, + ts_metadata=None, ): """ Returns an array giving the IDs of all individuals that are known to be @@ -576,26 +724,29 @@ def individuals_alive_at( population(s) with these population ID(s). :param bool samples_only: Whether to return only individuals who have at least one node marked as samples. + :param dict ts_metadata: Optionally, the top-level metadata for ``ts``. If + this does not match the actual top-level metadata, incorrect values may result. """ - is_current_version(ts, _warn=True) if stage not in ("late", "early", "first"): raise ValueError( f"Unknown stage '{stage}': should be either 'first', 'early' or 'late'." ) - + if ts_metadata is None: + ts_metadata = ts.metadata + is_current_version(ts_metadata, _warn=True) if remembered_stage is None: - remembered_stage = ts.metadata["SLiM"]["stage"] + remembered_stage = ts_metadata["SLiM"]["stage"] if remembered_stage not in ("late", "early", "first"): raise ValueError( f"Unknown remembered_stage '{remembered_stage}': " "should be either 'first', 'early' or 'late'." ) - if remembered_stage != ts.metadata["SLiM"]["stage"]: + if remembered_stage != ts_metadata["SLiM"]["stage"]: warnings.warn( f"Provided remembered_stage '{remembered_stage}' does not" " match the stage at which the tree sequence was saved" - f" ('{ts.metadata['SLiM']['stage']}'). This is not necessarily" + f" ('{ts_metadata['SLiM']['stage']}'). This is not necessarily" " an error, but mismatched stages will lead to inconsistencies:" " make sure you know what you're doing." ) @@ -615,12 +766,12 @@ def individuals_alive_at( # let x = 1 if the stage is 'first' or (is 'early' and WF) # and y = 1 if remembered stage is 'late' or (is 'early' and nonWF); # then t = time + x + y - 1 . - is_wf = ts.metadata["SLiM"]["model_type"] == "WF" + is_wf = ts_metadata["SLiM"]["model_type"] == "WF" x = stage == "first" or (stage == "early" and is_wf) y = remembered_stage == "late" or (remembered_stage == "early" and not is_wf) t = time + x + y - 1 birth_times = ts.individuals_time - ages = individual_ages(ts) + ages = individual_ages(ts, ts_metadata) if is_wf: alive_bool = birth_times == t else: @@ -643,7 +794,9 @@ def individuals_alive_at( return np.where(alive_bool)[0] -def individual_ages_at(ts, time, stage="late", remembered_stage="late"): +def individual_ages_at( + ts, time, stage="late", remembered_stage="late", ts_metadata=None +): """ Returns the `ages` of each individual at the corresponding time ago, which will be ``nan`` if the individual is either not born yet or dead. @@ -669,20 +822,24 @@ def individual_ages_at(ts, time, stage="late", remembered_stage="late"): is alive (either "early" or "late"; defaults to "late"). :param str remembered_stage: The stage in the SLiM life cycle during which individuals were Remembered. + :param dict ts_metadata: Optionally, the top-level metadata for ``ts``. If + this does not match the actual top-level metadata, incorrect values may result. """ + if ts_metadata is None: + ts_metadata = ts.metadata ages = np.repeat(np.nan, ts.num_individuals) alive = individuals_alive_at( - ts, time, stage=stage, remembered_stage=remembered_stage + ts, time, stage=stage, remembered_stage=remembered_stage, ts_metadata=ts_metadata ) # to convert individuals_time to number of ticks ago we subtract (y - 1), so - is_wf = ts.metadata["SLiM"]["model_type"] == "WF" + is_wf = ts_metadata["SLiM"]["model_type"] == "WF" y = remembered_stage == "late" or (remembered_stage == "early" and not is_wf) t = time + y - 1 ages[alive] = ts.individuals_time[alive] - t return ages -def slim_time(ts, time, stage="late"): +def slim_time(ts, time, stage="late", ts_metadata=None): """ Converts the given "tskit times" (i.e., in units of time before the end of the simulation) to SLiM times (those recorded by SLiM, usually in units @@ -707,17 +864,26 @@ def slim_time(ts, time, stage="late"): this may not return what you expect. See :ref:`sec_metadata_converting_times` for more discussion. + This method accesses top-level metadata, which may be a costly operation, + so if this method will be called many times, it is recommended to + extract this to a variable (e.g., ``ts_metadata = ts.metadata``) and pass it + to this method (as ``ts_metadata``). However, beware: if ``ts_metadata`` + is not in sync with the actual top-level metadata, incorrect values may result. + :param tskit.TreeSequence ts: A SLiM-compatible TreeSequence. :param numpy.ndarray time: An array of times to be converted. :param str stage: The stage of the SLiM life cycle that the SLiM time should be computed for. + :param dict ts_metadata: Optionally, the top-level metadata for ``ts``. """ - is_current_version(ts, _warn=True) - is_wf = ts.metadata["SLiM"]["model_type"] == "WF" - remembered_stage = ts.metadata["SLiM"]["stage"] + if ts_metadata is None: + ts_metadata = ts.metadata + is_current_version(ts_metadata, _warn=True) + is_wf = ts_metadata["SLiM"]["model_type"] == "WF" + remembered_stage = ts_metadata["SLiM"]["stage"] x = stage == "first" or (stage == "early" and is_wf) y = remembered_stage == "late" or (remembered_stage == "early" and not is_wf) - slim_time = ts.metadata["SLiM"]["tick"] - time + x + y - 1 + slim_time = ts_metadata["SLiM"]["tick"] - time + x + y - 1 return slim_time @@ -920,7 +1086,7 @@ def annotate(ts, **kwargs): :param str reference_sequence: A reference sequence of length equal to ts.sequence_length. :param bool annotate_mutations: Whether to replace mutation metadata - with defaults. (If False, the mutation table is unchanged.) + with defaults. (If False, information about mutations is unchanged.) """ tables = ts.dump_tables() annotate_tables(tables, **kwargs) @@ -965,12 +1131,15 @@ def annotate_tables( top_metadata["tick"] = tick top_metadata["cycle"] = cycle top_metadata["stage"] = stage - set_tree_sequence_metadata(tables, **top_metadata) + md = tables.metadata + if isinstance(md, dict) and "SLiM_mutation_list" in md: + top_metadata["SLiM_mutation_list"] = md["SLiM_mutation_list"] + ts_metadata = set_tree_sequence_metadata(tables, **top_metadata) set_metadata_schemas(tables) _annotate_nodes_individuals(tables, age=default_ages) _annotate_populations(tables) if annotate_mutations: - _annotate_sites_mutations(tables) + _annotate_sites_mutations(tables, ts_metadata=ts_metadata) if reference_sequence is not None: tables.reference_sequence.data = reference_sequence @@ -1109,46 +1278,46 @@ def _annotate_populations(tables): tables.populations[j] = p.replace(metadata=md) -def _annotate_sites_mutations(tables): +def _annotate_sites_mutations(tables, ts_metadata): """ Adds to a TableCollection the information relevant to mutations required - for SLiM to load in a tree sequence. This means adding to the metadata column - of the Mutation table, It will also + for SLiM to load in a tree sequence. This means adding metadata to the + SLiM_mutation_list in top-level metadata. It will also: - give SLiM IDs to each mutation - replace ancestral states with "" - This will replace any information already in the metadata or derived state - columns of the Mutation table. We set slim_time in metadata so that + This will replace any information already in the metadata, and the derived + state columns of the Mutation table. We set slim_time in metadata so that - tick = floor(tskit time) + slim_time """ - if len(tables.mutations.metadata) > 0: + if ( + isinstance(ts_metadata, dict) + and "SLiM_mutation_list" in ts_metadata + and len(ts_metadata["SLiM_mutation_list"]) > 0 + ): warnings.warn( - "The provided tree sequence already has some mutations with " + "The provided tree sequence already has top-level mutation " "metadata; this metadata will be overwritten." ) num_mutations = tables.mutations.num_rows default_mut = default_slim_metadata("mutation_list_entry") - dsb, dso = tskit.pack_bytes([str(j).encode() for j in range(num_mutations)]) - slim_time = tables.metadata["SLiM"]["tick"] - np.floor(tables.mutations.time).astype( + slim_time = ts_metadata["SLiM"]["tick"] - np.floor(tables.mutations.time).astype( "int" ) - mms = tables.mutations.metadata_schema - mutation_metadata = [ - mms.encode_row( - { - "mutation_list": [ - { - "mutation_type": default_mut["mutation_type"], - "selection_coeff": default_mut["selection_coeff"], - "subpopulation": default_mut["subpopulation"], - "slim_time": st, - "nucleotide": default_mut["nucleotide"], - } - ] - } - ) - for st in slim_time + mutation_list = [ + { + "mutation_id": j, + "mutation_type": default_mut["mutation_type"], + "per_trait": default_mut["per_trait"], + "subpopulation": default_mut["subpopulation"], + "slim_time": int(st), + "nucleotide": default_mut["nucleotide"], + "padding": None, + } + for j, st in enumerate(slim_time) ] - mdb, mdo = tskit.pack_bytes(mutation_metadata) + ts_metadata["SLiM_mutation_list"] = mutation_list + tables.metadata = ts_metadata + dsb, dso = tskit.pack_bytes([str(j).encode() for j in range(num_mutations)]) tables.mutations.set_columns( site=tables.mutations.site, node=tables.mutations.node, @@ -1156,8 +1325,6 @@ def _annotate_sites_mutations(tables): derived_state=dsb, derived_state_offset=dso, parent=tables.mutations.parent, - metadata=mdb, - metadata_offset=mdo, ) tables.sites.set_columns( position=tables.sites.position, diff --git a/pyslim/slim_metadata.py b/pyslim/slim_metadata.py index 0fb4144b..e515f559 100644 --- a/pyslim/slim_metadata.py +++ b/pyslim/slim_metadata.py @@ -1,6 +1,8 @@ +import copy import json import warnings +import numpy as np import tskit from ._version import * # noqa F403 @@ -38,227 +40,309 @@ def is_vacant_num_bytes(num_chromosomes): _raw_slim_metadata_schemas = { "tree_sequence": { "$schema": "http://json-schema.org/schema#", - "codec": "json", - "examples": [ - { - "SLiM": { - "file_version": "0.9", - "name": "fox", - "description": "foxes on Catalina island", - "cycle": 123, - "tick": 123, - "model_type": "WF", - "this_chromosome": { - "id": 1, - "index": 0, - "symbol": "1", - "name": "autosome_1", - "type": "A", - }, - "chromosomes": [ - {"id": 1, "symbol": "1", "name": "autosome_1", "type": "A"}, - {"id": 35, "symbol": "MT", "name": "mtDNA", "type": "HF"}, - ], - "nucleotide_based": False, - "separate_sexes": True, - "spatial_dimensionality": "xy", - "spatial_periodicity": "x", + "codec": "json+struct", + "json": { + "codec": "json", + "description": "SLiM schema for JSON top-level metadata.", + "examples": [ + { + "SLiM": { + "chromosomes": [ + {"id": 1, "name": "autosome_1", "symbol": "1", "type": "A"}, + {"id": 35, "name": "mtDNA", "symbol": "MT", "type": "HF"}, + ], + "cycle": 123, + "description": "foxes on Catalina island", + "file_version": "1.0", + "model_type": "WF", + "name": "fox", + "nucleotide_based": False, + "separate_sexes": True, + "spatial_dimensionality": "xy", + "spatial_periodicity": "x", + "this_chromosome": { + "id": 1, + "index": 0, + "name": "autosome_1", + "symbol": "1", + "type": "A", + }, + "tick": 123, + "traits": [ + {"index": 0, "name": "simT", "type": "multiplicative"} + ], + } } - } - ], - "properties": { - "SLiM": { - "description": "Top-level metadata for a SLiM tree sequence, file format version 0.9", - "properties": { - "file_version": { - "description": "The SLiM 'file format version' of this tree sequence.", - "type": "string", - }, - "name": { - "description": "The SLiM species name represented by this tree sequence.", - "type": "string", - }, - "description": { - "description": "A user-configurable description of the species represented by this tree sequence.", - "type": "string", - }, - "cycle": { - "description": "The 'SLiM cycle' counter when this tree sequence was recorded.", - "type": "integer", - }, - "tick": { - "description": "The 'SLiM tick' counter when this tree sequence was recorded.", - "type": "integer", - }, - "model_type": { - "description": "The model type used for the last part of this simulation (WF or nonWF).", - "enum": ["WF", "nonWF"], - "type": "string", - }, - "this_chromosome": { - "description": "The chromosome represented by the tree sequence in this file.", - "properties": { - "id": { - "description": "An integer identifier for the chromosome, unique within this set of tree sequences; often the chromosome number in the organism being represented, such as 1.", - "type": "integer", - }, - "index": { - "description": "The (zero-based) index of this chromosome in the chromosomes metadata array (if present), which should match the information given here.", - "type": "integer", - }, - "symbol": { - "description": 'A short string symbol for the chromosome, unique within this set of tree sequences, such as "1" or "MT".', - "type": "string", - }, - "name": { - "description": "A user-specified name for the chromosome, such as an accession identifier.", - "type": "string", - }, - "type": { - "description": "The type of chromosome, as specified by SLiM.", - "type": "string", + ], + "properties": { + "SLiM": { + "description": "Top-level metadata for a SLiM tree sequence, file format version 1.0", + "properties": { + "chromosomes": { + "description": "The chromosomes represented by the collection of tree sequences, of which this tree sequence is one member.", + "items": { + "properties": { + "id": { + "description": "An integer identifier for the chromosome, unique within this set of tree sequences; often the chromosome number in the organism being represented, such as 1.", + "type": "integer", + }, + "name": { + "description": "A user-specified name for the chromosome, such as an accession identifier.", + "type": "string", + }, + "symbol": { + "description": 'A short string symbol for the chromosome, unique within this set of tree sequences, such as "1" or "MT".', + "type": "string", + }, + "type": { + "description": "The type of chromosome, as specified by SLiM.", + "type": "string", + }, + }, + "required": ["id", "symbol", "type"], + "type": "object", }, + "type": "array", }, - "required": ["id", "index", "symbol", "type"], - "type": "object", - }, - "chromosomes": { - "description": "The chromosomes represented by the collection of tree sequences, of which this tree sequence is one member.", - "items": { + "cycle": { + "description": "The 'SLiM cycle' counter when this tree sequence was recorded.", + "type": "integer", + }, + "description": { + "description": "A user-configurable description of the species represented by this tree sequence.", + "type": "string", + }, + "file_version": { + "description": "The SLiM 'file format version' of this tree sequence.", + "type": "string", + }, + "model_type": { + "description": "The model type used for the last part of this simulation (WF or nonWF).", + "enum": ["WF", "nonWF"], + "type": "string", + }, + "name": { + "description": "The SLiM species name represented by this tree sequence.", + "type": "string", + }, + "nucleotide_based": { + "description": "Whether the simulation was nucleotide-based.", + "type": "boolean", + }, + "separate_sexes": { + "description": "Whether the simulation had separate sexes.", + "type": "boolean", + }, + "spatial_dimensionality": { + "description": "The spatial dimensionality of the simulation.", + "enum": ["", "x", "xy", "xyz"], + "type": "string", + }, + "spatial_periodicity": { + "description": "The spatial periodicity of the simulation.", + "enum": ["", "x", "y", "z", "xy", "xz", "yz", "xyz"], + "type": "string", + }, + "stage": { + "description": "The stage of the SLiM life cycle when this tree sequence was recorded.", + "type": "string", + }, + "this_chromosome": { + "description": "The chromosome represented by the tree sequence in this file.", "properties": { "id": { "description": "An integer identifier for the chromosome, unique within this set of tree sequences; often the chromosome number in the organism being represented, such as 1.", "type": "integer", }, - "symbol": { - "description": 'A short string symbol for the chromosome, unique within this set of tree sequences, such as "1" or "MT".', - "type": "string", + "index": { + "description": "The (zero-based) index of this chromosome in the chromosomes metadata array (if present), which should match the information given here.", + "type": "integer", }, "name": { "description": "A user-specified name for the chromosome, such as an accession identifier.", "type": "string", }, + "symbol": { + "description": 'A short string symbol for the chromosome, unique within this set of tree sequences, such as "1" or "MT".', + "type": "string", + }, "type": { "description": "The type of chromosome, as specified by SLiM.", "type": "string", }, }, - "required": ["id", "symbol", "type"], + "required": ["id", "index", "symbol", "type"], "type": "object", }, - "type": "array", - }, - "nucleotide_based": { - "description": "Whether the simulation was nucleotide-based.", - "type": "boolean", - }, - "separate_sexes": { - "description": "Whether the simulation had separate sexes.", - "type": "boolean", - }, - "spatial_dimensionality": { - "description": "The spatial dimensionality of the simulation.", - "enum": ["", "x", "xy", "xyz"], - "type": "string", - }, - "spatial_periodicity": { - "description": "The spatial periodicity of the simulation.", - "enum": ["", "x", "y", "z", "xy", "xz", "yz", "xyz"], - "type": "string", - }, - "stage": { - "description": "The stage of the SLiM life cycle when this tree sequence was recorded.", - "type": "string", - }, - }, - "required": [ - "model_type", - "tick", - "file_version", - "spatial_dimensionality", - "spatial_periodicity", - "this_chromosome", - "separate_sexes", - "nucleotide_based", - ], - "type": "object", - } - }, - "required": ["SLiM"], - "type": "object", - }, - "edge": None, - "site": None, - "mutation": { - "$schema": "http://json-schema.org/schema#", - "additionalProperties": False, - "codec": "struct", - "description": "SLiM schema for mutation metadata.", - "examples": [ - { - "mutation_list": [ - { - "mutation_type": 1, - "nucleotide": 3, - "selection_coeff": -0.2, - "slim_time": 243, - "subpopulation": 0, - } - ] - } - ], - "properties": { - "mutation_list": { - "items": { - "additionalProperties": False, - "properties": { - "mutation_type": { - "binaryFormat": "i", - "description": "The index of this mutation's mutationType.", - "index": 1, - "type": "integer", - }, - "nucleotide": { - "binaryFormat": "b", - "description": "The nucleotide for this mutation (0=A , 1=C , 2=G, 3=T, or -1 for none)", - "index": 5, - "type": "integer", - }, - "selection_coeff": { - "binaryFormat": "f", - "description": "This mutation's selection coefficient.", - "index": 2, - "type": "number", - }, - "slim_time": { - "binaryFormat": "i", - "description": "The SLiM tick counter when this mutation occurred.", - "index": 4, + "tick": { + "description": "The 'SLiM tick' counter when this tree sequence was recorded.", "type": "integer", }, - "subpopulation": { - "binaryFormat": "i", - "description": "The ID of the subpopulation this mutation occurred in.", - "index": 3, - "type": "integer", + "traits": { + "description": "The traits defined for this tree sequence; each mutation and individual will have per-trait metadata.", + "items": { + "properties": { + "baselineAccumulation": { + "description": "Whether the baseline offset includes accumulated effects from fixed (substituted) mutations.", + "type": "boolean", + }, + "baselineOffset": { + "description": "The baseline offset of the trait.", + "type": "number", + }, + "directFitnessEffect": { + "description": "Whether the trait's effects are used directly as fitness effects.", + "type": "boolean", + }, + "index": { + "description": "The integer index for the trait; indices must be sequential starting from zero.", + "type": "integer", + }, + "individualOffsetMean": { + "description": "The mean of the trait's individual offset distribution (which might or might not be used).", + "type": "number", + }, + "individualOffsetSD": { + "description": "The standard deviation of the trait's individual offset distribution (which might or might not be used).", + "type": "number", + }, + "name": { + "description": "The string name for the trait.", + "type": "string", + }, + "type": { + "description": "The type of the trait; this must be 'additive', 'multiplicative', or 'logistic'.", + "enum": [ + "additive", + "multiplicative", + "logistic", + ], + "type": "string", + }, + }, + "required": ["index", "name", "type"], + "type": "object", + }, + "type": "array", }, }, "required": [ - "mutation_type", - "selection_coeff", - "subpopulation", - "slim_time", - "nucleotide", + "model_type", + "tick", + "file_version", + "spatial_dimensionality", + "spatial_periodicity", + "this_chromosome", + "separate_sexes", + "nucleotide_based", + "traits", ], "type": "object", - }, - "noLengthEncodingExhaustBuffer": True, - "type": "array", - } + } + }, + "required": ["SLiM"], + "type": "object", + }, + "struct": { + "codec": "struct", + "description": "SLiM schema for binary top-level metadata.", + "properties": { + "SLiM_mutation_list": { + "arrayLengthFormat": "Q", + "items": { + "additionalProperties": False, + "properties": { + "mutation_id": { + "binaryFormat": "q", + "description": "The SLiM mutation ID for this mutation.", + "index": 1, + "type": "integer", + }, + "mutation_type": { + "binaryFormat": "i", + "description": "The id of this mutation's mutationType.", + "index": 2, + "type": "integer", + }, + "nucleotide": { + "binaryFormat": "b", + "description": "The nucleotide for this mutation (0=A , 1=C , 2=G, 3=T, or -1 for none)", + "index": 5, + "type": "integer", + }, + "padding": { + "binaryFormat": "3x", + "description": "Padding bytes for alignment", + "index": 6, + "type": "null", + }, + "per_trait": { + "index": 7, + "items": { + "additionalProperties": False, + "properties": { + "dominance": { + "binaryFormat": "f", + "description": "The dominance coefficient for this trait.", + "index": 2, + "type": "number", + }, + "effect_size": { + "binaryFormat": "f", + "description": "The effect size for this trait.", + "index": 1, + "type": "number", + }, + "hemizygous_dominance": { + "binaryFormat": "f", + "description": "The hemizygous dominance coefficient for this trait.", + "index": 3, + "type": "number", + }, + }, + "required": [ + "dominance", + "effect_size", + "hemizygous_dominance", + ], + "type": "object", + }, + "length": 1, # NOTE this may need to be changed to match the number of traits! + "type": "array", + }, + "slim_time": { + "binaryFormat": "i", + "description": "The SLiM tick counter when this mutation occurred.", + "index": 4, + "type": "integer", + }, + "subpopulation": { + "binaryFormat": "i", + "description": "The ID of the subpopulation this mutation occurred in.", + "index": 3, + "type": "integer", + }, + }, + "required": [ + "mutation_id", + "mutation_type", + "slim_time", + "subpopulation", + "nucleotide", + "per_trait", + ], + "type": "object", + }, + "type": "array", + } + }, + "required": ["SLiM_mutation_list"], + "type": "object", }, - "required": ["mutation_list"], - "type": "object", }, + "edge": None, + "site": None, + "mutation": None, "node": { "$schema": "http://json-schema.org/schema#", "additionalProperties": False, @@ -269,12 +353,12 @@ def is_vacant_num_bytes(num_chromosomes): "slim_id": { "binaryFormat": "q", "description": "The 'pedigree ID' of the haplosomes associated with this node in SLiM.", - "index": 0, + "index": 1, "type": "integer", }, "is_vacant": { "description": "A vector of byte (uint8_t) values, with each bit representing whether the node represents a vacant position, either unused or a null haplosome (1), or a non-null haplosome (0), in the corresponding chromosome. This field encodes vacancy for all of the chromosomes in the model, not just the chromosome represented in this file (so that the node table is identical across all chromosomes for a multi-chromosome model). Each chromosome receives one bit here; there are two node table entries per individual, used for the two haplosomes of every chromosome, so only one bit is needed in each entry (making two bits total per chromosome, across the two node table entries). The least significant bit of the first byte is used first (for one haplosome of the first chromosome); the most significant bit of the last byte is used last. The number of bytes present in this field is indicated by this schema's 'binaryFormat' field, which is variable (!), and can also be deduced from the number of chromosomes in the model as given in the top-level 'chromosomes' metadata key, which should always be present if this metadata is present.", - "index": 1, + "index": 2, "type": "array", "length": 1, # MAY NEED TO BE CHANGED (in SLiM code is "%d") "items": {"type": "number", "binaryFormat": "B"}, @@ -285,8 +369,8 @@ def is_vacant_num_bytes(num_chromosomes): }, "individual": { "$schema": "http://json-schema.org/schema#", - "additionalProperties": False, "codec": "struct", + "type": "object", "description": "SLiM schema for individual metadata.", "examples": [ { @@ -295,64 +379,170 @@ def is_vacant_num_bytes(num_chromosomes): "pedigree_id": 123, "pedigree_p1": 12, "pedigree_p2": 23, + "per_trait": [{"offset": 1.0, "phenotype": 1.1}], "sex": 0, "subpopulation": 0, + "tag": 1, + "tagF": 5.5, + "tagL0_set": True, + "tagL0": True, + "tagL1_set": True, + "tagL1": False, + "tagL2_set": False, + "tagL2": False, + "tagL3_set": False, + "tagL3": False, + "tagL4_set": False, + "tagL4": False, } ], "flags": { "SLIM_INDIVIDUAL_METADATA_MIGRATED": { - "description": "Whether this individual was a migrant, either in the tick when the tree sequence " - "was written out (if the individual was alive then), or in the tick of the last time " - "they were Remembered (if not).", + "description": "Whether this individual was a migrant, either in the tick when the tree sequence was written out (if the individual was alive then), or in the tick of the last time they were Remembered (if not).", "value": 1, } }, "properties": { - "age": { - "binaryFormat": "i", - "description": "The age of this individual, either when the tree sequence was written out " - "(if the individual was alive then), or the last time they were Remembered (if not).", - "index": 4, - "type": "integer", - }, - "flags": { - "binaryFormat": "I", - "description": "Other information about the individual: see 'flags'.", - "index": 7, - "type": "integer", - }, "pedigree_id": { - "binaryFormat": "q", - "description": "The 'pedigree ID' of this individual in SLiM.", "index": 1, "type": "integer", + "binaryFormat": "q", + "description": "The 'pedigree ID' of this individual in SLiM.", }, "pedigree_p1": { - "binaryFormat": "q", - "description": "The 'pedigree ID' of this individual's first parent in SLiM.", "index": 2, "type": "integer", + "binaryFormat": "q", + "description": "The 'pedigree ID' of this individual's first parent in SLiM.", }, "pedigree_p2": { + "index": 3, + "type": "integer", "binaryFormat": "q", "description": "The 'pedigree ID' of this individual's second parent in SLiM.", - "index": 3, + }, + "age": { + "index": 4, "type": "integer", + "binaryFormat": "i", + "description": "The age of this individual, either when the tree sequence was written out (if the individual was alive then), or the last time they were Remembered (if not).", + }, + "subpopulation": { + "index": 5, + "type": "integer", + "binaryFormat": "i", + "description": "The ID of the subpopulation the individual was part of, either when the tree sequence was written out (if the individual was alive then), or the last time they were Remembered (if not).", }, "sex": { + "index": 6, + "type": "integer", "binaryFormat": "i", "description": "The sex of the individual (0 for female, 1 for male, -1 for hermaphrodite).", - "index": 6, + }, + "flags": { + "index": 7, "type": "integer", + "binaryFormat": "I", + "description": "Other information about the individual: see 'flags'.", }, - "subpopulation": { - "binaryFormat": "i", - "description": "The ID of the subpopulation the individual was part of, either when the tree sequence " - "was written out (if the individual was alive then), or the last time they were Remembered (if not).", - "index": 5, + "tag": { + "index": 8, "type": "integer", + "binaryFormat": "q", + "description": "The `tag` property of this individual; INT64_MIN if unset.", + }, + "tagF": { + "index": 9, + "type": "number", + "binaryFormat": "d", + "description": "The `tagF` property of this individual; -DBL_MAX if unset.", + }, + "tagL0_set": { + "index": 10, + "type": "boolean", + "binaryFormat": "?", + "description": "A flag indicating whether the `tagL0` property is set; if false, accessing `tagL0` is invalid.", + }, + "tagL0": { + "index": 11, + "type": "boolean", + "binaryFormat": "?", + "description": "The `tagL0` property of this individual; only valid if `tagL0_set` is true.", + }, + "tagL1_set": { + "index": 12, + "type": "boolean", + "binaryFormat": "?", + "description": "A flag indicating whether the `tagL1` property is set; if false, accessing `tagL1` is invalid.", + }, + "tagL1": { + "index": 13, + "type": "boolean", + "binaryFormat": "?", + "description": "The `tagL1` property of this individual; only valid if `tagL1_set` is true.", + }, + "tagL2_set": { + "index": 14, + "type": "boolean", + "binaryFormat": "?", + "description": "A flag indicating whether the `tagL2` property is set; if false, accessing `tagL2` is invalid.", + }, + "tagL2": { + "index": 15, + "type": "boolean", + "binaryFormat": "?", + "description": "The `tagL2` property of this individual; only valid if `tagL2_set` is true.", + }, + "tagL3_set": { + "index": 16, + "type": "boolean", + "binaryFormat": "?", + "description": "A flag indicating whether the `tagL3` property is set; if false, accessing `tagL3` is invalid.", + }, + "tagL3": { + "index": 17, + "type": "boolean", + "binaryFormat": "?", + "description": "The `tagL3` property of this individual; only valid if `tagL3_set` is true.", + }, + "tagL4_set": { + "index": 18, + "type": "boolean", + "binaryFormat": "?", + "description": "A flag indicating whether the `tagL4` property is set; if false, accessing `tagL4` is invalid.", + }, + "tagL4": { + "index": 19, + "type": "boolean", + "binaryFormat": "?", + "description": "The `tagL4` property of this individual; only valid if `tagL4_set` is true.", + }, + "per_trait": { + "index": 20, + "type": "array", + "length": 1, # MAY NEED TO BE CHANGED (in SLiM code is "%d") + "items": { + "additionalProperties": False, + "properties": { + "phenotype": { + "index": 1, + "type": "number", + "binaryFormat": "d", + "description": "The phenotype for this trait.", + }, + "offset": { + "index": 2, + "type": "number", + "binaryFormat": "d", + "description": "The individual offset for this trait.", + }, + }, + "required": ["offset", "phenotype"], + "type": "object", + }, }, }, + "additionalProperties": False, "required": [ "pedigree_id", "pedigree_p1", @@ -360,9 +550,21 @@ def is_vacant_num_bytes(num_chromosomes): "age", "subpopulation", "sex", + "tag", + "tagF", + "tagL0_set", + "tagL0", + "tagL1_set", + "tagL1", + "tagL2_set", + "tagL2", + "tagL3_set", + "tagL3", + "tagL4_set", + "tagL4", "flags", + "per_trait", ], - "type": "object", }, "population": { "$schema": "http://json-schema.org/schema#", @@ -464,6 +666,45 @@ def is_vacant_num_bytes(num_chromosomes): } +def slim_tree_sequence_metadata_schema(num_traits=1): + """ + The top-level metadata schema depends on the number of traits, and + {data}`.slim_metadata_schemas` + returns the schema for a single-trait simulation. This function + returns the correct schema for a simulation with arbitrary number of + traits. (The resulting schemas only differ in the + "length" of the "per_trait" property of + ``schema["properties"]["SLiM_mutation_list"]["items"]``). + + :param int num_traits: The number of traits in the model. + :return tskit.MetadataSchema: The metadata schema to be used + in the node table. + """ + schema = _raw_slim_metadata_schemas["tree_sequence"] + schema["struct"]["properties"]["SLiM_mutation_list"]["items"]["properties"][ + "per_trait" + ]["length"] = num_traits + return tskit.MetadataSchema(schema) + + +def slim_individual_metadata_schema(num_traits=1): + """ + The individual metadata schema depends on the number of traits, and + {data}`.slim_metadata_schemas` + returns the schema for a single-trait simulation. This function + returns the correct schema for a simulation with arbitrary number of + traits. (The resulting schemas only differ in the + "length" of the "per_trait" property.) + + :param int num_traits: The number of traits in the model. + :return tskit.MetadataSchema: The metadata schema to be used + in the node table. + """ + schema = _raw_slim_metadata_schemas["individual"] + schema["properties"]["per_trait"]["length"] = num_traits + return tskit.MetadataSchema(schema) + + def slim_node_metadata_schema(num_chromosomes=1): """ Unlike other schema, the node metadata schema depends on the number of @@ -510,12 +751,15 @@ def slim_node_metadata_schema(num_chromosomes=1): """ -def default_slim_metadata(name, num_chromosomes=1): +def default_slim_metadata(name, num_chromosomes=1, num_traits=1, **kwargs): """ Returns default metadata of type ``name``, where ``name`` is one of "tree_sequence", "edge", "site", "mutation", "mutation_list_entry", "node", "individual", or "population". + Additional kwargs are used to update the resulting metadata + (without validity checking). + :param str name: The type of metadata requested. :rtype dict: """ @@ -540,21 +784,26 @@ def default_slim_metadata(name, num_chromosomes=1): "type": "A", }, "chromosomes": [{"id": 1, "index": 0, "symbol": "A", "type": "A"}], - } + "traits": [{"index": 0, "name": "simT", "type": "multiplicative"}], + }, + "SLiM_mutation_list": [], } elif name == "edge": out = None elif name == "site": out = None elif name == "mutation": - out = {"mutation_list": []} + out = None elif name == "mutation_list_entry": out = { + "mutation_id": 0, "mutation_type": 0, - "selection_coeff": 0.0, "subpopulation": tskit.NULL, "slim_time": 0, "nucleotide": -1, + "per_trait": num_traits + * [{"effect_size": 0.0, "dominance": 0.5, "hemizygous_dominance": 1.0}], + "padding": None, } elif name == "node": out = { @@ -570,6 +819,19 @@ def default_slim_metadata(name, num_chromosomes=1): "flags": 0, "pedigree_p1": tskit.NULL, "pedigree_p2": tskit.NULL, + "tag": np.iinfo(np.int64).min, + "tagF": np.finfo(np.float64).min, + "tagL0_set": False, + "tagL0": False, + "tagL1_set": False, + "tagL1": False, + "tagL2_set": False, + "tagL2": False, + "tagL3_set": False, + "tagL3": False, + "tagL4_set": False, + "tagL4": False, + "per_trait": num_traits * [{"phenotype": np.nan, "offset": 1.0}], } elif name == "population": out = { @@ -594,6 +856,8 @@ def default_slim_metadata(name, num_chromosomes=1): "'edge', 'site', 'mutation', 'mutation_list_entry', 'node', " "'individual', or 'population'." ) + if out is not None: + out.update(kwargs) return out @@ -606,6 +870,7 @@ def set_tree_sequence_metadata( tables, model_type, tick, + *, cycle=None, spatial_dimensionality="", spatial_periodicity="", @@ -618,35 +883,58 @@ def set_tree_sequence_metadata( chromosomes=None, file_version=None, set_table_schemas=True, + traits=None, + SLiM_mutation_list=None, ): if file_version is None: file_version = slim_file_version - if isinstance(tables.metadata, bytes): - if len(tables.metadata) > 0: + if traits is None: + traits = [{"index": 0, "name": "simT", "type": "multiplicative"}] + num_traits = len(traits) + schema_dict = slim_tree_sequence_metadata_schema(num_traits).schema + old_schema_dict = tables.metadata_schema.schema + old_json_schema_dict = {} + old_struct_schema_dict = {} + tmd = tables.metadata + if isinstance(tmd, bytes): + if len(tmd) > 0: raise ValueError( "Tree sequence has top-level metadata but no schema: this is a problem " "since pyslim is trying to add to the metadata." ) - schema_dict = slim_metadata_schemas["tree_sequence"].schema metadata_dict = {} else: # we need to keep other keys in the metadata (and schema) if there are any - schema_dict = tables.metadata_schema.schema metadata_dict = tables.metadata + if old_schema_dict["codec"] == "json": + old_json_schema_dict = tables.metadata_schema.schema + else: + assert old_schema_dict["codec"] == "json+struct", ( + "You are using an unexpected codec; " + "please raise an issue on pyslim if " + "you need this functionality." + ) + old_json_schema_dict = tables.metadata_schema.schema["json"] + old_struct_schema_dict = tables.metadata_schema.schema["struct"] if cycle is None: cycle = tick - defaults = default_slim_metadata("tree_sequence") + if chromosomes is None: + num_chromosomes = 1 + else: + num_chromosomes = len(chromosomes) + defaults = default_slim_metadata( + "tree_sequence", num_chromosomes=num_chromosomes, num_traits=num_traits + ) if this_chromosome is None: this_chromosome = defaults["SLiM"]["this_chromosome"] if chromosomes is None: chromosomes = defaults["SLiM"]["chromosomes"] - assert schema_dict["codec"] == "json" - assert schema_dict["type"] == "object" - if "properties" not in schema_dict: - schema_dict["properties"] = {} - schema_dict["properties"]["SLiM"] = slim_metadata_schemas["tree_sequence"].schema[ - "properties" - ]["SLiM"] + if "properties" in old_json_schema_dict: + schema_dict["json"]["properties"].update(old_json_schema_dict["properties"]) + if "properties" in old_struct_schema_dict: + schema_dict["struct"]["properties"].update(old_struct_schema_dict["properties"]) + if SLiM_mutation_list is None: + SLiM_mutation_list = [] tables.metadata_schema = tskit.MetadataSchema(schema_dict) metadata_dict["SLiM"] = { "model_type": model_type, @@ -662,16 +950,19 @@ def set_tree_sequence_metadata( "description": description, "this_chromosome": this_chromosome, "chromosomes": chromosomes, + "traits": traits, } + metadata_dict["SLiM_mutation_list"] = SLiM_mutation_list tables.metadata = metadata_dict + return metadata_dict -def set_metadata_schemas(tables, num_chromosomes=1): +def set_metadata_schemas(tables, num_chromosomes=1, num_traits=1): tables.edges.metadata_schema = slim_metadata_schemas["edge"] tables.sites.metadata_schema = slim_metadata_schemas["site"] tables.mutations.metadata_schema = slim_metadata_schemas["mutation"] tables.nodes.metadata_schema = slim_node_metadata_schema(num_chromosomes) - tables.individuals.metadata_schema = slim_metadata_schemas["individual"] + tables.individuals.metadata_schema = slim_individual_metadata_schema(num_traits) tables.populations.metadata_schema = slim_metadata_schemas["population"] @@ -683,6 +974,161 @@ def _old_metadata_schema(name, file_version): # Returns a metadata schema *if the format has changed*, # and None otherwise. ms = None + if name == "tree_sequence" and file_version == "0.9": + pre_1_0_tree_sequence = { + "$schema": "http://json-schema.org/schema#", + "codec": "json", + "examples": [ + { + "SLiM": { + "file_version": "0.9", + "name": "fox", + "description": "foxes on Catalina island", + "cycle": 123, + "tick": 123, + "model_type": "WF", + "this_chromosome": { + "id": 1, + "index": 0, + "symbol": "1", + "name": "autosome_1", + "type": "A", + }, + "chromosomes": [ + {"id": 1, "symbol": "1", "name": "autosome_1", "type": "A"}, + {"id": 35, "symbol": "MT", "name": "mtDNA", "type": "HF"}, + ], + "nucleotide_based": False, + "separate_sexes": True, + "spatial_dimensionality": "xy", + "spatial_periodicity": "x", + } + } + ], + "properties": { + "SLiM": { + "description": "Top-level metadata for a SLiM tree sequence, file format version 0.9", + "properties": { + "file_version": { + "description": "The SLiM 'file format version' of this tree sequence.", + "type": "string", + }, + "name": { + "description": "The SLiM species name represented by this tree sequence.", + "type": "string", + }, + "description": { + "description": "A user-configurable description of the species represented by this tree sequence.", + "type": "string", + }, + "cycle": { + "description": "The 'SLiM cycle' counter when this tree sequence was recorded.", + "type": "integer", + }, + "tick": { + "description": "The 'SLiM tick' counter when this tree sequence was recorded.", + "type": "integer", + }, + "model_type": { + "description": "The model type used for the last part of this simulation (WF or nonWF).", + "enum": ["WF", "nonWF"], + "type": "string", + }, + "this_chromosome": { + "description": "The chromosome represented by the tree sequence in this file.", + "properties": { + "id": { + "description": "An integer identifier for the chromosome, unique within this set of tree sequences; often the chromosome number in the organism being represented, such as 1.", + "type": "integer", + }, + "index": { + "description": "The (zero-based) index of this chromosome in the chromosomes metadata array (if present), which should match the information given here.", + "type": "integer", + }, + "symbol": { + "description": 'A short string symbol for the chromosome, unique within this set of tree sequences, such as "1" or "MT".', + "type": "string", + }, + "name": { + "description": "A user-specified name for the chromosome, such as an accession identifier.", + "type": "string", + }, + "type": { + "description": "The type of chromosome, as specified by SLiM.", + "type": "string", + }, + }, + "required": ["id", "index", "symbol", "type"], + "type": "object", + }, + "chromosomes": { + "description": "The chromosomes represented by the collection of tree sequences, of which this tree sequence is one member.", + "items": { + "properties": { + "id": { + "description": "An integer identifier for the chromosome, unique within this set of tree sequences; often the chromosome number in the organism being represented, such as 1.", + "type": "integer", + }, + "symbol": { + "description": 'A short string symbol for the chromosome, unique within this set of tree sequences, such as "1" or "MT".', + "type": "string", + }, + "name": { + "description": "A user-specified name for the chromosome, such as an accession identifier.", + "type": "string", + }, + "type": { + "description": "The type of chromosome, as specified by SLiM.", + "type": "string", + }, + }, + "required": ["id", "symbol", "type"], + "type": "object", + }, + "type": "array", + }, + "nucleotide_based": { + "description": "Whether the simulation was nucleotide-based.", + "type": "boolean", + }, + "separate_sexes": { + "description": "Whether the simulation had separate sexes.", + "type": "boolean", + }, + "spatial_dimensionality": { + "description": "The spatial dimensionality of the simulation.", + "enum": ["", "x", "xy", "xyz"], + "type": "string", + }, + "spatial_periodicity": { + "description": "The spatial periodicity of the simulation.", + "enum": ["", "x", "y", "z", "xy", "xz", "yz", "xyz"], + "type": "string", + }, + "stage": { + "description": "The stage of the SLiM life cycle when this tree sequence was recorded.", + "type": "string", + }, + }, + "required": [ + "model_type", + "tick", + "file_version", + "spatial_dimensionality", + "spatial_periodicity", + "this_chromosome", + "separate_sexes", + "nucleotide_based", + ], + "type": "object", + } + }, + "required": ["SLiM"], + "type": "object", + } + + ms = pre_1_0_tree_sequence + if name == "tree_sequence" and file_version == "0.8": pre_0_9_tree_sequence = { "$schema": "http://json-schema.org/schema#", @@ -963,6 +1409,90 @@ def _old_metadata_schema(name, file_version): } ms = pre_0_7_population + if name == "individual" and file_version in ["0.7", "0.8", "0.9"]: + pre_1_0_individual = { + "$schema": "http://json-schema.org/schema#", + "additionalProperties": False, + "codec": "struct", + "description": "SLiM schema for individual metadata.", + "examples": [ + { + "age": -1, + "flags": 0, + "pedigree_id": 123, + "pedigree_p1": 12, + "pedigree_p2": 23, + "sex": 0, + "subpopulation": 0, + } + ], + "flags": { + "SLIM_INDIVIDUAL_METADATA_MIGRATED": { + "description": "Whether this individual was a migrant, either in the tick when the tree sequence " + "was written out (if the individual was alive then), or in the tick of the last time " + "they were Remembered (if not).", + "value": 1, + } + }, + "properties": { + "age": { + "binaryFormat": "i", + "description": "The age of this individual, either when the tree sequence was written out " + "(if the individual was alive then), or the last time they were Remembered (if not).", + "index": 4, + "type": "integer", + }, + "flags": { + "binaryFormat": "I", + "description": "Other information about the individual: see 'flags'.", + "index": 7, + "type": "integer", + }, + "pedigree_id": { + "binaryFormat": "q", + "description": "The 'pedigree ID' of this individual in SLiM.", + "index": 1, + "type": "integer", + }, + "pedigree_p1": { + "binaryFormat": "q", + "description": "The 'pedigree ID' of this individual's first parent in SLiM.", + "index": 2, + "type": "integer", + }, + "pedigree_p2": { + "binaryFormat": "q", + "description": "The 'pedigree ID' of this individual's second parent in SLiM.", + "index": 3, + "type": "integer", + }, + "sex": { + "binaryFormat": "i", + "description": "The sex of the individual (0 for female, 1 for male, -1 for hermaphrodite).", + "index": 6, + "type": "integer", + }, + "subpopulation": { + "binaryFormat": "i", + "description": "The ID of the subpopulation the individual was part of, either when the tree sequence " + "was written out (if the individual was alive then), or the last time they were Remembered (if not).", + "index": 5, + "type": "integer", + }, + }, + "required": [ + "pedigree_id", + "pedigree_p1", + "pedigree_p2", + "age", + "subpopulation", + "sex", + "flags", + ], + "type": "object", + } + ms = pre_1_0_individual + if name == "individual" and file_version in [ "0.1", "0.2", @@ -1019,6 +1549,87 @@ def _old_metadata_schema(name, file_version): } ms = pre_0_7_individual + if name == "mutation" and file_version in [ + "0.3", + "0.4", + "0.5", + "0.6", + "0.7", + "0.8", + "0.9", + ]: + mutation_pre_1_0 = { + "$schema": "http://json-schema.org/schema#", + "additionalProperties": False, + "codec": "struct", + "description": "SLiM schema for mutation metadata.", + "examples": [ + { + "mutation_list": [ + { + "mutation_type": 1, + "nucleotide": 3, + "selection_coeff": -0.2, + "slim_time": 243, + "subpopulation": 0, + } + ] + } + ], + "properties": { + "mutation_list": { + "items": { + "additionalProperties": False, + "properties": { + "mutation_type": { + "binaryFormat": "i", + "description": "The index of this mutation's mutationType.", + "index": 1, + "type": "integer", + }, + "nucleotide": { + "binaryFormat": "b", + "description": "The nucleotide for this mutation (0=A , 1=C , 2=G, 3=T, or -1 for none)", + "index": 5, + "type": "integer", + }, + "selection_coeff": { + "binaryFormat": "f", + "description": "This mutation's selection coefficient.", + "index": 2, + "type": "number", + }, + "slim_time": { + "binaryFormat": "i", + "description": "The SLiM tick counter when this mutation occurred.", + "index": 4, + "type": "integer", + }, + "subpopulation": { + "binaryFormat": "i", + "description": "The ID of the subpopulation this mutation occurred in.", + "index": 3, + "type": "integer", + }, + }, + "required": [ + "mutation_type", + "selection_coeff", + "subpopulation", + "slim_time", + "nucleotide", + ], + "type": "object", + }, + "noLengthEncodingExhaustBuffer": True, + "type": "array", + } + }, + "required": ["mutation_list"], + "type": "object", + } + ms = mutation_pre_1_0 + if name == "mutation" and file_version in ["0.1", "0.2"]: mutation_pre_0_3 = { "$schema": "http://json-schema.org/schema#", @@ -1072,6 +1683,33 @@ def _old_metadata_schema(name, file_version): } ms = mutation_pre_0_3 + if name == "node" and file_version == "0.9": + node_0_9 = { + "$schema": "http://json-schema.org/schema#", + "additionalProperties": False, + "codec": "struct", + "description": "SLiM schema for node metadata.", + "examples": [{"slim_id": 123, "is_vacant": 0}], + "properties": { + "slim_id": { + "binaryFormat": "q", + "description": "The 'pedigree ID' of the haplosomes associated with this node in SLiM.", + "index": 0, + "type": "integer", + }, + "is_vacant": { + "description": "A vector of byte (uint8_t) values, with each bit representing whether the node represents a vacant position, either unused or a null haplosome (1), or a non-null haplosome (0), in the corresponding chromosome. This field encodes vacancy for all of the chromosomes in the model, not just the chromosome represented in this file (so that the node table is identical across all chromosomes for a multi-chromosome model). Each chromosome receives one bit here; there are two node table entries per individual, used for the two haplosomes of every chromosome, so only one bit is needed in each entry (making two bits total per chromosome, across the two node table entries). The least significant bit of the first byte is used first (for one haplosome of the first chromosome); the most significant bit of the last byte is used last. The number of bytes present in this field is indicated by this schema's 'binaryFormat' field, which is variable (!), and can also be deduced from the number of chromosomes in the model as given in the top-level 'chromosomes' metadata key, which should always be present if this metadata is present.", + "index": 1, + "type": "array", + "length": 1, # MAY NEED TO BE CHANGED (in SLiM code is "%d") + "items": {"type": "number", "binaryFormat": "B"}, + }, + }, + "required": ["slim_id", "is_vacant"], + "type": ["object", "null"], + } + ms = node_0_9 + if name == "node" and file_version in [ "0.1", "0.2", @@ -1119,24 +1757,66 @@ def _old_metadata_schema(name, file_version): return ms +def _make_mutation_list(mutations, file_version): + # Prior to 1.0, mutation metadata was a list of entries like: + # {'mutation_type': 1, 'selection_coeff': -0.1, + # 'subpopulation': 1, 'slim_time': 5, 'nucleotide': -1} + # with mutation id stored in the derived state. + # + # As of 1.0, this lives in the top-level ts.metadata['SLiM_mutation_list'], + # with entries like + # {'mutation_id': 87, 'mutation_type': 1, 'subpopulation': 1, 'slim_time': 5, + # 'nucleotide': -1, 'padding': None, + # 'per_trait': [{'effect_size': -0.1, 'dominance': 0.5, + # 'hemizygous_dominance': 1.0}]} + if mutations.metadata_schema == tskit.MetadataSchema(None): + mutations.metadata_schema = _old_metadata_schema("mutation", file_version) + mutation_list = [] + for mut in mutations: + for sid, md in zip(mut.derived_state.split(","), mut.metadata["mutation_list"]): + if "nucleotide" not in md: + md["nucleotide"] = -1 + md["mutation_id"] = int(sid) + md["per_trait"] = [ + { + "effect_size": md["selection_coeff"], + "dominance": 0.5, # WE DON'T KNOW THIS + "hemizygous_dominance": 1.0, # OR THIS + }, + ] + del md["selection_coeff"] + md["padding"] = None + mutation_list.append(md) + return mutation_list + + def is_current_version(ts, _warn=False): """ - Tests whether the tree sequence or table collection provided is the current - SLiM file format or not. If not, use `pyslim.update( )` to bring it up to - date. + Tests whether the metadata provided is the current SLiM file format or not. + If not, use `pyslim.update( )` to bring it up to date. - :param TreeSequence ts: The tree sequence or table collection. + This method may be provided either a TreeSequence or TableCollection directly, + or the metadata from one of these. The latter is useful because + accessing top-level metadata can be a costly operation. + + :param dict ts: Either the top-level metadata of a tree sequence, + or a TreeSequence or TableCollection that carries this metadata. :return bool: Whether the tree sequence is the current version. """ + if ( + isinstance(ts, tskit.TreeSequence) + or isinstance(ts, tskit.TableCollection) + or isinstance(ts, tskit.ImmutableTableCollection) + ): + ts = ts.metadata out = ( - isinstance(ts.metadata, dict) - and ("SLiM" in ts.metadata) - and (ts.metadata["SLiM"]["file_version"] == slim_file_version) + isinstance(ts, dict) + and ("SLiM" in ts) + and (ts["SLiM"]["file_version"] == slim_file_version) ) if _warn and not out: warnings.warn( - "This tree sequence is not the current SLiM format, " - "so some operations may not work. " + "This tree sequence is not the current SLiM format. " "Use `pyslim.update( )` to update the tree sequence." ) return out @@ -1161,10 +1841,13 @@ def update_tables(tables): """ # First we ensure we can find the file format version number # in top-level metadata. Then we proceed to fix up the tables as necessary. - if not (isinstance(tables.metadata, dict) and "SLiM" in tables.metadata): + md = tables.metadata + if not (isinstance(md, dict) and "SLiM" in md): # Old versions kept information in provenance, not top-level metadata. # Note this uses defaults on keys not present in provenance, # which prior to 0.5 was everything but generation and model_type. + # Recovering from provenance has also been useful for operations + # that discard metadata (eg as msprime did prior to 0.7.5). values = default_slim_metadata("tree_sequence")["SLiM"] prov = None file_version = "unknown" @@ -1192,25 +1875,38 @@ def update_tables(tables): values[k] = record["slim"][k] except: raise ValueError("Failed to obtain metadata from provenance.") - set_tree_sequence_metadata(tables, **values) + md = set_tree_sequence_metadata(tables, **values) - file_version = tables.metadata["SLiM"]["file_version"] + file_version = md["SLiM"]["file_version"] if file_version != slim_file_version: warnings.warn( - "This is a version {} SLiM tree sequence.".format(file_version) - + " If you write this out to a file, " - + "it will be converted to version {}.".format(slim_file_version) + f"This is a version {file_version} SLiM tree sequence. " + "If you write this out to a file, " + f"it will be converted to version {slim_file_version}." ) - # the only tables to have metadata schema changed thus far - # are nodes, populations, individuals, mutations, and top-level: old_schema = _old_metadata_schema("tree_sequence", file_version) if old_schema is not None: - md = tables.metadata - new_schema = slim_metadata_schemas["tree_sequence"] - new_properties = new_schema.asdict()["properties"]["SLiM"]["required"] + assert ( + "struct" not in old_schema.schema + or "SLiM_mutation_list" not in old_schema.schema["struct"]["properties"] + ) + # we should get the number of traits from the metadata, + # but for old file versions, this won't be present + assert ( + "json" not in old_schema.schema + or "traits" not in old_schema.schema["json"]["properties"] + ) + md["SLiM_mutation_list"] = _make_mutation_list( + tables.mutations, file_version + ) + num_traits = 1 + new_schema = slim_tree_sequence_metadata_schema(num_traits=num_traits) + new_properties = new_schema.asdict()["json"]["properties"]["SLiM"][ + "required" + ] tables.metadata_schema = new_schema - defaults = default_slim_metadata("tree_sequence") + defaults = default_slim_metadata("tree_sequence", num_traits=num_traits) for k in new_properties: if k not in md["SLiM"]: if k == "tick": @@ -1226,63 +1922,71 @@ def update_tables(tables): tables.nodes.clear() if nodes.metadata_schema == tskit.MetadataSchema(None): nodes.metadata_schema = old_schema - assert "chromosomes" not in tables.metadata - # if chromosomes was in metadata - # we should use its length to get num_chroms, - # but old file versions did not have this. - num_chroms = 1 + if "chromosomes" not in md["SLiM"]: + num_chroms = 1 + else: + num_chroms = len(md["SLiM"]["chromosomes"]) new_schema = slim_node_metadata_schema(num_chroms) tables.nodes.metadata_schema = new_schema - not_vacant = [0] # single chromosome - yes_vacant = [1] - gt = None - for n in nodes: - md = n.metadata - if len(md) > 0: - md["is_vacant"] = yes_vacant if md["is_null"] else not_vacant - if not md["is_null"]: - if gt is None: - gt = md["genome_type"] - else: - assert md["genome_type"] == gt, ( - "Inconsistent tables: " - f"mismatching genome types {gt} and " - f"{md['genome_type']} in node metadata." - ) - del md["is_null"] - del md["genome_type"] - tables.nodes.append(n.replace(metadata=md)) - # flags for node genome type pre-0.9: - # confusingly and sub-optimally, these were redundant: - # all non-null nodes in the same sim would have the same genome type - assert gt is not None - GENOME_TYPE_AUTOSOME = 0 - GENOME_TYPE_X = 1 - GENOME_TYPE_Y = 2 - top_md = tables.metadata - i = top_md["SLiM"]["this_chromosome"]["index"] - # 'chromosomes' is not required so won't be inserted, - # so we won't update it also - assert "chromosomes" not in top_md["SLiM"]["this_chromosome"], ( - "" - "This is an unexpected result: if you hit this, " - "please file a bug at " - "https://github.com/tskit-dev/pyslim." - ) - if gt == GENOME_TYPE_X: - top_md["SLiM"]["this_chromosome"]["type"] = "X" - top_md["SLiM"]["this_chromosome"]["symbol"] = "X" - # if "chromosomes" in tables.metadata['SLiM']: - # top_md['SLiM']['chromosomes'][i]['type'] = "X" - # top_md['SLiM']['chromosomes'][i]['symbol'] = "X" - elif gt == GENOME_TYPE_Y: - top_md["SLiM"]["this_chromosome"]["type"] = "-Y" - top_md["SLiM"]["this_chromosome"]["symbol"] = "Y" - # top_md['SLiM']['chromosomes'][i]['type'] = "Y" - # top_md['SLiM']['chromosomes'][i]['symbol'] = "Y" + new_node_schema = new_schema + if file_version in ("0.1", "0.2", "0.3", "0.4", "0.5", "0.6", "0.7", "0.8"): + # 0.8->0.9 switched from is_null to is_vacant, + # and moved chromosome type from node metadata to top-level + not_vacant = [0] # single chromosome + yes_vacant = [1] + gt = None + for n in nodes: + md = n.metadata + if len(md) > 0: + md["is_vacant"] = yes_vacant if md["is_null"] else not_vacant + if not md["is_null"]: + if gt is None: + gt = md["genome_type"] + else: + assert md["genome_type"] == gt, ( + "Inconsistent tables: " + f"mismatching genome types {gt} and " + f"{md['genome_type']} in node metadata." + ) + del md["is_null"] + del md["genome_type"] + tables.nodes.append(n.replace(metadata=md)) + # flags for node genome type pre-0.9: + # confusingly and sub-optimally, these were redundant: + # all non-null nodes in the same sim would have the same genome type + GENOME_TYPE_AUTOSOME = 0 + GENOME_TYPE_X = 1 + GENOME_TYPE_Y = 2 + top_md = tables.metadata + i = top_md["SLiM"]["this_chromosome"]["index"] + # 'chromosomes' is not required so won't be inserted, + # so we won't update it also + assert "chromosomes" not in top_md["SLiM"]["this_chromosome"], ( + "" + "This is an unexpected result: if you hit this, " + "please file a bug at " + "https://github.com/tskit-dev/pyslim." + ) + if gt == GENOME_TYPE_X: + top_md["SLiM"]["this_chromosome"]["type"] = "X" + top_md["SLiM"]["this_chromosome"]["symbol"] = "X" + # if "chromosomes" in tables.metadata['SLiM']: + # top_md['SLiM']['chromosomes'][i]['type'] = "X" + # top_md['SLiM']['chromosomes'][i]['symbol'] = "X" + elif gt == GENOME_TYPE_Y: + top_md["SLiM"]["this_chromosome"]["type"] = "-Y" + top_md["SLiM"]["this_chromosome"]["symbol"] = "Y" + # top_md['SLiM']['chromosomes'][i]['type'] = "Y" + # top_md['SLiM']['chromosomes'][i]['symbol'] = "Y" + else: + assert gt == GENOME_TYPE_AUTOSOME + tables.metadata = top_md else: - assert gt == GENOME_TYPE_AUTOSOME - tables.metadata = top_md + assert file_version == "0.9" + # just needs recoding (and doesn't really need that, we just + # changed the index of some entries in the schema) + for n in nodes: + tables.nodes.append(n) old_schema = _old_metadata_schema("population", file_version) if old_schema is not None: @@ -1302,15 +2006,33 @@ def update_tables(tables): tables.individuals.clear() if inds.metadata_schema == tskit.MetadataSchema(None): inds.metadata_schema = old_schema - new_schema = slim_metadata_schemas["individual"] + num_traits = len(tables.metadata["SLiM"]["traits"]) + new_schema = slim_individual_metadata_schema(num_traits=num_traits) tables.individuals.metadata_schema = new_schema - defaults = default_slim_metadata("individual") - d = {} - for k in ["pedigree_p1", "pedigree_p2"]: - d[k] = defaults[k] + # new(er) additions are pedigree_pX in 0.7 + # and per_trait in 1.0 + defaults = default_slim_metadata("individual", num_traits=num_traits) for ind in inds: md = ind.metadata - md.update(d) + for k in [ + "pedigree_p1", + "pedigree_p2", + "tag", + "tagF", + "tagL0", + "tagL0_set", + "tagL1", + "tagL1_set", + "tagL2", + "tagL2_set", + "tagL3", + "tagL3_set", + "tagL4", + "tagL4_set", + ]: + md.setdefault(k, defaults[k]) + if "per_trait" not in md: + md["per_trait"] = copy.deepcopy(defaults["per_trait"]) tables.individuals.append(ind.replace(metadata=md)) old_schema = _old_metadata_schema("mutation", file_version) @@ -1321,10 +2043,8 @@ def update_tables(tables): muts.metadata_schema = old_schema tables.mutations.metadata_schema = slim_metadata_schemas["mutation"] for mut in muts: - md = mut.metadata - for ml in md["mutation_list"]: - ml["nucleotide"] = -1 - tables.mutations.append(mut.replace(metadata=md)) + # drop metadata: it should have been copied into top-level above + tables.mutations.append(mut.replace(metadata=None)) if file_version == "0.1": # shift times @@ -1364,7 +2084,6 @@ def update_tables(tables): tskit.validate_provenance(new_record) tables.provenances.add_row(json.dumps(new_record)) - set_metadata_schemas(tables) md = tables.metadata md["SLiM"]["file_version"] = slim_file_version tables.metadata = md diff --git a/pyslim/slim_tree_sequence.py b/pyslim/slim_tree_sequence.py index 95de0333..f6bc9c0a 100644 --- a/pyslim/slim_tree_sequence.py +++ b/pyslim/slim_tree_sequence.py @@ -10,6 +10,45 @@ def load(*args, **kwargs): raise RuntimeError("This method has been removed: use tskit.load( ) instead.") +def mutation_metadata(ts, check=True, _ts_metadata=None): + """ + Returns a dictionary whose keys are the numeric SLiM IDs of mutations, + and whose values are metadata entries for those mutations. + *Note:* this is indexed by integers, not strings, so if you obtain SLiM IDs + from something like ``mut.derived_state.split(",")``, you must convert + the result to integers before looking up metadata! + + This is a simple extraction function that places the list of metadata entries + stored in ``ts.metadata["SLiM_mutation_list"]`` in a dictionary + indexed by SLiM ID. It is recommended to extract this information once + and use the result in script, because calling this function many times + (or, even just referring to ``ts.metadata`` many times) + can slow down scripts considerably. + + :param tskit.TreeSequence ts: The tree sequence. + + :returns dict: A dictionary of metadata entries, indexed by SLiM ID + and in sorted order by SLiM ID. + """ + if _ts_metadata is None: + _ts_metadata = ts.metadata + # Note that dictionaries preserve insertion order + ml = _ts_metadata["SLiM_mutation_list"] + ml.sort(key=lambda x: x["mutation_id"]) + out = {mut["mutation_id"]: mut for mut in ml} + if check: + ids = {int(j) for x in ts.mutations_derived_state for j in x.split(",")} + for k in ids: + if k not in out: + raise ValueError( + "Top-level mutation metadata is missing " + f"information for mutation ID {k}: " + "do you need to run " + "pyslim.add_mutation_metadata(ts)?" + ) + return out + + def mutation_at(ts, node, position, time=None): """ Finds the mutation present in the genome of ``node`` at ``position``, @@ -59,7 +98,7 @@ def mutation_at(ts, node, position, time=None): return out -def nucleotide_at(ts, node, position, time=None): +def nucleotide_at(ts, node, position, time=None, mut_metadata=None): """ Finds the nucleotide present in the genome of ``node`` at ``position``. Warning: if ``node`` is not actually in the tree sequence (e.g., not @@ -69,20 +108,34 @@ def nucleotide_at(ts, node, position, time=None): at ``position`` inherited by ``node`` that occurred at or before ``time`` ago. + This method uses a dictionary of mutation metadata, computed by + :meth:`mut_metadata`. This step can be expensive if there are + many mutations, so this can be pre-computed and passed in as + ``mutations``. If not provided, it will be computed. + :param int node: The index of a node in the tree sequence. :param float position: A position along the genome. :param int time: The time ago that we want the nucleotide, or None, in which case the ``time`` of ``node`` is used. + :param dict mut_metadata: If provided, a dictionary mapping + mutation ID to metadata, as returned by ``pyslim.mutation_metadata(ts)``. :returns: Index of the nucleotide in ``NUCLEOTIDES`` (0=A, 1=C, 2=G, 3=T). """ if not ts.has_reference_sequence(): raise ValueError("This tree sequence has no reference sequence.") + if mut_metadata is None: + mut_metadata = mutation_metadata(ts) mut_id = mutation_at(ts, node, position, time) if mut_id == tskit.NULL: out = NUCLEOTIDES.index(ts.reference_sequence.data[int(position)]) else: mut = ts.mutation(mut_id) - k = np.argmax([u["slim_time"] for u in mut.metadata["mutation_list"]]) - out = mut.metadata["mutation_list"][k]["nucleotide"] + _, k = max( + [ + (mut_metadata[int(j)]["slim_time"], int(j)) + for j in mut.derived_state.split(",") + ] + ) + out = mut_metadata[k]["nucleotide"] return out diff --git a/tests/__init__.py b/tests/__init__.py index 0996d1a3..376b6ce8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -20,6 +20,25 @@ class PyslimTestCase: Base class for test cases in pyslim. """ + def assert_indiv_metadata_equal(self, a, b): + # individual metadata can have nan's in it, thus this function + assert isinstance(a, dict) + assert type(a) == type(b) + assert a.keys() == b.keys() + for k in a: + if k != "per_trait": + assert a[k] == b[k] + apt = a["per_trait"] + bpt = b["per_trait"] + assert len(apt) == len(bpt) + for x, y in zip(apt, bpt): + assert x.keys() == y.keys() + for k in x: + if k == "phenotype": + assert (np.isnan(x[k]) and np.isnan(y[k])) or (x[k] == y[k]) + else: + assert x[k] == y[k] + def verify_haplotype_equality(self, ts, slim_ts): assert ts.num_sites == slim_ts.num_sites for j, v1, v2 in zip(range(ts.num_sites), ts.variants(), slim_ts.variants()): @@ -53,9 +72,8 @@ def assertMetadataEqual(self, t1, t2): assert t1.metadata_schema == t2.metadata_schema assert t1.metadata == t2.metadata # and now check the underlying bytes - # TODO: use the public interface if https://github.com/tskit-dev/tskit/issues/832 happens - md1 = t1._ll_tables.metadata - md2 = t2._ll_tables.metadata + md1 = t1.metadata_bytes + md2 = t2.metadata_bytes assert md1 == md2 def verify_trees_equal(self, ts1, ts2): @@ -76,10 +94,6 @@ def verify_trees_equal(self, ts1, ts2): if n.metadata is not None: map2[n.metadata["slim_id"]] = j assert set(map1.keys()) == set(map2.keys()) - print(ts1) - print(map1) - print(ts2) - print(map2) sids = list(map1.keys()) for sid in sids: n1 = ts1.node(map1[sid]) @@ -88,22 +102,13 @@ def verify_trees_equal(self, ts1, ts2): assert n1.metadata == n2.metadata i1 = ts1.individual(n1.individual) i2 = ts2.individual(n2.individual) - if i1.metadata != i2.metadata: - print("i1: ", i1.metadata) - print("i2: ", i2.metadata) - assert i1.metadata == i2.metadata + self.assert_indiv_metadata_equal(i1.metadata, i2.metadata) for _ in range(10): pos = random.uniform(0, ts1.sequence_length) t1 = ts1.at(pos) t2 = ts2.at(pos) for _ in range(10): a, b = random.choices(sids, k=2) - print(a, b, map1[a], map1[b], map2[a], map2[b]) - print(t1) - print("a", t1.time(map1[a])) - print("b", t1.time(map1[b])) - print("1", t1.tmrca(map1[a], map1[b])) - print("2", t2.tmrca(map2[a], map2[b])) assert t1.tmrca(map1[a], map1[b]) == t2.tmrca(map2[a], map2[b]) def assertTableCollectionsEqual( diff --git a/tests/conftest.py b/tests/conftest.py index 0a3fa875..6895b5ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,7 @@ import pytest import tskit from filelock import FileLock +from frozendict import frozendict from .recipe_specs import recipe_specs @@ -73,7 +74,7 @@ def load_ts(self, path): c, e = os.path.splitext(cfile) if e == ".trees": out[c] = tskit.load(os.path.join(path, cfile)) - return out + return frozendict(out) def __init__(self, out_dir): # Note: the 'key' below cannot match a 'key' in recipe_specs diff --git a/tests/recipe_specs.py b/tests/recipe_specs.py index 283daff1..d31c2f69 100644 --- a/tests/recipe_specs.py +++ b/tests/recipe_specs.py @@ -12,11 +12,16 @@ # retained: has retained individuals # multipop: has more than one population # multichrom: has more than one chromosome +# traits: has more than just the usual trait +# no_simplify: does not run simplify when writing out +# old_mutations: uses addMutation to add back in some previously lost mutations +# record_mutations: whether mutations and substitutions and reference sequence +# are in top-level metadata # long: kinda big # (chromosome type) # All files are of the form `tests/test_recipes/{key}` recipe_specs = { - "recipe_nonWF.slim": {"nonWF": True, "pedigree": True}, + "recipe_nonWF.slim": {"nonWF": True, "pedigree": True, "record_mutations": True}, "recipe_nonWF_X.slim": {"nonWF": True, "pedigree": True, "X": True}, "recipe_nonWF_Y.slim": {"nonWF": True, "pedigree": True, "Y": True}, "recipe_nonWF_H.slim": {"nonWF": True, "pedigree": True, "H": True}, @@ -78,14 +83,23 @@ }, "recipe_long_nonWF.slim": {"nonWF": True, "long": True}, "recipe_old_nonWF.slim": {"nonWF": True, "remembered_first": True}, - "recipe_WF.slim": {"WF": True, "pedigree": True}, + "recipe_WF.slim": {"WF": True, "pedigree": True, "record_mutations": True}, + "recipe_no_simplify.slim": {"WF": True, "no_simplify": True}, "recipe_long_WF.slim": {"WF": True, "long": True}, "recipe_WF_migration.slim": {"WF": True, "pedigree": True, "multipop": True}, - "recipe_nucleotides_WF.slim": {"WF": True, "pedigree": True, "nucleotides": True}, + "recipe_nucleotides_WF.slim": { + "WF": True, + "pedigree": True, + "nucleotides": True, + "record_mutations": True, + "refseq": True, + }, "recipe_nucleotides_nonWF.slim": { "nonWF": True, "pedigree": True, "nucleotides": True, + "record_mutations": True, + "refseq": True, }, "recipe_nucleotides_plus_others.slim": { "WF": True, @@ -177,7 +191,10 @@ "adds_mutations": True, "nucleotides": True, "non-nucleotides": True, + "record_mutations": True, + "refseq": True, }, + "recipe_adds_old_muts.slim": {"WF": True, "old_mutations": True}, "recipe_many_chromosomes.slim": { "nonWF": True, "pedigree": True, @@ -189,6 +206,15 @@ "multichrom": True, "H-": True, }, + "recipe_with_traits.slim": { + "WF": True, + "traits": True, + "multichrom": True, + "begun_late": True, + "X": True, + "Y": True, + "H": True, + }, } for x in ("first", "early", "late"): diff --git a/tests/test_annotation.py b/tests/test_annotation.py index 6336bc44..ead0a046 100644 --- a/tests/test_annotation.py +++ b/tests/test_annotation.py @@ -15,17 +15,33 @@ import pyslim import tests -from .recipe_specs import restarted_recipe_eq +from .recipe_specs import recipe_eq, restarted_recipe_eq def mutcontext(ts): - if ts.num_mutations > 0: + md = ts.metadata + if ( + isinstance(md, dict) + and "SLiM_mutation_list" in md + and len(md["SLiM_mutation_list"]) > 0 + ): handler = pytest.warns(Warning, match="already has.*metadata") else: handler = contextlib.nullcontext() return handler +def canonicalise_tables(tables): + md = tables.metadata + num_muts = len(md["SLiM_mutation_list"]) + md["SLiM_mutation_list"] = list( + pyslim.mutation_metadata(tables, check=False).values() + ) + assert num_muts == len(md["SLiM_mutation_list"]) + tables.metadata = md + tables.canonicalise() + + def verify_slim_restart_equality(in_ts_dict, out_ts_dict, check_prov=True): """ Check for equality, in everything but the last provenance. @@ -37,9 +53,9 @@ def verify_slim_restart_equality(in_ts_dict, out_ts_dict, check_prov=True): if check_prov: assert in_ts.num_provenances + 1 == out_ts.num_provenances in_tables = in_ts.dump_tables() - in_tables.canonicalise() + canonicalise_tables(in_tables) out_tables = out_ts.dump_tables() - out_tables.canonicalise() + canonicalise_tables(out_tables) in_tables.assert_equals(out_tables, ignore_provenance=True) @@ -103,13 +119,14 @@ def verify_defaults(self, ts): Verify the default values have been entered into metadata. """ do_pops = [False for _ in ts.populations()] - for m in ts.mutations(): - md = m.metadata - assert isinstance(md["mutation_list"], list) - for mdl in md["mutation_list"]: - assert mdl["mutation_type"] == 0 - assert mdl["selection_coeff"] == 0.0 - assert mdl["subpopulation"] == tskit.NULL + for mdl in ts.metadata["SLiM_mutation_list"]: + assert mdl["mutation_type"] == 0 + assert mdl["per_trait"] == [ + {"effect_size": 0.0, "dominance": 0.5, "hemizygous_dominance": 1.0} + ] + assert mdl["subpopulation"] == tskit.NULL + # assert mdl["slim_time"] == 0 # this is set to something meaningful + assert mdl["nucleotide"] == -1 for n in ts.nodes(): md = n.metadata if not n.is_sample(): @@ -220,17 +237,17 @@ def verify_remapping(self, ts, rts, subpop_map): assert m == rm assert ts.num_mutations == rts.num_mutations + mut_info = pyslim.mutation_metadata(ts) + rmut_info = pyslim.mutation_metadata(rts) + assert len(mut_info) == len(rmut_info) # mutations may have changed order - tsm = {m.derived_state: m for m in ts.mutations()} - rtsm = {m.derived_state: m for m in rts.mutations()} + tsm = {m.derived_state for m in ts.mutations()} + rtsm = {m.derived_state for m in rts.mutations()} for x in tsm: assert x in rtsm - m = tsm[x] - rm = rtsm[x] - md = m.metadata - rmd = rm.metadata - assert len(md["mutation_list"]) == len(rmd["mutation_list"]) - for x, y in zip(md["mutation_list"], rmd["mutation_list"]): + for sid in x.split(","): + x = mut_info[int(sid)].copy() + y = rmut_info[int(sid)] x["subpopulation"] = fwd_map[x["subpopulation"]] assert x == y @@ -239,7 +256,7 @@ def verify_remapping(self, ts, rts, subpop_map): md = i.metadata rmd = ri.metadata md["subpopulation"] = fwd_map[md["subpopulation"]] - assert md == rmd + self.assert_indiv_metadata_equal(md, rmd) def test_annotate_errors(self, helper_functions): for ts in helper_functions.get_msprime_examples(): @@ -279,6 +296,11 @@ def test_warns_overwriting_mutations(self, helper_functions): ts, rate=1, random_seed=12, model=msprime.SLiMMutationModel(type=1) ) assert ts.num_mutations > 0 + t = ts.dump_tables() + t.metadata_schema = pyslim.slim_metadata_schemas["tree_sequence"] + t.metadata = pyslim.default_slim_metadata("tree_sequence") + ts = t.tree_sequence() + ts = pyslim.add_mutation_metadata(ts) with pytest.warns(Warning, match="already has.*metadata"): slim_ts = pyslim.annotate(ts, model_type="WF", tick=1) @@ -335,45 +357,35 @@ def test_just_simulate(self, helper_functions, tmp_path): def test_basic_annotation(self, helper_functions, tmp_path): for ts in helper_functions.get_msprime_examples(): - for do_mutations in [False, True]: - tick = 4 - cycle = 1 - stage = "late" - if do_mutations: - handler = mutcontext(ts) - else: - handler = contextlib.nullcontext() - with handler: - slim_ts = pyslim.annotate( - ts, - model_type="WF", - tick=tick, - cycle=cycle, - stage=stage, - annotate_mutations=do_mutations, - ) - assert slim_ts.metadata["SLiM"]["model_type"] == "WF" - assert slim_ts.metadata["SLiM"]["tick"] == tick - assert slim_ts.metadata["SLiM"]["cycle"] == cycle - assert slim_ts.metadata["SLiM"]["stage"] == stage - assert ( - slim_ts.metadata["SLiM"]["file_version"] == pyslim.slim_file_version - ) - self.verify_annotated_tables( - ts, slim_ts, check_alleles=(not do_mutations) + tick = 4 + cycle = 1 + stage = "late" + with mutcontext(ts): + slim_ts = pyslim.annotate( + ts, + model_type="WF", + tick=tick, + cycle=cycle, + stage=stage, + annotate_mutations=True, ) - self.verify_annotated_trees(ts, slim_ts) - if not do_mutations: - self.verify_haplotype_equality(ts, slim_ts) - self.verify_defaults(slim_ts) - self.verify_provenance(slim_ts) - # try loading this into SLiM - loaded_ts = helper_functions.run_msprime_restart( - {"default": slim_ts}, tmp_path, multichrom=False, WF=True - )["default"] - self.verify_annotated_tables(loaded_ts, slim_ts) - self.verify_annotated_trees(loaded_ts, slim_ts) - self.verify_haplotype_equality(loaded_ts, slim_ts) + slim_ts_md = slim_ts.metadata["SLiM"] + assert slim_ts_md["model_type"] == "WF" + assert slim_ts_md["tick"] == tick + assert slim_ts_md["cycle"] == cycle + assert slim_ts_md["stage"] == stage + assert slim_ts_md["file_version"] == pyslim.slim_file_version + self.verify_annotated_tables(ts, slim_ts, check_alleles=False) + self.verify_annotated_trees(ts, slim_ts) + self.verify_defaults(slim_ts) + self.verify_provenance(slim_ts) + # try loading this into SLiM + loaded_ts = helper_functions.run_msprime_restart( + {"default": slim_ts}, tmp_path, multichrom=False, WF=True + )["default"] + self.verify_annotated_tables(loaded_ts, slim_ts) + self.verify_annotated_trees(loaded_ts, slim_ts) + self.verify_haplotype_equality(loaded_ts, slim_ts) def test_annotate_refseq(self): ts = msprime.sim_ancestry(2, sequence_length=10, random_seed=77) @@ -447,22 +459,22 @@ def test_annotate_nodes(self, helper_functions): # not testing SLiM because needs annotation of indivs to make sense def test_annotate_mutations(self, helper_functions): + # test workflow of annotating and editing + rng = np.random.default_rng(seed=123) for ts in helper_functions.get_msprime_examples(): with mutcontext(ts): slim_ts = pyslim.annotate(ts, model_type="nonWF", tick=1) tables = slim_ts.dump_tables() - metadata = [m.metadata for m in tables.mutations] - selcoefs = [random.uniform(0, 1) for _ in metadata] + md = tables.metadata + metadata = md["SLiM_mutation_list"] + # these are stored as "f" meaning "float" (not "double") + selcoefs = rng.uniform(size=len(metadata)).astype("float32") for j in range(len(metadata)): - metadata[j]["mutation_list"][0]["selection_coeff"] = selcoefs[j] - ms = tables.mutations.metadata_schema - tables.mutations.packset_metadata( - [ms.validate_and_encode_row(r) for r in metadata] - ) + metadata[j]["per_trait"][0]["effect_size"] = selcoefs[j] + tables.metadata = md new_ts = tables.tree_sequence() - for j, x in enumerate(new_ts.mutations()): - md = x.metadata - assert np.isclose(md["mutation_list"][0]["selection_coeff"], selcoefs[j]) + for j, md in enumerate(new_ts.metadata["SLiM_mutation_list"]): + assert md["per_trait"][0]["effect_size"] == selcoefs[j] def test_dont_annotate_mutations(self): # Test the option to not overwrite mutation annotations @@ -828,11 +840,14 @@ def test_remapping(self, helper_functions, tmp_path): random_seed=455, ) ts = pyslim.annotate(ts, model_type="WF", tick=1) - ts = msprime.sim_mutations( - ts, - rate=1e-2, - random_seed=9, - model=msprime.SLiMMutationModel(type=1), + ts = pyslim.add_mutation_metadata( + msprime.sim_mutations( + ts, + rate=1e-2, + random_seed=9, + model=msprime.SLiMMutationModel(type=1), + ), + mutation_type=1, ) assert ts.num_mutations > 0 for subpop_map in ( @@ -947,8 +962,8 @@ def test_reload_annotate(self, restart_name, recipe, helper_functions, tmp_path) in_ts = {} for chrom, ts in recipe["ts"].items(): tables = ts.dump_tables() - metadata = [m.metadata for m in tables.mutations] - has_nucleotides = tables.metadata["SLiM"]["nucleotide_based"] + metadata = tables.metadata + has_nucleotides = metadata["SLiM"]["nucleotide_based"] if has_nucleotides: nucs = [random.choice([0, 1, 2, 3]) for _ in metadata] refseq = "".join( @@ -957,17 +972,13 @@ def test_reload_annotate(self, restart_name, recipe, helper_functions, tmp_path) k=int(ts.sequence_length), ), ) - for n, md in zip(nucs, metadata): - for m in md["mutation_list"]: - m["nucleotide"] = n + for n, md in zip(nucs, metadata["SLiM_mutation_list"]): + md["nucleotide"] = n tables.reference_sequence.data = refseq - for md in metadata: - for m in md["mutation_list"]: - m["selection_coeff"] = random.random() - ms = tables.mutations.metadata_schema - tables.mutations.packset_metadata( - [ms.validate_and_encode_row(r) for r in metadata] - ) + for md in metadata["SLiM_mutation_list"]: + for x in md["per_trait"]: + x["effect_size"] = random.random() + tables.metadata = metadata in_ts[chrom] = tables.tree_sequence() # put it through SLiM (which just reads in and writes out) out_ts = helper_functions.run_slim_restart( @@ -1046,3 +1057,136 @@ def test_restarts_and_runs_simplified( it = in_ts[k] ot = out_ts[k] assert ot.metadata["SLiM"]["tick"] >= it.metadata["SLiM"]["tick"] + + +class TestAddMutationMetadata(tests.PyslimTestCase): + def test_add_mutation_metadata_errors(self): + ts = msprime.sim_ancestry(10, random_seed=5) + ts = msprime.sim_mutations( + ts, rate=5, random_seed=3, model=msprime.SLiMMutationModel(type=0) + ) + # bad metadata schema + t = ts.dump_tables() + t.metadata_schema = tskit.MetadataSchema(None) + bad_ts = t.tree_sequence() + with pytest.raises(ValueError, match="metadata schema is not"): + _ = pyslim.add_mutation_metadata(bad_ts) + t.metadata_schema = tskit.MetadataSchema.permissive_json() + t.metadata = {} + bad_ts = t.tree_sequence() + with pytest.raises(ValueError, match="metadata schema is not"): + _ = pyslim.add_mutation_metadata(bad_ts) + t.metadata["SLiM"] = pyslim.default_slim_metadata("tree_sequence")["SLiM"] + bad_ts = t.tree_sequence() + with pytest.raises(ValueError, match="metadata schema is not"): + _ = pyslim.add_mutation_metadata(bad_ts) + + def test_add_mutation_metadata_mutation_type(self): + ts = msprime.sim_ancestry(10, random_seed=5) + ts = msprime.sim_mutations( + ts, rate=5, random_seed=3, model=msprime.SLiMMutationModel(type=0) + ) + t = ts.dump_tables() + t.metadata_schema = pyslim.slim_metadata_schemas["tree_sequence"] + t.metadata = pyslim.default_slim_metadata("tree_sequence") + ts = t.tree_sequence() + for k in (0, 1, 5): + new_ts = pyslim.add_mutation_metadata(ts, mutation_type=k) + for x in new_ts.metadata["SLiM_mutation_list"]: + assert x["mutation_type"] == k + + @pytest.mark.parametrize( + "recipe", + recipe_eq( + exclude=( + "long", + "no_simplify", + "multichrom", + "everyone", + "init_mutated", + "old_mutations", + ) + ), + indirect=True, + ) + def test_add_mutation_metadata(self, recipe): + for _, ts in recipe["ts"].items(): + tables = ts.dump_tables() + md = tables.metadata + md["SLiM_mutation_list"] = [] + tables.metadata = md + new_ts = pyslim.add_mutation_metadata( + tables.tree_sequence(), + ) + old_metadata = pyslim.mutation_metadata(ts) + new_metadata = pyslim.mutation_metadata(new_ts) + assert len(old_metadata) == len(new_metadata) + for k in old_metadata: + assert 0 == new_metadata[k]["mutation_type"] # default + assert old_metadata[k]["slim_time"] == new_metadata[k]["slim_time"] + assert old_metadata[k]["mutation_id"] == new_metadata[k]["mutation_id"] + + @pytest.mark.parametrize( + "recipe", + recipe_eq( + exclude=( + "long", + "no_simplify", + "multichrom", + "everyone", + "init_mutated", + "old_mutations", + ) + ), + indirect=True, + ) + def test_add_mutation_metadata_keeps(self, recipe): + for _, ts in recipe["ts"].items(): + if ts.num_mutations > 15: + tables = ts.dump_tables() + md = tables.metadata + metadata = md["SLiM_mutation_list"] + del metadata[:2] + del metadata[10:] + kept_ids = {m["mutation_id"] for m in metadata} + md["SLiM_mutation_list"] = metadata + tables.metadata = md + new_ts = pyslim.add_mutation_metadata( + tables.tree_sequence(), + ) + old_metadata = pyslim.mutation_metadata(ts) + new_metadata = pyslim.mutation_metadata(new_ts) + assert len(old_metadata) == len(new_metadata) + for k in old_metadata: + if k in kept_ids: + assert old_metadata[k] == new_metadata[k] + else: + assert 0 == new_metadata[k]["mutation_type"] # default + assert ( + old_metadata[k]["slim_time"] == new_metadata[k]["slim_time"] + ) + assert ( + old_metadata[k]["mutation_id"] + == new_metadata[k]["mutation_id"] + ) + + def test_add_mutation_metadata_removes(self): + ts = msprime.sim_ancestry(10, random_seed=5) + ts = msprime.sim_mutations( + ts, rate=5, random_seed=3, model=msprime.SLiMMutationModel(type=0) + ) + ts = pyslim.add_mutation_metadata(pyslim.annotate(ts, model_type="WF", tick=1)) + sts = ts.simplify([0, 1]) + assert sts.num_mutations < ts.num_mutations + assert ts.metadata["SLiM_mutation_list"] == sts.metadata["SLiM_mutation_list"] + nsts = pyslim.add_mutation_metadata(sts, remove_unused=True) + mut_info = pyslim.mutation_metadata(ts) + nmut_info = pyslim.mutation_metadata(nsts) + mut_ids = np.unique( + [int(k) for mut in nsts.mutations() for k in mut.derived_state.split(",")] + ) + assert len(mut_ids) == len(nsts.metadata["SLiM_mutation_list"]) + for k in mut_ids: + assert k in nmut_info + assert k in mut_info + assert mut_info[k] == nmut_info[k] diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b0a44336..e472e8ff 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -12,6 +12,20 @@ from .recipe_specs import recipe_eq +def assert_nan_equal(a, b): + if isinstance(a, float): + assert (a == b) or (np.isnan(a) and np.isnan(b)) + elif isinstance(a, dict) and isinstance(b, dict): + assert a.keys() == b.keys() + for k in a: + assert_nan_equal(a[k], b[k]) + elif isinstance(a, list) and isinstance(b, list): + for x, y in zip(a, b, strict=True): + assert_nan_equal(x, y) + else: + assert a == b + + class TestMetadataSchemas(tests.PyslimTestCase): def validate_table_metadata(self, table): ms = table.metadata_schema @@ -46,35 +60,47 @@ def test_default_metadata(self): schema = pyslim.slim_metadata_schemas[k] entry = pyslim.default_slim_metadata(k) sd = schema.asdict() - if sd is not None: - for p in sd["properties"]: - assert p in entry - encoded = schema.validate_and_encode_row(entry) - decoded = schema.decode_row(encoded) - if entry is None: - assert decoded is None + if k != "tree_sequence": + if sd is not None: + for p in sd["properties"]: + assert p in entry + encoded = schema.validate_and_encode_row(entry) + decoded = schema.decode_row(encoded) + if entry is None: + assert decoded is None + else: + # some defaults have nans, which are not equal + assert_nan_equal(entry, decoded) else: + assert k == "tree_sequence" + for p in sd["json"]["properties"]: + assert p in entry + encoded = schema.validate_and_encode_row(entry) + decoded = schema.decode_row(encoded) + assert entry == decoded + entry["SLiM_mutation_list"].append( + pyslim.default_slim_metadata("mutation_list_entry") + ) + encoded = schema.validate_and_encode_row(entry) + decoded = schema.decode_row(encoded) + assert entry == decoded + entry["SLiM_mutation_list"].append( + pyslim.default_slim_metadata("mutation_list_entry") + ) + encoded = schema.validate_and_encode_row(entry) + decoded = schema.decode_row(encoded) assert entry == decoded - schema = pyslim.slim_metadata_schemas["mutation"] - entry = pyslim.default_slim_metadata("mutation") - entry["mutation_list"].append( - pyslim.default_slim_metadata("mutation_list_entry") - ) - encoded = schema.validate_and_encode_row(entry) - decoded = schema.decode_row(encoded) - assert entry == decoded - entry["mutation_list"].append( - pyslim.default_slim_metadata("mutation_list_entry") - ) - encoded = schema.validate_and_encode_row(entry) - decoded = schema.decode_row(encoded) - assert entry == decoded def test_slim_metadata_schema_equality(self, recipe): num_chromosomes = len(recipe["ts"]) for ts in recipe["ts"].values(): t = ts.dump_tables() - assert t.metadata_schema == pyslim.slim_metadata_schemas["tree_sequence"] + num_traits = len(t.metadata["SLiM"]["traits"]) + ts_schema = pyslim.slim_metadata_schemas["tree_sequence"].asdict() + ts_schema["struct"]["properties"]["SLiM_mutation_list"]["items"][ + "properties" + ]["per_trait"]["length"] = num_traits + assert t.metadata_schema.asdict() == ts_schema assert t.edges.metadata_schema == pyslim.slim_metadata_schemas["edge"] assert t.sites.metadata_schema == pyslim.slim_metadata_schemas["site"] assert ( @@ -85,10 +111,9 @@ def test_slim_metadata_schema_equality(self, recipe): (num_chromosomes + 7) / 8 ) assert t.nodes.metadata_schema.asdict() == node_schema - assert ( - t.individuals.metadata_schema - == pyslim.slim_metadata_schemas["individual"] - ) + ind_schema = pyslim.slim_metadata_schemas["individual"].asdict() + ind_schema["properties"]["per_trait"]["length"] = num_traits + assert t.individuals.metadata_schema.asdict() == ind_schema assert ( t.populations.metadata_schema == pyslim.slim_metadata_schemas["population"] @@ -117,18 +142,25 @@ class TestTreeSequenceMetadata(tests.PyslimTestCase): def validate_slim_metadata(self, t): # t could be tables or a tree sequence schema = t.metadata_schema.schema - assert "SLiM" in schema["properties"] - assert "SLiM" in t.metadata + assert schema["codec"] == "json+struct" + assert "SLiM" in schema["json"]["properties"] + tmd = t.metadata + assert "SLiM" in tmd for k in pyslim.default_slim_metadata("tree_sequence")["SLiM"]: - assert k in schema["properties"]["SLiM"]["properties"] - assert k in t.metadata["SLiM"] + assert k in schema["json"]["properties"]["SLiM"]["properties"] + assert k in tmd["SLiM"] + sml = schema["struct"]["properties"] + assert "SLiM_mutation_list" in sml + for k in pyslim.default_slim_metadata("mutation_list_entry"): + assert k in sml["SLiM_mutation_list"]["items"]["properties"] def validate_model_type(self, tsdict, model_type): for _, ts in tsdict.items(): - assert ts.metadata["SLiM"]["file_version"] == pyslim.slim_file_version - assert ts.metadata["SLiM"]["model_type"] == model_type - assert ts.metadata["SLiM"]["tick"] > 0 - assert ts.metadata["SLiM"]["tick"] >= np.max(ts.tables.nodes.time) + md = ts.metadata + assert md["SLiM"]["file_version"] == pyslim.slim_file_version + assert md["SLiM"]["model_type"] == model_type + assert md["SLiM"]["tick"] > 0 + assert md["SLiM"]["tick"] >= np.max(ts.tables.nodes.time) @pytest.mark.parametrize("recipe", arbitrary_recipe, indirect=True) def test_set_tree_sequence_metadata_errors(self, recipe): @@ -156,19 +188,59 @@ def test_set_tree_sequence_metadata_keeps(self, recipe): tables.metadata = dummy_metadata pyslim.set_tree_sequence_metadata(tables, "nonWF", 0) schema = tables.metadata_schema.schema + tmd = tables.metadata for k in dummy_metadata: if len(x) > 0: - assert k in schema["properties"] - assert k in tables.metadata - assert tables.metadata[k] == dummy_metadata[k] + assert k in schema["json"]["properties"] + assert k in tmd + assert tmd[k] == dummy_metadata[k] self.validate_slim_metadata(tables) - assert tables.metadata["SLiM"]["model_type"] == "nonWF" - assert tables.metadata["SLiM"]["tick"] == 0 + assert tmd["SLiM"]["model_type"] == "nonWF" + assert tmd["SLiM"]["tick"] == 0 + + @pytest.mark.parametrize("recipe", arbitrary_recipe, indirect=True) + def test_set_tree_sequence_metadata_keeps_struct(self, recipe): + # make sure doesn't overwrite other stuff + ts = list(recipe["ts"].values())[0] + json_props = {"num": {"type": "number"}} + struct_props = {"binnum": {"type": "integer", "binaryFormat": "i", "default": 0}} + schema_dict = { + "codec": "json+struct", + "type": "object", + "json": {"codec": "json", "type": "object", "properties": json_props}, + "struct": {"struct": "json", "type": "object", "properties": struct_props}, + } + dummy_schema = tskit.MetadataSchema(schema_dict) + dummy_metadata = {"num": 12, "binnum": 42} + tables = ts.dump_tables() + tables.metadata_schema = dummy_schema + tables.metadata = dummy_metadata + pyslim.set_tree_sequence_metadata(tables, "nonWF", 0) + schema = tables.metadata_schema.schema + tmd = tables.metadata + for k in dummy_metadata: + if k in json_props: + assert schema["json"]["properties"][k] == json_props[k] + else: + assert schema["struct"]["properties"][k] == struct_props[k] + assert k in tmd + assert tmd[k] == dummy_metadata[k] + self.validate_slim_metadata(tables) + assert tmd["SLiM"]["model_type"] == "nonWF" + assert tmd["SLiM"]["tick"] == 0 @pytest.mark.parametrize("recipe", arbitrary_recipe, indirect=True) def test_set_tree_sequence_metadata(self, recipe): ts = list(recipe["ts"].values())[0] tables = ts.dump_tables() + chroms = [ + {"id": 1, "name": "autosome_1", "symbol": "1", "type": "A", "index": 0}, + {"id": 35, "name": "mtDNA", "symbol": "MT", "type": "HF", "index": 1}, + ] + traits = [ + {"index": 0, "name": "theTrait", "type": "additive"}, + {"index": 1, "name": "perfectness", "type": "multiplicative"}, + ] pyslim.set_tree_sequence_metadata( tables, "WF", @@ -179,16 +251,23 @@ def test_set_tree_sequence_metadata(self, recipe): spatial_periodicity="y", separate_sexes=False, nucleotide_based=True, + this_chromosome=chroms[1], + chromosomes=chroms, + traits=traits, ) self.validate_slim_metadata(tables) - assert tables.metadata["SLiM"]["model_type"] == "WF" - assert tables.metadata["SLiM"]["tick"] == 99 - assert tables.metadata["SLiM"]["cycle"] == 40 - assert tables.metadata["SLiM"]["stage"] == "early" - assert tables.metadata["SLiM"]["spatial_dimensionality"] == "xy" - assert tables.metadata["SLiM"]["spatial_periodicity"] == "y" - assert tables.metadata["SLiM"]["separate_sexes"] == False - assert tables.metadata["SLiM"]["nucleotide_based"] == True + tmd = tables.metadata + assert tmd["SLiM"]["model_type"] == "WF" + assert tmd["SLiM"]["tick"] == 99 + assert tmd["SLiM"]["cycle"] == 40 + assert tmd["SLiM"]["stage"] == "early" + assert tmd["SLiM"]["spatial_dimensionality"] == "xy" + assert tmd["SLiM"]["spatial_periodicity"] == "y" + assert tmd["SLiM"]["separate_sexes"] == False + assert tmd["SLiM"]["nucleotide_based"] == True + assert tmd["SLiM"]["chromosomes"] == chroms + assert tmd["SLiM"]["this_chromosome"] == chroms[1] + assert tmd["SLiM"]["traits"] == traits @pytest.mark.parametrize("recipe", recipe_eq("WF"), indirect=True) def test_WF_model_type(self, recipe): @@ -199,12 +278,12 @@ def test_nonWF_model_type(self, recipe): self.validate_model_type(recipe["ts"], "nonWF") @pytest.mark.parametrize( - "recipe", recipe_eq(exclude=["user_metadata", "multichrom"]), indirect=True + "recipe", + recipe_eq(exclude=["user_metadata", "multichrom", "record_mutations"]), + indirect=True, ) def test_recover_metadata(self, recipe): # msprime <=0.7.5 discards metadata, but we can recover it from provenance - # HOWEVER: multichromosome information is not saved - # but this is not something we need to maintain any more. for _, ts in recipe["ts"].items(): tables = ts.dump_tables() tables.metadata_schema = tskit.MetadataSchema(None) @@ -212,13 +291,12 @@ def test_recover_metadata(self, recipe): pyslim.update_tables(tables) md = tables.metadata assert "SLiM" in md - for k in ts.metadata["SLiM"]: - if k in ("chromosomes", "this_chromosome"): - continue # TODO: see https://github.com/MesserLab/SLiM/issues/520 + tsmd = ts.metadata["SLiM"] + for k in tsmd: assert k in md["SLiM"] # slim does not write out empty descriptions - if k != "description" or ts.metadata["SLiM"][k] != "": - assert ts.metadata["SLiM"][k] == md["SLiM"][k] + if k != "description" or tsmd[k] != "": + assert tsmd[k] == md["SLiM"][k] @pytest.mark.parametrize( "recipe", recipe_eq("recipe_with_metadata.slim"), indirect=True @@ -267,10 +345,9 @@ def test_nucleotides(self, recipe): -1, 0, 1, 2, or 3. """ for _, ts in recipe["ts"].items(): - for mut in ts.mutations(): - for u in mut.metadata["mutation_list"]: - assert u["nucleotide"] >= -1 - assert u["nucleotide"] <= 3 + for u in ts.metadata["SLiM_mutation_list"]: + assert u["nucleotide"] >= -1 + assert u["nucleotide"] <= 3 class TestMultichrom(tests.PyslimTestCase): diff --git a/tests/test_provenance.py b/tests/test_provenance.py index d95b275d..71568b22 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -122,6 +122,18 @@ old_provenance_examples = [_slim_v3_0_example, _slim_v3_1_example, _slim_v3_3_1_example] +def yield_ts(path): + out = {} + if os.path.isfile(path): + yield tskit.load(path) + elif os.path.isdir(path): + chroms = os.listdir(path) + for cfile in os.listdir(path): + _, e = os.path.splitext(cfile) + if e == ".trees": + yield tskit.load(os.path.join(path, cfile)) + + class TestProvenance(tests.PyslimTestCase): script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -183,6 +195,18 @@ def get_0_8_slim_examples(self): ]: yield tskit.load(filename) + def get_0_9_slim_examples(self): + for filename in [ + os.path.join(self.script_dir, "test_recipes", "recipe_WF.v5.2.trees"), + os.path.join(self.script_dir, "test_recipes", "recipe_WF_X.v5.2.trees"), + os.path.join(self.script_dir, "test_recipes", "recipe_WF_Y.v5.2.trees"), + os.path.join(self.script_dir, "test_recipes", "recipe_nonWF.v5.2.trees"), + os.path.join( + self.script_dir, "test_recipes", "recipe_WF_many_chromosomes.v5.2.trees" + ), + ]: + yield from yield_ts(filename) + def get_mixed_slim_examples(self): for filename in [ os.path.join( @@ -209,6 +233,115 @@ def verify_upgrade(self, ts): for x in t: _ = ms.validate_and_encode_row(x.metadata) + def verify_consistency(self, ts, pts, file_version): + # Check for stuff we know should be copied over verbatim + # 0.1-0.4 we had no metadata schemas; we could pull those old ones + # from slim_metadata.py but we're not + self.verify_top_level_consistency(ts, pts, file_version) + self.verify_nodes_consistency(ts, pts, file_version) + self.verify_edges_consistency(ts, pts, file_version) + self.verify_sites_consistency(ts, pts, file_version) + self.verify_mutations_consistency(ts, pts, file_version) + self.verify_individuals_consistency(ts, pts, file_version) + self.verify_populations_consistency(ts, pts, file_version) + + def verify_top_level_consistency(self, ts, pts, file_version): + # 0.1-0.7: + # model_type, generation, spatial_dimesionality, spatial_periodicity, + # separate_sexes, nucleotide_based + # 0.8: + # changed generation to tick + # 0.9: + # added this_chromosome + # 1.0: + # added traits + if file_version not in ("0.1", "0.2", "0.3", "0.4"): + md = ts.metadata["SLiM"] + pmd = pts.metadata["SLiM"] + for k in ( + "model_type", + "spatial_dimensionality", + "spatial_periodicity", + "separate_sexes", + "nucleotide_based", + ): + assert md[k] == pmd[k] + k = pk = "tick" + if file_version in ("0.5", "0.6", "0.7"): + k = "generation" + assert md[k] == pmd[pk] + k = "this_chromosome" + if file_version == "0.9": + assert md[k] == pmd[k] + + def verify_edges_consistency(self, ts, pts, file_version): + # no metadata + ts.tables.edges.assert_equals(pts.tables.edges, ignore_metadata=True) + + def verify_sites_consistency(self, ts, pts, file_version): + # no metadata + ts.tables.sites.assert_equals(pts.tables.sites, ignore_metadata=True) + + def verify_mutations_consistency(self, ts, pts, file_version): + ts.tables.mutations.assert_equals(pts.tables.mutations, ignore_metadata=True) + # As of 1.0, metadata moved to top level + if file_version not in ("0.1", "0.2", "0.3", "0.4"): + ptsmd = pts.metadata + num_traits = len(ptsmd["SLiM"]["traits"]) + mut_info = {x["mutation_id"]: x for x in ptsmd["SLiM_mutation_list"]} + for mut in ts.mutations(): + for sid, md in zip( + mut.derived_state.split(","), mut.metadata["mutation_list"] + ): + assert int(sid) in mut_info + mi = mut_info[int(sid)] + for k in ("mutation_type", "subpopulation", "slim_time"): + assert mi[k] == md[k] + assert len(mi["per_trait"]) == 1 + if "nucleotide" in md: + assert mi["nucleotide"] == md["nucleotide"] + assert len(mi["per_trait"]) == num_traits + # we're only converting from single-trait slim so far + assert num_traits == 1 + assert mi["per_trait"][0]["effect_size"] == md["selection_coeff"] + + def verify_individuals_consistency(self, ts, pts, file_version): + ts.tables.individuals.assert_equals(pts.tables.individuals, ignore_metadata=True) + # This has: + # pedigree_id, age, subpopulation, sex, flags + # Starting in 0.7 also: + # pedigree_p1, pedigree_p2 + # Starting in 1.0: per_trait + for a, b in zip(ts.individuals(), pts.individuals()): + if file_version not in ("0.1", "0.2", "0.3", "0.4"): + for k in ("pedigree_id", "age", "subpopulation", "sex", "flags"): + assert a.metadata[k] == b.metadata[k] + if file_version not in ("0.5", "0.6"): + for k in ("pedigree_p1", "pedigree_p2"): + assert a.metadata[k] == b.metadata[k] + + def verify_nodes_consistency(self, ts, pts, file_version): + # 0.1-0.8: had slim_id, is_null, genome_type + # 0.9: removed genome_type + # and changed is_null to is_vacant + # 1.0: same as 0.9 but changed some indexes + if file_version != "0.1": + # 0.1 had a shift in time we're not checking here + ts.tables.nodes.assert_equals(pts.tables.nodes, ignore_metadata=True) + for a, b in zip(ts.nodes(), pts.nodes()): + if file_version in ("0.5", "0.6", "0.7", "0.8"): + assert a.metadata["slim_id"] == b.metadata["slim_id"] + elif file_version == "0.9": + assert a.metadata == b.metadata + + def verify_populations_consistency(self, ts, pts, file_version): + ts.tables.populations.assert_equals(pts.tables.populations, ignore_metadata=True) + # This has a whole bunch of things, none of which are required. + for a, b in zip(ts.individuals(), pts.individuals()): + if file_version not in ("0.1", "0.2", "0.3", "0.4"): + for k in a.metadata: + assert a.metadata[k] == b.metadata[k] + def test_convert_0_1_files(self): for ts in self.get_0_1_slim_examples(): assert not pyslim.is_current_version(ts) @@ -216,24 +349,16 @@ def test_convert_0_1_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.1") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert record["model_type"] == pts.metadata["SLiM"]["model_type"] - assert record["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_2_files(self): for ts in self.get_0_2_slim_examples(): @@ -242,26 +367,16 @@ def test_convert_0_2_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.2") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_3_files(self): for ts in self.get_0_3_slim_examples(): @@ -270,26 +385,16 @@ def test_convert_0_3_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.3") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_4_files(self): # Note that with version 0.5 and above, we *don't* get information from @@ -300,26 +405,16 @@ def test_convert_0_4_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.4") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_5_files(self): for ts in self.get_0_5_slim_examples(): @@ -328,26 +423,16 @@ def test_convert_0_5_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.5") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_6_files(self): for ts in self.get_0_6_slim_examples(): @@ -356,26 +441,16 @@ def test_convert_0_6_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.6") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_7_files(self): for ts in self.get_0_7_slim_examples(): @@ -384,26 +459,16 @@ def test_convert_0_7_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.7") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] - assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) - samples = list(ts.samples()) - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] def test_convert_0_8_files(self): for ts in self.get_0_8_slim_examples(): @@ -412,18 +477,17 @@ def test_convert_0_8_files(self): pts = pyslim.update(ts) assert pyslim.is_current_version(pts) self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.8") assert ts.num_provenances == 1 assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["tick"] == pts.metadata["SLiM"]["tick"] + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["tick"] == ptsmd["SLiM"]["tick"] assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) samples = list(ts.samples()) genome_type = None for n in samples: @@ -432,7 +496,7 @@ def test_convert_0_8_files(self): genome_type = md["genome_type"] break assert genome_type is not None - chromosome_type = pts.metadata["SLiM"]["this_chromosome"]["type"] + chromosome_type = ptsmd["SLiM"]["this_chromosome"]["type"] GENOME_TYPE_AUTOSOME = 0 GENOME_TYPE_X = 1 GENOME_TYPE_Y = 2 @@ -442,13 +506,24 @@ def test_convert_0_8_files(self): assert chromosome_type == "X" elif genome_type == GENOME_TYPE_Y: assert chromosome_type == "-Y" - t = ts.first() - pt = pts.first() - for _ in range(20): - u = random.sample(samples, 1)[0] - assert t.parent(u) == pt.parent(u) - if t.parent(u) != tskit.NULL: - assert t.branch_length(u) == pt.branch_length(u) + + def test_convert_0_9_files(self): + for ts in self.get_0_9_slim_examples(): + assert not pyslim.is_current_version(ts) + with pytest.warns(Warning): + pts = pyslim.update(ts) + assert pyslim.is_current_version(pts) + self.verify_upgrade(pts) + self.verify_consistency(ts, pts, file_version="0.9") + assert ts.num_provenances == 1 + assert pts.num_provenances == 2 + assert ts.provenance(0).record == pts.provenance(0).record + record = json.loads(ts.provenance(0).record) + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["tick"] == ptsmd["SLiM"]["tick"] def test_convert_mixed_files(self): for ts in self.get_mixed_slim_examples(): @@ -461,14 +536,13 @@ def test_convert_mixed_files(self): assert pts.num_provenances == 2 assert ts.provenance(0).record == pts.provenance(0).record record = json.loads(ts.provenance(0).record) - assert isinstance(pts.metadata, dict) - assert "SLiM" in pts.metadata - assert ( - record["parameters"]["model_type"] == pts.metadata["SLiM"]["model_type"] - ) - assert record["slim"]["generation"] == pts.metadata["SLiM"]["tick"] + ptsmd = pts.metadata + assert isinstance(ptsmd, dict) + assert "SLiM" in ptsmd + assert record["parameters"]["model_type"] == ptsmd["SLiM"]["model_type"] + assert record["slim"]["generation"] == ptsmd["SLiM"]["tick"] assert list(ts.samples()) == list(pts.samples()) - assert np.array_equal(ts.tables.nodes.flags, pts.tables.nodes.flags) + assert np.array_equal(ts.tables.nodes.flags, pts.nodes_flags) samples = list(ts.samples()) t = ts.first() pt = pts.first() diff --git a/tests/test_recipes/make_v3_tests.sh b/tests/test_recipes/make_old_file_versions.sh similarity index 80% rename from tests/test_recipes/make_v3_tests.sh rename to tests/test_recipes/make_old_file_versions.sh index eb88af9b..ab6b48f9 100644 --- a/tests/test_recipes/make_v3_tests.sh +++ b/tests/test_recipes/make_old_file_versions.sh @@ -95,4 +95,20 @@ $SLIMDIR/slim recipe_WF_X.slim && mv out.trees recipe_WF_X.${TAG}.trees $SLIMDIR/slim recipe_WF_Y.slim && mv out.trees recipe_WF_Y.${TAG}.trees git add -f recipe_nonWF.${TAG}.trees recipe_WF.${TAG}.trees recipe_WF_X.${TAG}.trees recipe_WF_Y.${TAG}.trees +# To make the v5.2 files: + +TAG=v5.2 +git checkout $TAG +mkdir -p build_$TAG && cd build_$TAG +cmake .. && make +SLIMDIR=$(pwd) +cd ../.. +$SLIMDIR/slim recipe_nonWF.slim && mv out.trees recipe_nonWF.${TAG}.trees +$SLIMDIR/slim recipe_WF.slim && mv out.trees recipe_WF.${TAG}.trees +$SLIMDIR/slim recipe_WF_X.slim && mv out.trees recipe_WF_X.${TAG}.trees +$SLIMDIR/slim recipe_WF_Y.slim && mv out.trees recipe_WF_Y.${TAG}.trees +$SLIMDIR/slim recipe_all_the_chromosome_types.slim && mv out.trees recipe_all_the_chromosome_types.${TAG}.trees +git add -f recipe_nonWF.${TAG}.trees recipe_WF.${TAG}.trees recipe_WF_X.${TAG}.trees recipe_WF_Y.${TAG}.trees recipe_all_the_chromosome_types.${TAG}.trees + + diff --git a/tests/test_recipes/recipe_WF.slim b/tests/test_recipes/recipe_WF.slim index 25032360..11702500 100644 --- a/tests/test_recipes/recipe_WF.slim +++ b/tests/test_recipes/recipe_WF.slim @@ -10,14 +10,49 @@ initialize() initializeGenomicElementType("g1", m1, 1.0); initializeGenomicElement(g1, 0, 99); initializeRecombinationRate(1e-2); + defineGlobal("MD", Dictionary()); } 1 early() { sim.addSubpop("p1", 10); } +// MUTATION/GENOTYPE INFO +1 first() { // mutation information + MD.setValue("mutations", Dictionary()); +} +mutation() { + muts = MD.getValue("mutations"); + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "originTick", mut.originTick + ); + muts.setValue(asString(mut.id), m); + return T; +} +10 late() { + subs = Dictionary(); + for (mut in sim.substitutions) { + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "fixationTick", mut.fixationTick + ); + subs.setValue(asString(mut.id), m); + } + MD.setValue("substitutions", subs); +} + +// OUTPUT/FINISH 10 late() { - sim.treeSeqOutput(TREES_FILE); + sim.treeSeqOutput(TREES_FILE, metadata=MD); catn("Done."); sim.simulationFinished(); } diff --git a/tests/test_recipes/recipe_WF.v5.2.trees b/tests/test_recipes/recipe_WF.v5.2.trees new file mode 100644 index 00000000..936f1cd1 Binary files /dev/null and b/tests/test_recipes/recipe_WF.v5.2.trees differ diff --git a/tests/test_recipes/recipe_WF_X.v5.2.trees b/tests/test_recipes/recipe_WF_X.v5.2.trees new file mode 100644 index 00000000..d09e122b Binary files /dev/null and b/tests/test_recipes/recipe_WF_X.v5.2.trees differ diff --git a/tests/test_recipes/recipe_WF_Y.v5.2.trees b/tests/test_recipes/recipe_WF_Y.v5.2.trees new file mode 100644 index 00000000..e1f33097 Binary files /dev/null and b/tests/test_recipes/recipe_WF_Y.v5.2.trees differ diff --git a/tests/test_recipes/recipe_adds_old_muts.slim b/tests/test_recipes/recipe_adds_old_muts.slim new file mode 100644 index 00000000..a148a6bf --- /dev/null +++ b/tests/test_recipes/recipe_adds_old_muts.slim @@ -0,0 +1,30 @@ +initialize() +{ + setSeed(23); + if (!exists("TREES_FILE")) defineGlobal("TREES_FILE", "out.trees"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); + // record mutations and add them back at the end + defineGlobal("MUTS", c()); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +early() { + defineGlobal("MUTS", c(MUTS, sim.mutations)); +} + +10 late() { + ind = p1.individuals[0]; + ind.haploidGenome1.addMutations(MUTS); + sim.treeSeqOutput(TREES_FILE); + catn("Done."); + sim.simulationFinished(); +} diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_A.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_A.trees new file mode 100644 index 00000000..3df791e1 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_A.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_FL.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_FL.trees new file mode 100644 index 00000000..3d3fe255 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_FL.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_H.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_H.trees new file mode 100644 index 00000000..a0fdca80 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_H.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_HF.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_HF.trees new file mode 100644 index 00000000..473023a9 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_HF.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_HM.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_HM.trees new file mode 100644 index 00000000..1fb4b3f5 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_HM.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_ML.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_ML.trees new file mode 100644 index 00000000..09bd52f4 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_ML.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_W.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_W.trees new file mode 100644 index 00000000..740a592d Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_W.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_X.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_X.trees new file mode 100644 index 00000000..b70be3f8 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_X.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_Y.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_Y.trees new file mode 100644 index 00000000..78604afe Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_Y.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_Z.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_Z.trees new file mode 100644 index 00000000..252e4426 Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_Z.trees differ diff --git a/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_nY.trees b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_nY.trees new file mode 100644 index 00000000..b599982e Binary files /dev/null and b/tests/test_recipes/recipe_all_the_chromosome_types.v5.2.trees/chromosome_nY.trees differ diff --git a/tests/test_recipes/recipe_chromosomes_adds_muts.slim b/tests/test_recipes/recipe_chromosomes_adds_muts.slim index e53e7332..2ffd3299 100644 --- a/tests/test_recipes/recipe_chromosomes_adds_muts.slim +++ b/tests/test_recipes/recipe_chromosomes_adds_muts.slim @@ -20,10 +20,11 @@ initialize() { for (id in ids, symbol in symbols, type in types) { initializeChromosome(id, length, type, symbol); - initializeAncestralNucleotides(paste0(rep("A", length))); + initializeAncestralNucleotides(randomNucleotides(length)); initializeRecombinationRate(1e-5); initializeGenomicElement(g1, 0, length-1); } + defineGlobal("MD", Dictionary()); } 1 early() { sim.addSubpop("p1", 10); @@ -34,13 +35,68 @@ initialize() { inds = p1.individuals; haps = inds.haplosomesForChromosomes(chrom, includeNulls=F); if (length(haps) > 0) { - sample(haps, 1 + asInteger(length(haps)/2)).addNewDrawnMutation(m2, rdunif(10, 0, chrom.length-1)); + mutl = sample(haps, 1 + asInteger(length(haps)/2)).addNewDrawnMutation(m2, rdunif(10, 0, chrom.length-1)); + muts = MD.getValue("mutations"); + for (mut in mutl) { + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "originTick", mut.originTick + ); + muts.setValue(asString(mut.id), m); + } } } } + +// MUTATION/GENOTYPE INFO +1 first() { // reference sequences + refseqs = Dictionary(); + for (chrom in sim.chromosomes) { + refseqs.setValue("chr" + chrom.id, chrom.ancestralNucleotides()); + } + MD.setValue("reference_sequence", refseqs); +} + +1 first() { // mutation information + MD.setValue("mutations", Dictionary()); +} +mutation() { + muts = MD.getValue("mutations"); + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "originTick", mut.originTick + ); + muts.setValue(asString(mut.id), m); + return T; +} +10 late() { + subs = Dictionary(); + for (mut in sim.substitutions) { + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "fixationTick", mut.fixationTick + ); + subs.setValue(asString(mut.id), m); + } + MD.setValue("substitutions", subs); +} + +// OUTPUT/FINISH 10 late() { - sim.treeSeqOutput(TREES_FILE); + sim.treeSeqOutput(TREES_FILE, metadata=MD); catn("Done."); sim.simulationFinished(); } diff --git a/tests/test_recipes/recipe_no_simplify.slim b/tests/test_recipes/recipe_no_simplify.slim new file mode 100644 index 00000000..ceb49b0a --- /dev/null +++ b/tests/test_recipes/recipe_no_simplify.slim @@ -0,0 +1,22 @@ +initialize() +{ + setSeed(23); + if (!exists("TREES_FILE")) defineGlobal("TREES_FILE", "out.trees"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(simplificationRatio=INF, timeUnit="generations"); + initializeMutationRate(1e-2); + initializeMutationType("m1", 0.5, "f", -0.1); + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElement(g1, 0, 99); + initializeRecombinationRate(1e-2); +} + +1 early() { + sim.addSubpop("p1", 10); +} + +10 late() { + sim.treeSeqOutput(TREES_FILE, simplify=F); + catn("Done."); + sim.simulationFinished(); +} diff --git a/tests/test_recipes/recipe_nonWF.slim b/tests/test_recipes/recipe_nonWF.slim index 72727ad8..7174a853 100644 --- a/tests/test_recipes/recipe_nonWF.slim +++ b/tests/test_recipes/recipe_nonWF.slim @@ -12,6 +12,7 @@ initialize() initializeGenomicElement(g1, 0, 99); initializeRecombinationRate(1e-2); defineConstant("K", 10); + defineGlobal("MD", Dictionary()); } reproduction() { @@ -26,8 +27,43 @@ early() { p1.fitnessScaling = K / p1.individualCount; } + +// MUTATION/GENOTYPE INFO +1 first() { // mutation information + MD.setValue("mutations", Dictionary()); +} +mutation() { + muts = MD.getValue("mutations"); + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "originTick", mut.originTick + ); + muts.setValue(asString(mut.id), m); + return T; +} +10 late() { + subs = Dictionary(); + for (mut in sim.substitutions) { + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "fixationTick", mut.fixationTick + ); + subs.setValue(asString(mut.id), m); + } + MD.setValue("substitutions", subs); +} + +// OUTPUT/FINISH 10 late() { - sim.treeSeqOutput(TREES_FILE); + sim.treeSeqOutput(TREES_FILE, metadata=MD); catn("Done."); sim.simulationFinished(); } diff --git a/tests/test_recipes/recipe_nonWF.v5.2.trees b/tests/test_recipes/recipe_nonWF.v5.2.trees new file mode 100644 index 00000000..137d2b03 Binary files /dev/null and b/tests/test_recipes/recipe_nonWF.v5.2.trees differ diff --git a/tests/test_recipes/recipe_nucleotides_WF.slim b/tests/test_recipes/recipe_nucleotides_WF.slim index 29baf777..32b30e9b 100644 --- a/tests/test_recipes/recipe_nucleotides_WF.slim +++ b/tests/test_recipes/recipe_nucleotides_WF.slim @@ -12,14 +12,57 @@ initialize() { initializeGenomicElementType("g1", m1, 1.0, mmJukesCantor(4e-2)); initializeGenomicElement(g1, 0, L-1); initializeRecombinationRate(1e-2); + defineGlobal("MD", Dictionary()); } 1 early() { sim.addSubpop("p1", 10); } +// MUTATION/GENOTYPE INFO +1 first() { // reference sequences + refseqs = Dictionary(); + for (chrom in sim.chromosomes) { + refseqs.setValue("chr" + chrom.id, chrom.ancestralNucleotides()); + } + MD.setValue("reference_sequence", refseqs); +} + +1 first() { // mutation information + MD.setValue("mutations", Dictionary()); +} +mutation() { + muts = MD.getValue("mutations"); + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "originTick", mut.originTick + ); + muts.setValue(asString(mut.id), m); + return T; +} +10 late() { + subs = Dictionary(); + for (mut in sim.substitutions) { + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "fixationTick", mut.fixationTick + ); + subs.setValue(asString(mut.id), m); + } + MD.setValue("substitutions", subs); +} + +// OUTPUT/FINISH 10 late() { - sim.treeSeqOutput(TREES_FILE); + sim.treeSeqOutput(TREES_FILE, metadata=MD); catn("Done."); sim.simulationFinished(); } diff --git a/tests/test_recipes/recipe_nucleotides_nonWF.slim b/tests/test_recipes/recipe_nucleotides_nonWF.slim index 0ae0dc03..5896984f 100644 --- a/tests/test_recipes/recipe_nucleotides_nonWF.slim +++ b/tests/test_recipes/recipe_nucleotides_nonWF.slim @@ -12,6 +12,7 @@ initialize() { initializeGenomicElement(g1, 0, L-1); initializeRecombinationRate(1e-2); defineConstant("K", 10); + defineGlobal("MD", Dictionary()); } reproduction() { @@ -26,8 +27,51 @@ early() { p1.fitnessScaling = K / p1.individualCount; } + +// MUTATION/GENOTYPE INFO +1 first() { // reference sequences + refseqs = Dictionary(); + for (chrom in sim.chromosomes) { + refseqs.setValue("chr" + chrom.id, chrom.ancestralNucleotides()); + } + MD.setValue("reference_sequence", refseqs); +} + +1 first() { // mutation information + MD.setValue("mutations", Dictionary()); +} +mutation() { + muts = MD.getValue("mutations"); + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "originTick", mut.originTick + ); + muts.setValue(asString(mut.id), m); + return T; +} +10 late() { + subs = Dictionary(); + for (mut in sim.substitutions) { + nuc = mut.mutationType.nucleotideBased ? mut.nucleotide else "N"; + m = Dictionary( + "chromosome_id", mut.chromosome.id, + "mutationType", mut.mutationType.id, + "nucleotide", nuc, + "position", mut.position, + "fixationTick", mut.fixationTick + ); + subs.setValue(asString(mut.id), m); + } + MD.setValue("substitutions", subs); +} + +// OUTPUT/FINISH 10 late() { - sim.treeSeqOutput(TREES_FILE); + sim.treeSeqOutput(TREES_FILE, metadata=MD); catn("Done."); sim.simulationFinished(); } diff --git a/tests/test_recipes/recipe_with_traits.slim b/tests/test_recipes/recipe_with_traits.slim new file mode 100644 index 00000000..72602d26 --- /dev/null +++ b/tests/test_recipes/recipe_with_traits.slim @@ -0,0 +1,101 @@ +initialize() { + setSeed(23); + if (!exists("TREES_FILE")) defineGlobal("TREES_FILE", "out.trees"); + initializeSLiMOptions(keepPedigrees=T); + initializeTreeSeq(timeUnit="generations"); + defineConstant("I1", 5.0); + defineConstant("I2", -5.0); + defineConstant("OPT1", 10.0); + defineConstant("OPT2", 10.0); + defineConstant("SD1", 2.0); + defineConstant("SD2", 2.0); + + initializeSex(); + + popgen1T = initializeTrait("popgen1T", "m", 1.0, 0.0, 0.01, directFitnessEffect=T); + popgen2T = initializeTrait("popgen2T", "m", 1.0, 0.0, 0.01, directFitnessEffect=T); + n1T = initializeTrait("n1T", "m", directFitnessEffect=T); + n2T = initializeTrait("n2T", "m", directFitnessEffect=F); + + quant1T = initializeTrait("quant1T", "a", I1, 0.0, 0.01, directFitnessEffect=F); + quant2T = initializeTrait("quant2T", "a", I2, 0.0, 0.01, directFitnessEffect=F); + n3T = initializeTrait("n3T", "a", directFitnessEffect=F, baselineAccumulation=F); + + + logistic1T = initializeTrait("logistic1T", "l", 0.0, 0.01, 0.01, directFitnessEffect=T); + initializeMutationType("m1", 0.4, "f", 0.0); + initializeMutationType("m2", 0.4, "e", 0.05); + m2.setEffectSizeDistributionForTrait(c(n1T, n2T), "f", 0.0); + m2.setEffectSizeDistributionForTrait(c(quant1T, quant2T), "n", 0.0, 0.1); + m2.setEffectSizeDistributionForTrait(c(logistic1T), "n", -0.05, 0.1); + + initializeMutationType("m3", 0.4, "g", -0.05, 1.0); + m3.setEffectSizeDistributionForTrait(c(n1T, n2T), "f", 0.0); + m3.setEffectSizeDistributionForTrait(c(quant1T, quant2T), "n", 0.0, 0.1); + m3.setEffectSizeDistributionForTrait(c(logistic1T), "n", -0.05, 0.1); + + c(m2,m3).setEffectSizeDistributionForTrait(n3T, "n", -5.0, 0.5); + + c(m2,m3).setDefaultDominanceForTrait(c(popgen2T, quant2T), NAN); + + c(m2,m3).logMutationData(T, trait=NULL, effectSize=T, dominance=T); + + initializeGenomicElementType("g1", m1, 1.0); + initializeGenomicElementType("g2", 1:3, c(3, 1, 2)); + + ids = 1:5; + symbols = c(1, 2, "X", "Y", "MT"); + lengths = rdunif(5, 1e7, 2e7); + types = c("A", "A", "X", "Y", "H"); + names = c("A1", "A2", "X", "Y", "MT"); + + for (id in ids, symbol in symbols, length in lengths, type in types, name in names) + { + initializeChromosome(id, length, type, symbol, name); + initializeMutationRate(1e-7); + initializeRecombinationRate(1e-8); + + if (id == 1) + initializeGenomicElement(g1); // autosome 1 is pure-neutral, using only m1 + else + initializeGenomicElement(g2); // autosome 2 is a mix, using m1 / m2 / m3 + } +} + +mutation(m2) { + // set random dominance effects for the popgen1T and quant1T and logistic1TDominance traits + // other effects are generated as specified by the mutation type DES + mut.popgen1TDominance = runif(1); + mut.quant1TDominance = runif(1); + mut.logistic1TDominance = runif(1); + return T; +} +mutation(m3) { + // set random dominance effects for the popgen1T and quant1T and logistic1TDominance traits + // other effects are generated as specified by the mutation type DES + mut.popgen1TDominance = runif(1); + mut.quant1TDominance = runif(1); + mut.logistic1TDominance = runif(1); + return T; +} + +1 late() { + sim.addSubpop("p1", 20); +} + +1: late() { + inds = sim.subpopulations.individuals; + sim.demandPhenotype(NULL, c(sim.quant1T, sim.quant2T)); + phenotypes_q1 = inds.quant1T; + phenotypes_q2 = inds.quant2T; + fitnessEffect_q1 = dnorm(phenotypes_q1, OPT1, SD1) / dnorm(0.0, 0.0, SD1); + fitnessEffect_q2 = dnorm(phenotypes_q2, OPT2, SD2) / dnorm(0.0, 0.0, SD2); + inds.fitnessScaling = fitnessEffect_q1 * fitnessEffect_q2; +} + + +10 late() { + sim.treeSeqOutput(TREES_FILE); + catn("Done."); + sim.simulationFinished(); +} diff --git a/tests/test_tree_sequence.py b/tests/test_tree_sequence.py index 57f9b0ea..18a96111 100644 --- a/tests/test_tree_sequence.py +++ b/tests/test_tree_sequence.py @@ -2,6 +2,7 @@ Test cases for tree sequences. """ +import copy import json import random import sys @@ -17,6 +18,18 @@ from .recipe_specs import recipe_eq, restarted_recipe_eq +def run_with_ts_metadata(f, ts_metadata, *args, **kwargs): + # check for equality in a method with and without passing in + # the ts_metadata argument + a = f(*args, **kwargs) + new_kwargs = copy.deepcopy(kwargs) + new_kwargs["ts_metadata"] = ts_metadata + b = f(*args, **new_kwargs) + assert len(a) == len(b) + np.testing.assert_equal(a, b) + return a + + def mutations_above(ts, node, pos): for s in ts.sites(): if s.position == pos: @@ -42,26 +55,65 @@ def naive_mutation_at(ts, node, pos, time=None): return mut_id +def verify_mutation_metadata(ts): + # Verify that all derived states are properly accounted for + # in mutation metadata. + mdl = ts.metadata["SLiM_mutation_list"] + mut_info = {str(mut["mutation_id"]): mut for mut in mdl} + assert len(mut_info) == len(mdl) + for mut in ts.mutations(): + for j in mut.derived_state.split(","): + assert j in mut_info + + +class TestMutationMetadata(tests.PyslimTestCase): + @pytest.mark.parametrize("recipe", recipe_eq("multichrom"), indirect=True) + def test_mutation_IDs_unique(self, recipe): + ids = set() + for _, ts in recipe["ts"].items(): + mut_info = pyslim.mutation_metadata(ts) + new_ids = set(mut_info.keys()) + assert len(ids.intersection(new_ids)) == 0 + ids = ids.union(new_ids) + + def test_mutation_metadata(self, recipe): + # test that mutation metadata is properly present + for _, ts in recipe["ts"].items(): + verify_mutation_metadata(ts) + + @pytest.mark.parametrize("recipe", [next(recipe_eq())], indirect=True) + def test_check(self, recipe): + for _, ts in recipe["ts"].items(): + assert ts.num_mutations > 5 + t = ts.dump_tables() + md = t.metadata + del md["SLiM_mutation_list"][5:] + t.metadata = md + ts = t.tree_sequence() + mut_info = pyslim.mutation_metadata(ts, check=False) + assert len(mut_info) == 5 + with pytest.raises(ValueError, match="missing information for mutation"): + _ = pyslim.mutation_metadata(ts) + + class TestSlimTime(tests.PyslimTestCase): # Tests for slim_time() - @pytest.mark.parametrize("recipe", recipe_eq(exclude="long"), indirect=True) + @pytest.mark.parametrize( + "recipe", recipe_eq(exclude=["long", "old_mutations"]), indirect=True + ) def test_slim_time(self, recipe): for _, ts in recipe["ts"].items(): - if "init_mutated" not in recipe: - for mut in ts.mutations(): - mut_time = max( - [x["slim_time"] for x in mut.metadata["mutation_list"]] - ) - assert mut_time == pyslim.slim_time(ts, mut.time) + muts = pyslim.mutation_metadata(ts) # the mutations in "init_mutated" examples have mutations that are *added* # in *early*, and so their times match in that stage. - else: - for mut in ts.mutations(): - mut_time = max( - [x["slim_time"] for x in mut.metadata["mutation_list"]] - ) - assert mut_time == pyslim.slim_time(ts, mut.time, stage="early") + stage = "early" if "init_mutated" in recipe else None + slim_times = pyslim.slim_time(ts, ts.mutations_time, stage=stage) + for t, mut in zip(slim_times, ts.mutations()): + mut_time = max( + [muts[int(j)]["slim_time"] for j in mut.derived_state.split(",")] + ) + assert mut_time == t class TestNextMutationID(tests.PyslimTestCase): @@ -99,12 +151,15 @@ def test_reload_slim(self, recipe, helper_functions, tmp_path): ) next_id = pyslim.next_slim_mutation_id(rts) T = max(1, rts.segregating_sites(mode="branch", span_normalise=False)) - mts = msprime.sim_mutations( - rts, - rate=max(6e-4, 10 / T), - keep=True, - model=msprime.SLiMMutationModel(type=1, next_id=next_id), - random_seed=135, + mts = pyslim.add_mutation_metadata( + msprime.sim_mutations( + rts, + rate=max(6e-4, 10 / T), + keep=True, + model=msprime.SLiMMutationModel(type=1, next_id=next_id), + random_seed=135, + ), + mutation_type=1, ) assert mts.num_mutations > rts.num_mutations recapped[chrom] = mts @@ -127,11 +182,16 @@ def test_reload_slim(self, recipe, helper_functions, tmp_path): assert chrom in recipe["ts"] assert pyslim.next_slim_mutation_id(mts) == pyslim.next_slim_mutation_id(ts) assert ts.num_mutations == recapped[chrom].num_mutations - a = ts.metadata - a["SLiM"].pop("user_metadata", None) - b = recipe["ts"][chrom].metadata - b["SLiM"].pop("user_metadata", None) - assert a == b + ots = recipe["ts"][chrom] + assert ts.metadata["SLiM"] == ots.metadata["SLiM"] + mut_info = pyslim.mutation_metadata(ts) + omut_info = pyslim.mutation_metadata(ots) + assert len(mut_info) == len(ts.metadata["SLiM_mutation_list"]) + assert len(omut_info) == len(ots.metadata["SLiM_mutation_list"]) + # we've added new mutations but originals should all be there + for k in omut_info: + assert k in mut_info + assert omut_info[k] == mut_info[k] def test_invalid_derived_state(self): ts = msprime.sim_ancestry( @@ -156,10 +216,12 @@ class TestRecapitate(tests.PyslimTestCase): """ def check_recap_consistency(self, ts, recap, with_ancestral_Ne=True): - assert ts.metadata["SLiM"]["tick"] == recap.metadata["SLiM"]["tick"] - assert ts.metadata["SLiM"]["cycle"] == recap.metadata["SLiM"]["cycle"] - assert ts.metadata["SLiM"]["stage"] == recap.metadata["SLiM"]["stage"] - assert ts.metadata["SLiM"]["name"] == recap.metadata["SLiM"]["name"] + tsmd = ts.metadata + remd = recap.metadata + assert tsmd["SLiM"]["tick"] == remd["SLiM"]["tick"] + assert tsmd["SLiM"]["cycle"] == remd["SLiM"]["cycle"] + assert tsmd["SLiM"]["stage"] == remd["SLiM"]["stage"] + assert tsmd["SLiM"]["name"] == remd["SLiM"]["name"] assert all(tree.num_roots == 1 for tree in recap.trees()) assert ts.has_reference_sequence() == recap.has_reference_sequence() if ts.has_reference_sequence(): @@ -264,6 +326,7 @@ def test_unique_names(self): assert names[0] == "ancestral" assert names[-2] == "ancestral_ancestral" + @pytest.mark.parametrize("recipe", recipe_eq(exclude="no_simplify"), indirect=True) def test_recapitation(self, recipe): for _, ts in recipe["ts"].items(): recomb_rate = 1.0 / ts.sequence_length @@ -282,7 +345,9 @@ def test_recapitation(self, recipe): assert t.num_roots == 1 assert recap.node(t.root).time >= old_root_time - @pytest.mark.parametrize("recipe", recipe_eq(exclude="long"), indirect=True) + @pytest.mark.parametrize( + "recipe", recipe_eq(exclude=["long", "no_simplify"]), indirect=True + ) def test_with_recomb_map(self, recipe): for _, ts in recipe["ts"].items(): recomb_rate = 1.0 / ts.sequence_length @@ -333,9 +398,10 @@ def test_first_gen_nodes(self, recipe): # (note this will fail if some populations were started at different # times than others or if the tick has been changed) for _, ts in recipe["ts"].items(): - root_time = ts.metadata["SLiM"]["tick"] - is_wf = ts.metadata["SLiM"]["model_type"] == "WF" - remembered_stage = ts.metadata["SLiM"]["stage"] + tsmd = ts.metadata + root_time = tsmd["SLiM"]["tick"] + is_wf = tsmd["SLiM"]["model_type"] == "WF" + remembered_stage = tsmd["SLiM"]["stage"] if (not is_wf) or (remembered_stage != "late"): root_time -= 1 if (not is_wf) and ("begun_first" in recipe): @@ -344,12 +410,10 @@ def test_first_gen_nodes(self, recipe): root_time -= 1 if is_wf and ("begun_late" in recipe): root_time -= 1 + vacant = pyslim.nodes_vacant(ts) for t in ts.trees(): for u in t.roots: - assert ( - pyslim.node_is_vacant(ts, ts.node(u)) - or ts.node(u).time == root_time - ) + assert vacant[u] or ts.node(u).time == root_time class TestIndividualAges(tests.PyslimTestCase): @@ -384,14 +448,23 @@ def test_mismatched_remembered_stage(self, recipe): def test_population(self, recipe): for _, ts in recipe["ts"].items(): individual_populations = ts.individuals_population - all_inds = pyslim.individuals_alive_at(ts, 0) + ts_metadata = ts.metadata + all_inds = run_with_ts_metadata( + pyslim.individuals_alive_at, ts_metadata, ts, 0 + ) assert len(all_inds) > 0 for p in range(ts.num_populations): - sub_inds = pyslim.individuals_alive_at(ts, 0, population=p) + sub_inds = pyslim.individuals_alive_at( + ts, 0, population=p, ts_metadata=ts_metadata + ) assert set(sub_inds) == set(all_inds[individual_populations == p]) - sub_inds = pyslim.individuals_alive_at(ts, 0, population=[p]) + sub_inds = pyslim.individuals_alive_at( + ts, 0, population=[p], ts_metadata=ts_metadata + ) assert set(sub_inds) == set(all_inds[individual_populations == p]) - sub_inds = pyslim.individuals_alive_at(ts, 0, population=np.arange(p)) + sub_inds = pyslim.individuals_alive_at( + ts, 0, population=np.arange(p), ts_metadata=ts_metadata + ) assert set(sub_inds) == set(all_inds[individual_populations != p]) @pytest.mark.parametrize( @@ -399,9 +472,14 @@ def test_population(self, recipe): ) def test_samples_only(self, recipe): for _, ts in recipe["ts"].items(): - all_inds = pyslim.individuals_alive_at(ts, 0) + ts_metadata = ts.metadata + all_inds = run_with_ts_metadata( + pyslim.individuals_alive_at, ts_metadata, ts, 0 + ) assert set(all_inds) == set( - pyslim.individuals_alive_at(ts, 0, samples_only=False) + pyslim.individuals_alive_at( + ts, 0, samples_only=False, ts_metadata=ts_metadata + ) ) sub_inds = np.random.choice( all_inds, size=min(len(all_inds), 4), replace=False @@ -440,18 +518,19 @@ def test_after_simplify(self, recipe): @pytest.mark.parametrize("recipe", recipe_eq("pedigree"), indirect=True) def test_ages(self, recipe): for _, ts in recipe["ts"].items(): + ts_metadata = ts.metadata info = recipe["info"] remembered_stage = "late" if "remembered_first" in recipe: remembered_stage = "first" elif "remembered_early" in recipe: remembered_stage = "early" - assert remembered_stage == ts.metadata["SLiM"]["stage"] - max_time_ago = ts.metadata["SLiM"]["tick"] + assert remembered_stage == ts_metadata["SLiM"]["stage"] + max_time_ago = ts_metadata["SLiM"]["tick"] if remembered_stage in ("first", "early"): max_time_ago -= 1 for time in range(0, max_time_ago): - slim_tick = ts.metadata["SLiM"]["tick"] - time + slim_tick = ts_metadata["SLiM"]["tick"] - time check_stages = ("first", "early", "late") if time == 0: if remembered_stage == "first": @@ -470,10 +549,18 @@ def test_ages(self, recipe): check_stages = ("late",) for stage in check_stages: alive = pyslim.individuals_alive_at( - ts, time, stage=stage, remembered_stage=remembered_stage + ts, + time, + stage=stage, + remembered_stage=remembered_stage, + ts_metadata=ts_metadata, ) ages = pyslim.individual_ages_at( - ts, time, stage=stage, remembered_stage=remembered_stage + ts, + time, + stage=stage, + remembered_stage=remembered_stage, + ts_metadata=ts_metadata, ) for ind in ts.individuals(): ind_time = ts.node(ind.nodes[0]).time @@ -489,7 +576,7 @@ def test_ages(self, recipe): assert slim_alive == pyslim_alive if slim_alive: slim_age = info[slim_id]["age"][(slim_tick, stage)] - if ts.metadata["SLiM"]["model_type"] == "WF": + if ts_metadata["SLiM"]["model_type"] == "WF": # SLiM records -1 but we return 0 in late and 1 in early slim_age = 0 + (stage in ("first", "early")) assert ages[ind.id] == slim_age @@ -595,9 +682,10 @@ def test_post_simplify(self, recipe): for _, ts in recipe["ts"].items(): rng = np.random.default_rng(seed=3) individual_times = ts.individuals_time + md_tick = ts.metadata["SLiM"]["tick"] keep_indivs = rng.choice( # assumes tick hasn't been changed - np.where(individual_times < ts.metadata["SLiM"]["tick"] - 1)[0], + np.where(individual_times < md_tick - 1)[0], size=30, replace=False, ) @@ -694,14 +782,66 @@ def test_pedigree_parents(self, recipe): gfolks = [] for a in set(info[sid]["parents"]) - set(ts_p): gfolks.extend(info[a]["parents"]) - print("===== ", hasp, ind) - print("sid: ", sid, "ts_p: ", ts_p) - print("slim_p: ", slim_p) - print(gfolks) + # print("===== ", hasp, ind) + # print("sid: ", sid, "ts_p: ", ts_p) + # print("slim_p: ", slim_p) + # print(gfolks) for a in set(ts_p) - set(slim_p): assert a in gfolks +class TestMutationConsistency(tests.PyslimTestCase): + """ + Test for consistency between what SLiM has written down in top-level metadata + and what's in the tree sequence + """ + + @pytest.mark.parametrize("recipe", recipe_eq("refseq"), indirect=True) + def test_reference_sequence_consistency(self, recipe): + for n, ts in recipe["ts"].items(): + tsmd = ts.metadata + chrom_id = tsmd["SLiM"]["this_chromosome"]["id"] + assert ts.has_reference_sequence() + ref = list( + tsmd["SLiM"]["user_metadata"]["reference_sequence"][0][f"chr{chrom_id}"][ + 0 + ] + ) + ts_ref = ts.reference_sequence.data + subs = [ + x[0] + for x in tsmd["SLiM"]["user_metadata"]["substitutions"][0].values() + if x[0]["chromosome_id"][0] == chrom_id + ] + subs.sort(key=lambda x: (x["position"][0], x["fixationTick"][0])) + for s in subs: + nuc = s["nucleotide"][0] + if nuc != "N": + ref[s["position"][0]] = nuc + assert ts_ref == "".join(ref) + + @pytest.mark.parametrize("recipe", recipe_eq("record_mutations"), indirect=True) + def test_mutation_consistency(self, recipe): + for n, ts in recipe["ts"].items(): + tsmd = ts.metadata + chrom_id = tsmd["SLiM"]["this_chromosome"]["id"] + # this is just making these things not lists, mostly + debug_info = { + int(k): {x: y[0] for x, y in v[0].items()} + for k, v in tsmd["SLiM"]["user_metadata"]["mutations"][0].items() + } + mut_info = pyslim.mutation_metadata(ts) + for mut in ts.mutations(): + for k in mut.derived_state.split(","): + k = int(k) + assert k in debug_info or mut_info[k]["mutation_id"] == 2 + assert k in mut_info + assert debug_info[k]["chromosome_id"] == chrom_id + assert debug_info[k]["position"] == ts.site(mut.site).position + assert debug_info[k]["mutationType"] == mut_info[k]["mutation_type"] + assert debug_info[k]["originTick"] == mut_info[k]["slim_time"] + + class TestReferenceSequence(tests.PyslimTestCase): """ Test for operations involving the reference sequence @@ -749,6 +889,7 @@ def test_nucleotide_at_errors(self, recipe): def test_mutation_at(self, recipe): rng = random.Random(42) for _, ts in recipe["ts"].items(): + L = int(min(50000, ts.sequence_length)) for _ in range(min(10, ts.num_sites)): site = rng.choice(ts.sites()) pos = site.position @@ -766,10 +907,8 @@ def test_mutation_at(self, recipe): a = pyslim.mutation_at(ts, node, pos, time=time) b = naive_mutation_at(ts, node, pos, time=time) assert a == b - for _ in range(min(10, int(ts.sequence_length - ts.num_sites))): - pos = rng.choice( - list(set(range(int(ts.sequence_length))) - set(ts.sites_position)) - ) + for _ in range(min(10, int(L - ts.num_sites))): + pos = rng.choice(list(set(range(L)) - set(ts.sites_position))) tree = ts.at(pos) for _ in range(10): node = rng.randint(0, ts.num_nodes - 1) @@ -778,33 +917,63 @@ def test_mutation_at(self, recipe): for time in [None, ts.node(node).time, ut]: assert naive_mutation_at(ts, node, pos, time=time) == -1 + @pytest.mark.parametrize("recipe", recipe_eq("nucleotides"), indirect=True) def test_nucleotide_at(self, recipe): random.seed(42) for _, ts in recipe["ts"].items(): if ts.num_mutations > 0: + mut_metadata = pyslim.mutation_metadata(ts) mut_md = ts.mutation(0).metadata - has_nucleotides = mut_md["mutation_list"][0]["nucleotide"] >= 0 - if has_nucleotides: - assert ts.has_reference_sequence() - assert len(ts.reference_sequence.data) == ts.sequence_length - for _ in range(100): - node = random.randint(0, ts.num_nodes - 1) - pos = random.randint(0, int(ts.sequence_length) - 1) - tree = ts.at(pos) - parent = tree.parent(node) - a = pyslim.nucleotide_at(ts, node, pos) - if parent == tskit.NULL: - nuc = ts.reference_sequence.data[int(pos)] - assert a == pyslim.NUCLEOTIDES.index(nuc) - else: - b = pyslim.nucleotide_at(ts, parent, pos) - c = pyslim.nucleotide_at(ts, node, pos, ts.node(parent).time) - assert b == c - for k in np.where(node == ts.tables.mutations.node)[0]: - mut = ts.mutation(k) - if ts.site(mut.site).position == pos: - b = mut.metadata["mutation_list"][0]["nucleotide"] - assert a == b + tsmd = ts.metadata + # check we've got nucleotide mutations + nucs = np.array([x["nucleotide"] for x in tsmd["SLiM_mutation_list"]]) + assert np.sum(nucs >= 0) > 1 + mut_info = { + str(mut["mutation_id"]): mut for mut in tsmd["SLiM_mutation_list"] + } + assert ts.has_reference_sequence() + assert len(ts.reference_sequence.data) == ts.sequence_length + for _ in range(100): + node = random.randint(0, ts.num_nodes - 1) + pos = random.randint(0, int(ts.sequence_length) - 1) + tree = ts.at(pos) + parent = tree.parent(node) + a = pyslim.nucleotide_at(ts, node, pos) + if parent == tskit.NULL: + nuc = ts.reference_sequence.data[int(pos)] + assert a == pyslim.NUCLEOTIDES.index(nuc) + else: + b = pyslim.nucleotide_at( + ts, parent, pos, mut_metadata=mut_metadata + ) + c = pyslim.nucleotide_at( + ts, + node, + pos, + ts.node(parent).time, + mut_metadata=mut_metadata, + ) + assert b == c + for k in np.where(node == ts.tables.mutations.node)[0]: + mut = ts.mutation(k) + if ts.site(mut.site).position == pos: + b = mut_info[mut.derived_state.split(",")[0]][ + "nucleotide" + ] + assert a == b + + @pytest.mark.parametrize("recipe", [next(recipe_eq("nucleotides"))], indirect=True) + def test_nucleotide_at_without_mut_metadata(self, recipe): + random.seed(23) + for _, ts in recipe["ts"].items(): + assert ts.num_mutations > 0 + mut_metadata = pyslim.mutation_metadata(ts) + for _ in range(100): + node = random.randint(0, ts.num_nodes - 1) + pos = random.randint(0, int(ts.sequence_length) - 1) + a = pyslim.nucleotide_at(ts, node, pos) + b = pyslim.nucleotide_at(ts, node, pos, mut_metadata=mut_metadata) + assert a == b @pytest.mark.parametrize("recipe", recipe_eq("mutation_spectrum"), indirect=True) def test_nucleotide_spectrum(self, recipe): @@ -814,6 +983,7 @@ def test_nucleotide_spectrum(self, recipe): # access to the parental genome, so if two adjacent mutations # occur in the same meiosis then each will not know about the other. for _, ts in recipe["ts"].items(): + mut_info = pyslim.mutation_metadata(ts) mutation_spectrum = recipe["mutation_info"] M = { a + b + c + "," + d: 0 @@ -827,7 +997,7 @@ def test_nucleotide_spectrum(self, recipe): pos = ts.site(mut.site).position if pos > 0 and pos < ts.sequence_length - 1: nmuts += 1 - mut_list = mut.metadata["mutation_list"] + mut_list = [mut_info[int(k)] for k in mut.derived_state.split(",")] k = np.argmax([u["slim_time"] for u in mut_list]) derived_nuc = mut_list[k]["nucleotide"] left_nuc = pyslim.nucleotide_at( @@ -847,13 +1017,8 @@ def test_nucleotide_spectrum(self, recipe): ) key = context + "," + pyslim.NUCLEOTIDES[derived_nuc] M[key] += 1 - if key == "ACA,T" or key == "CCA,T": - print(key, pos, mut.node, mut.time) assert sum([M[k] for k in M]) == nmuts assert sum([mutation_spectrum[k][0] for k in mutation_spectrum]) == nmuts - for k in M: - if M[k] != mutation_spectrum[k][0]: - print(k, M[k], mutation_spectrum[k]) for k in M: assert len(mutation_spectrum[k]) == 1 assert M[k] == mutation_spectrum[k][0] @@ -869,12 +1034,12 @@ def last_slim_mutations(self, ts): # (slim id, slim mutation metadata) of the slim mutation that is the # *most recent* one of any possibly stacked mutations. Note that it # is possible that this is ambiguous. + mut_info = pyslim.mutation_metadata(ts) for mut in ts.mutations(): slim_muts = { k: v - for k, v in zip( - mut.derived_state.split(","), mut.metadata["mutation_list"] - ) + for k, v in mut_info.items() + if str(k) in mut.derived_state.split(",") } if mut.parent == tskit.NULL: parent_slim_ids = [] @@ -938,10 +1103,7 @@ def scramble_mutations(self, ts): for m in ts.mutations(): a = np.array(m.derived_state.split(",")) ii = rng.permutation(len(a)) - ml = [m.metadata["mutation_list"][i] for i in ii] - t.mutations.append( - m.replace(derived_state=",".join(a[ii]), metadata={"mutation_list": ml}) - ) + t.mutations.append(m.replace(derived_state=",".join(a[ii]))) t.compute_mutation_parents() return t.tree_sequence() @@ -956,6 +1118,7 @@ def test_convert_alleles_errors(self): ts, model=msprime.SLiMMutationModel(type=1), rate=0.1, random_seed=23 ) assert mts.num_mutations > 0 + mts = pyslim.add_mutation_metadata(mts) mtt = mts.dump_tables() mtt.reference_sequence.data = "A" * int(mts.sequence_length) mts = mtt.tree_sequence() @@ -967,6 +1130,7 @@ def test_convert_alleles_errors(self): ) def test_convert_alleles(self, recipe): for _, ts in recipe["ts"].items(): + verify_mutation_metadata(ts) cts = pyslim.convert_alleles(ts) self.verify_converted_nucleotides(ts, cts) @@ -1021,18 +1185,19 @@ def test_generate_nucleotides_errors(self): def verify_generate_nucleotides(self, ts, check_transitions=False): # if check_transitions is True, verify that derived states differ # from parental states - which we try to do but is not guaranteed, - # for instance, if keep=True or in other weird situations. + # for instance, if keep=True, there was more than one mutation in + # single generation, or in other weird situations. assert ts.metadata["SLiM"]["nucleotide_based"] assert len(ts.reference_sequence.data) == ts.sequence_length + mut_info = pyslim.mutation_metadata(ts) muts = {} ts_muts = { j: v["nucleotide"] for j, (_, v) in enumerate(self.last_slim_mutations(ts)) } for mut in ts.mutations(): aa = ts.reference_sequence.data[int(ts.site(mut.site).position)] - for i, md in zip( - mut.derived_state.split(","), mut.metadata["mutation_list"] - ): + for i in mut.derived_state.split(","): + md = mut_info[int(i)] nuc = md["nucleotide"] assert nuc in [0, 1, 2, 3] if i in muts: @@ -1042,9 +1207,14 @@ def verify_generate_nucleotides(self, ts, check_transitions=False): if mut.parent == tskit.NULL: assert pyslim.NUCLEOTIDES[nuc] != aa else: - if ts.mutation(mut.parent).derived_state != mut.derived_state: - assert ts_muts[mut.parent] != ts_muts[mut.id] - + mp = ts.mutation(mut.parent) + if mp.derived_state != mut.derived_state: + assert (ts_muts[mut.parent] != ts_muts[mut.id]) or ( + len(mut.derived_state.split(",")) + > 1 + len(mp.derived_state.split(",")) + ) + + @pytest.mark.parametrize("recipe", recipe_eq(exclude="old_mutations"), indirect=True) def test_generate_nucleotides(self, recipe): for _, ts in recipe["ts"].items(): nts = pyslim.generate_nucleotides(ts, keep=False, seed=5) @@ -1061,8 +1231,10 @@ def test_generate_nucleotides_refseq(self): random_seed=10, ) ts = pyslim.annotate(ts, model_type="nonWF", tick=1) - mts = msprime.sim_mutations( - ts, model=msprime.SLiMMutationModel(type=1), rate=0.5, random_seed=23 + mts = pyslim.add_mutation_metadata( + msprime.sim_mutations( + ts, model=msprime.SLiMMutationModel(type=1), rate=0.5, random_seed=23 + ) ) refseq = "A" * int(mts.sequence_length) nts = pyslim.generate_nucleotides(mts, reference_sequence=refseq, seed=6) @@ -1072,35 +1244,42 @@ def test_generate_nucleotides_refseq(self): def test_generate_nucleotides_keep(self): ts = msprime.sim_ancestry(4, sequence_length=10, population_size=10) ts = pyslim.annotate(ts, model_type="nonWF", tick=1) - mts1 = msprime.sim_mutations( - ts, model=msprime.SLiMMutationModel(type=1), rate=0.1, random_seed=23 + mts1 = pyslim.add_mutation_metadata( + msprime.sim_mutations( + ts, model=msprime.SLiMMutationModel(type=1), rate=0.1, random_seed=23 + ) ) - mts1.dump("out.trees") nts1 = pyslim.generate_nucleotides(mts1, seed=10, keep=False) assert nts1.num_mutations > 0 self.verify_generate_nucleotides(nts1, check_transitions=False) - mts2 = msprime.sim_mutations( - nts1, - model=msprime.SLiMMutationModel( - type=2, - next_id=nts1.num_mutations, - ), - rate=0.1, - random_seed=24, + mut_info1 = { + str(mut["mutation_id"]): mut for mut in nts1.metadata["SLiM_mutation_list"] + } + mts2 = pyslim.add_mutation_metadata( + msprime.sim_mutations( + nts1, + model=msprime.SLiMMutationModel( + type=2, + next_id=nts1.num_mutations, + ), + rate=0.1, + random_seed=24, + ) ) # keep defaults to True nts2 = pyslim.generate_nucleotides(mts2, seed=12) assert nts2.num_mutations > nts1.num_mutations + mut_info2 = { + str(mut["mutation_id"]): mut for mut in nts2.metadata["SLiM_mutation_list"] + } muts1 = {} for mut in nts1.mutations(): - for i, md in zip( - mut.derived_state.split(","), mut.metadata["mutation_list"] - ): + for i in mut.derived_state.split(","): + md = mut_info1[i] muts1[i] = md["nucleotide"] for mut in nts2.mutations(): - for i, md in zip( - mut.derived_state.split(","), mut.metadata["mutation_list"] - ): + for i in mut.derived_state.split(","): + md = mut_info2[i] if md["mutation_type"] == 1: assert i in muts1 assert muts1[i] == md["nucleotide"] @@ -1160,8 +1339,9 @@ def get_vacant_samples(self, ts): def verify_remove_vacant(self, ts, rts): vacant_samples = self.get_vacant_samples(ts) - for node in rts.nodes(): - assert not (pyslim.node_is_vacant(rts, node) and (node.is_sample() == 1)) + vacant = pyslim.nodes_vacant(ts) + for v, node in zip(vacant, rts.nodes()): + assert not (v and (node.is_sample() == 1)) assert (node.id in vacant_samples) == ( node.flags & pyslim.NODE_IS_VACANT_SAMPLE > 0 ) @@ -1262,14 +1442,23 @@ def test_has_vacant_msprime(self): tables.nodes.metadata_schema = pyslim.slim_metadata_schemas["node"] assert not pyslim.has_vacant_samples(tables.tree_sequence()) - def test_node_is_vacant(self, recipe): + def test_nodes_vacant(self, recipe): + np.random.seed(123) num_chromosomes = len(recipe["ts"]) for _, ts in recipe["ts"].items(): + test_nodes = np.random.choice( + np.arange(ts.num_nodes), size=min(1, ts.num_nodes), replace=False + ) k = ts.metadata["SLiM"]["this_chromosome"]["index"] - for node in ts.nodes(): + vacant = pyslim.nodes_vacant(ts) + for pv, node in zip(vacant, ts.nodes()): v = self.vacancy_values(node) isv = v is not None and v[k] - assert isv == pyslim.node_is_vacant(ts, node) + assert pv == isv + if node.id in test_nodes: + # node_is_vacant is kinda slow so don't test all of them + with pytest.warns(FutureWarning, match="deprecated"): + assert isv == pyslim.node_is_vacant(ts, node) for j in range(num_chromosomes, len(v)): assert not v[j] @@ -1394,6 +1583,16 @@ def test_no_change(self, restart_name, recipe, helper_functions, tmp_path): ) for chrom, ts in recipe["ts"].items(): self.verify_reset(ts, out_ts[chrom]) + # again!! + out_out_ts = helper_functions.run_slim_restart( + out_ts, + restart_name, + tmp_path, + "multichrom" in recipe, + WF="WF" in recipe, + ) + for chrom, ts in recipe["ts"].items(): + self.verify_reset(ts, out_out_ts[chrom]) @pytest.mark.parametrize( "restart_name, recipe", restarted_recipe_eq("no_op"), indirect=["recipe"] @@ -1425,16 +1624,20 @@ def test_set_individuals( ): in_ts = {} ts = list(recipe["ts"].values())[0] + tsmd = ts.metadata assert ( - "user_metadata" in ts.metadata["SLiM"] - and "reset_tick" in ts.metadata["SLiM"]["user_metadata"] + "user_metadata" in tsmd["SLiM"] + and "reset_tick" in tsmd["SLiM"]["user_metadata"] ), "Simulation not set up for this test." - reset_tick = ts.metadata["SLiM"]["user_metadata"]["reset_tick"][0] + reset_tick = tsmd["SLiM"]["user_metadata"]["reset_tick"][0] if time is None: - for time in range(ts.metadata["SLiM"]["tick"] + 1): - if pyslim.slim_time(ts, time) == reset_tick: - break - individuals = pyslim.individuals_alive_at(ts, time)[:num_indivs] + pytimes = run_with_ts_metadata( + pyslim.slim_time, tsmd, ts, np.arange(tsmd["SLiM"]["tick"] + 1) + ) + time = np.searchsorted(pytimes, reset_tick) + individuals = pyslim.individuals_alive_at(ts, time, ts_metadata=tsmd)[ + :num_indivs + ] for chrom, ts in recipe["ts"].items(): in_ts[chrom] = pyslim.set_slim_state(ts, time=time, individuals=individuals) out_ts = helper_functions.run_slim_restart( diff --git a/uv.lock b/uv.lock index 68ae0211..fbf98b33 100644 --- a/uv.lock +++ b/uv.lock @@ -645,6 +645,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/4e/ce75a57ff3aebf6fc1f4e9d508b8e5810618a33d900ad6c19eb30b290b97/fonttools-4.61.1-py3-none-any.whl", hash = "sha256:17d2bf5d541add43822bcf0c43d7d847b160c9bb01d15d5007d84e2217aaa371", size = 1148996, upload-time = "2025-12-12T17:31:21.03Z" }, ] +[[package]] +name = "frozendict" +version = "2.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/90/b2/2a3d1374b7780999d3184e171e25439a8358c47b481f68be883c14086b4c/frozendict-2.4.7.tar.gz", hash = "sha256:e478fb2a1391a56c8a6e10cc97c4a9002b410ecd1ac28c18d780661762e271bd", size = 317082, upload-time = "2025-11-11T22:40:14.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/74/f94141b38a51a553efef7f510fc213894161ae49b88bffd037f8d2a7cb2f/frozendict-2.4.7-py3-none-any.whl", hash = "sha256:972af65924ea25cf5b4d9326d549e69a9a4918d8a76a9d3a7cd174d98b237550", size = 16264, upload-time = "2025-11-11T22:40:12.836Z" }, +] + [[package]] name = "greenlet" version = "3.3.2" @@ -1844,6 +1853,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "filelock" }, + { name = "frozendict" }, { name = "jupyter-book" }, { name = "matplotlib" }, { name = "msprime" }, @@ -1879,6 +1889,7 @@ packaging = [ ] test = [ { name = "filelock" }, + { name = "frozendict" }, { name = "msprime" }, { name = "pandas" }, { name = "pytest" }, @@ -1891,12 +1902,13 @@ test = [ requires-dist = [ { name = "msprime", specifier = ">=1.0.1" }, { name = "numpy" }, - { name = "tskit" }, + { name = "tskit", specifier = ">=1.0.3" }, ] [package.metadata.requires-dev] dev = [ { name = "filelock" }, + { name = "frozendict" }, { name = "jupyter-book", specifier = "<2" }, { name = "matplotlib" }, { name = "msprime" }, @@ -1932,6 +1944,7 @@ packaging = [ ] test = [ { name = "filelock" }, + { name = "frozendict" }, { name = "msprime" }, { name = "pandas" }, { name = "pytest" }, @@ -2798,23 +2811,38 @@ wheels = [ [[package]] name = "tskit" -version = "0.6.4" +version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jsonschema" }, { name = "numpy" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/94/95/2c2d8bdaae4a3948181de68d1fac0569d9c937a42f7ccfcf097f9a428721/tskit-0.6.4.tar.gz", hash = "sha256:bdac1bb7e3ae3d1f562ec191b5d840156e082dd2adc6af7c41b170c4fb1be792", size = 874772, upload-time = "2025-05-21T18:18:18.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/28/547acef423709fad5b70bbc68332c1fa0a4c1887ec4e76cb93a433a95bbb/tskit-0.6.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:510fd219f2c6d5d669e178a3ea1e4cd60d7fc0b40a50fae4859a6a8f35d2394a", size = 748024, upload-time = "2025-05-21T18:17:54.919Z" }, - { url = "https://files.pythonhosted.org/packages/88/0d/5816ee9ac9708f2e1def387e36b2d0763bdea2267db3ff4a55c17f079e85/tskit-0.6.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b631d38c352b618c3ede33515cf97ffcbc24e3a4252b6979a80b1770434a534", size = 1322885, upload-time = "2025-05-21T18:17:56.958Z" }, - { url = "https://files.pythonhosted.org/packages/bd/db/b98964916b3f9c603f8ad92045db0c050ca4699a976051acdc1368085c3c/tskit-0.6.4-cp311-cp311-win_amd64.whl", hash = "sha256:e121226092816a1e36b2835aa0348e652c09bdf0604a936f63c4fd0d64ca6422", size = 470677, upload-time = "2025-05-21T18:17:58.584Z" }, - { url = "https://files.pythonhosted.org/packages/b4/cf/d2a1c6a6ad29b16310b60bf08dd5b1d1e4e0d23819c49284662373c9fc54/tskit-0.6.4-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:8fd5a0c94f302f5fe69a9f7a662b8ddf8219e4dc7325a1990b279afd084bb649", size = 749765, upload-time = "2025-05-21T18:18:00.234Z" }, - { url = "https://files.pythonhosted.org/packages/80/46/1630514e8a9f97a8f75e805a7872fb3083119d7e421309e05e99d5f5ca66/tskit-0.6.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f086e648a624004343882ca57f9365d5005cc2e472113b61874c9403c14be272", size = 1324515, upload-time = "2025-05-21T18:18:02.461Z" }, - { url = "https://files.pythonhosted.org/packages/07/44/2116904f37ffe1db0e675b4c70b70cdaf7ab38b474cd546f6ff9d46de4b9/tskit-0.6.4-cp312-cp312-win_amd64.whl", hash = "sha256:507eee5b20c5e47202d90a70143c601aef3b3ec70321b5251c2e6d896b1e0722", size = 470047, upload-time = "2025-05-21T18:18:04.185Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d7/5ee3b118281f2a7032f5d38fca0ee7e11bf7cf1823d2dbfa5a991ccf3497/tskit-0.6.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:66f07b76f18ad576a7585b6e4d9e46d58994e3df1dd2d1808d32d294354695cf", size = 749773, upload-time = "2025-05-21T18:18:06.48Z" }, - { url = "https://files.pythonhosted.org/packages/f2/6d/6368c2ffbccb4d7ee8d82096f1ad0b3cb0d639b958c408589cceff09600e/tskit-0.6.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfb40717382cf47b8844c1f73d73c1e7ca7f63e6a506e7f859f0980667677de2", size = 1324529, upload-time = "2025-05-21T18:18:08.674Z" }, - { url = "https://files.pythonhosted.org/packages/ee/b9/5b092c5b409cad0005b560622b34f9fd38fe0945939974671306fedef30a/tskit-0.6.4-cp313-cp313-win_amd64.whl", hash = "sha256:c6b1f1b22e5d55a906ee33bca33447500df0cf5b12ec20926773ca1e3c3c8931", size = 470097, upload-time = "2025-05-21T18:18:11.022Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/c23333b3ffd86cbe79520bbee85f050f815c16a625b014dbacbe486fb6c2/tskit-1.0.3.tar.gz", hash = "sha256:8a305b3bcf4145688be17962f6ea2ac20aa65db440ed0c02b49f9b4a3c00836b", size = 933435, upload-time = "2026-05-14T18:20:16.89Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/6c/7fdf85e62204319d77da52d2647e1e65c6dc94660a27d7f202fd134f42c9/tskit-1.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a5d5f5e0d0ad53ac554563c76b91c5f2bb721d021b7c761606a9a72fe1e49d74", size = 526356, upload-time = "2026-05-14T18:19:40.482Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ce/4ff9ba98eca28992250fa8eab25db5bc4a3c7b3af4ad46652f7d1b5b22c3/tskit-1.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:af978aa415f04fbabd613ad2933526110b4767e8e64f319ec03cefca9e2242bd", size = 496008, upload-time = "2026-05-14T18:19:42.332Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3d/d4d8567a2bca51df46f0c7ad706678b106ce2ba094d8584d8829a24451c0/tskit-1.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f509b7b271dd80a68ee3f6bfdff614e3b8ac989dcc44dd8d36c25c8181443b79", size = 1395725, upload-time = "2026-05-14T18:19:43.83Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f3/d5ce78eeeda65c4673966d00567f4166b9500843ce953c0dad927a2200c3/tskit-1.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0722a4c1fc9c287c96a2141ae3587df6defed2a049cca7ab90479958c613835f", size = 1377087, upload-time = "2026-05-14T18:19:45.238Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7c/9e53fb9f71bf89eb8327cb2a859d4c9764dc447a42d3407f7ec7c7a25fb8/tskit-1.0.3-cp311-cp311-win32.whl", hash = "sha256:3901c9fc02497e2c7e0ee5ca36ce5b65749f9c630e5ace1daa990963d8f5a63d", size = 449670, upload-time = "2026-05-14T18:19:46.641Z" }, + { url = "https://files.pythonhosted.org/packages/72/88/a80beb0adfb8ceade30b1913010c56e6bd4f837ef2f75aeb76222377d148/tskit-1.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:0d374768d422e941f8ad6ffdba85d6ca1585467532f6f4945b171f18d6613285", size = 496418, upload-time = "2026-05-14T18:19:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/56/1f/d6726034289911b326e74b8b5339c9d1ec6fe46bf203253ae8d90b146a89/tskit-1.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3d035946888c9da5eefb93ca99c2ae3ad2cb7cb64bc6037588fe285f775ee67a", size = 528485, upload-time = "2026-05-14T18:19:49.299Z" }, + { url = "https://files.pythonhosted.org/packages/ad/ed/7fa3f43bc0d7569b75bc6a5c03bc14bd2e9de12a3f6a2f8b3251d505226a/tskit-1.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:31aee6a8b6d9a0bdae06f0b186fb7aad54d53f34050c0ee72a6caa560506950f", size = 496860, upload-time = "2026-05-14T18:19:50.716Z" }, + { url = "https://files.pythonhosted.org/packages/28/b1/fb828d0479be0f178e9f33b4483003a421b069e6331dc5f35066dc3029e4/tskit-1.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb07ead500c7e88c69677469b4f489c8997ad944103102b04d724c855b4b980b", size = 1398919, upload-time = "2026-05-14T18:19:52.1Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3e/1c7adaae07a00ce009e692a6d171c7a0882d1ed9753017b78a7db7ae7d72/tskit-1.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:241192b12d310e49b21bfae524c76768a15d0ed3f7b950d6ee09dce1528b8973", size = 1378470, upload-time = "2026-05-14T18:19:53.881Z" }, + { url = "https://files.pythonhosted.org/packages/71/a1/10d12e9b7908775cf97ad9e6c3d94b4b970940c74a161cc9aaf540fa83c0/tskit-1.0.3-cp312-cp312-win32.whl", hash = "sha256:3e11c9fe328b27c0e4a3ff64a0fed972d2cd48a743c03c8bc339352664322ee8", size = 450010, upload-time = "2026-05-14T18:19:55.419Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b7/d1842bec89cd4993958165d6da774e85ad79b9426baca571c39457a1b6b5/tskit-1.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:66c83c7f971bc160fd29ca6169e1ee31023a5283ca88936df3eed1e568eee209", size = 496000, upload-time = "2026-05-14T18:19:56.75Z" }, + { url = "https://files.pythonhosted.org/packages/e6/10/aa8cc65060669b911946649cd1802f29640998c5b6ca78c4c845fb3c473d/tskit-1.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d52e2ad0eb79e3b21da8a236d0f5eb888f9b18569e48f4b84a3875523e80ee22", size = 528459, upload-time = "2026-05-14T18:19:58.156Z" }, + { url = "https://files.pythonhosted.org/packages/eb/42/34d981123a335d80e3030e9b0a81a41f654c9cf1c5db0ac630d6b20d8bfd/tskit-1.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5bd1744a020a518e5835b85971fea14ca827d76f778650eb37e3f44c6db0a711", size = 496838, upload-time = "2026-05-14T18:19:59.909Z" }, + { url = "https://files.pythonhosted.org/packages/65/9c/09b072c01b8e58a3137011851d4761daf957f2f86447bc2a2c25e0d5aa4a/tskit-1.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:405c20fad5256edd6abd1f8b514c5bc653d808102f509c4512c9b174507dbe84", size = 1399080, upload-time = "2026-05-14T18:20:01.353Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bb/789b91f7a2deca4d50f649b685e2b3dca4b3358e36bf7a5b2b84709bb8d5/tskit-1.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bc1c97f00ce8e79460aa9ff3b36987daf70f2dc267e36a32db77018dbefc822b", size = 1378631, upload-time = "2026-05-14T18:20:03.273Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8b/3b2b19246b2ba3159456e4068887e505d7a2ec857ecd7ae7606d6c129fda/tskit-1.0.3-cp313-cp313-win32.whl", hash = "sha256:53444ef3b2fd7ea9ca863e562c33402e46a55e645887dd4d983686b607e3f788", size = 449996, upload-time = "2026-05-14T18:20:04.788Z" }, + { url = "https://files.pythonhosted.org/packages/23/fc/53ef7ad8b3f2183584af81b0ff046a660c3164530a1f768770dbd4d60cb7/tskit-1.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:721ae7f02730ed91233a6da2a9d4b59632d1512877cd0d6fe948ea5d9ddc17e9", size = 496006, upload-time = "2026-05-14T18:20:06.577Z" }, + { url = "https://files.pythonhosted.org/packages/66/b9/91194216907ebf278d7a58288f3e64881e11a0eedcca7125a49e0bbdafdd/tskit-1.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:62a45d5e57f5a9181eb113c0dd478783ef3c4829856c0831ea2eba2cdf00a8de", size = 528594, upload-time = "2026-05-14T18:20:08.032Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2c/0a888f4f140922c7a3945dbca5e444fa8052a620c784359673fe9eb1eda7/tskit-1.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8df716837129652fc82d095d80fd4e23a728213a4b18de9233f3aebb073b60b4", size = 496866, upload-time = "2026-05-14T18:20:09.34Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ce/56aa69b768734ede0b8b36c5ea2ce842d50c786b74cfefdfc036a1fd29da/tskit-1.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:151eefbbae9a339f3ea64980937fa89d6ac9fb371dba880a9b30791a461753ad", size = 1397616, upload-time = "2026-05-14T18:20:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/5a/b05eab4a4a32bce6a51f991033ed460a836aafc1f406eebc624831558d9b/tskit-1.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2170cacd8ade47fd8f1510aca78549e09e9ba875c25496f0ccfb755e09c0b06b", size = 1377477, upload-time = "2026-05-14T18:20:12.649Z" }, + { url = "https://files.pythonhosted.org/packages/ea/c5/116419617468dc06428e507001d0a7b212116e9e8d336bd8c05a6531b2a0/tskit-1.0.3-cp314-cp314-win32.whl", hash = "sha256:24f8fffc1e1dd4154654e91bb7bac22078b3cadd9837e5c490225aa5c313b819", size = 455586, upload-time = "2026-05-14T18:20:14.165Z" }, + { url = "https://files.pythonhosted.org/packages/6e/7f/61818ed922629178abb07e45439d0af3bb5867b629fe392e9864df58ce5c/tskit-1.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:cbe791757cbd4d060a9c06c270bbcacc932f60ce1d77e98dcea35c931d0b0aa9", size = 505409, upload-time = "2026-05-14T18:20:15.527Z" }, ] [[package]]