From 0db746b4ab70e8e4e87407cece3c5571a1fdf532 Mon Sep 17 00:00:00 2001 From: michalm Date: Sun, 10 Oct 2021 19:35:57 +0200 Subject: [PATCH] [RoseTTAFold] Initial release --- .../DrugDiscovery/RoseTTAFold/Dockerfile | 73 ++ DGLPyTorch/DrugDiscovery/RoseTTAFold/LICENSE | 21 + .../RoseTTAFold/README-ROSETTAFOLD.md | 94 ++ .../DrugDiscovery/RoseTTAFold/README.md | 138 +++ .../RoseTTAFold/RoseTTAFold-linux-cu101.yml | 107 +++ .../RoseTTAFold/RoseTTAFold-linux.yml | 108 +++ .../RoseTTAFold/example/input.fa | 2 + .../RoseTTAFold/folding-linux.yml | 9 + .../DrugDiscovery/RoseTTAFold/folding/README | 43 + .../RoseTTAFold/folding/RosettaTR.py | 340 +++++++ .../RoseTTAFold/folding/arguments.py | 38 + .../RoseTTAFold/folding/data/params.json | 17 + .../RoseTTAFold/folding/data/relax_round1.txt | 17 + .../RoseTTAFold/folding/data/relax_round2.txt | 8 + .../RoseTTAFold/folding/data/scorefxn.wts | 7 + .../RoseTTAFold/folding/data/scorefxn1.wts | 7 + .../folding/data/scorefxn_cart.wts | 9 + .../RoseTTAFold/folding/data/scorefxn_vdw.wts | 2 + .../RoseTTAFold/folding/utils.py | 373 ++++++++ .../DrugDiscovery/RoseTTAFold/funding.md | 1 + .../images/NetworkArchitecture.jpg | Bin 0 -> 117715 bytes .../RoseTTAFold/input_prep/make_msa.sh | 57 ++ .../RoseTTAFold/input_prep/make_ss.sh | 29 + .../input_prep/prepare_templates.sh | 18 + .../RoseTTAFold/install_dependencies.sh | 27 + .../network/Attention_module_w_str.py | 480 ++++++++++ .../RoseTTAFold/network/DistancePredictor.py | 36 + .../RoseTTAFold/network/Embeddings.py | 175 ++++ .../RoseTTAFold/network/InitStrGenerator.py | 116 +++ .../RoseTTAFold/network/Refine_module.py | 175 ++++ .../RoseTTAFold/network/RoseTTAFoldModel.py | 131 +++ .../RoseTTAFold/network/SE3_network.py | 108 +++ .../RoseTTAFold/network/Transformer.py | 480 ++++++++++ .../network/equivariant_attention/fibers.py | 163 ++++ .../equivariant_attention/from_se3cnn/SO3.py | 289 ++++++ .../from_se3cnn/cache/trans_Q/0.pkl.gz | Bin 0 -> 346 bytes .../from_se3cnn/cache/trans_Q/1.pkl.gz | Bin 0 -> 420 bytes .../from_se3cnn/cache/trans_Q/10.pkl.gz | Bin 0 -> 546 bytes .../from_se3cnn/cache/trans_Q/11.pkl.gz | Bin 0 -> 696 bytes .../from_se3cnn/cache/trans_Q/12.pkl.gz | Bin 0 -> 659 bytes .../from_se3cnn/cache/trans_Q/13.pkl.gz | Bin 0 -> 765 bytes .../from_se3cnn/cache/trans_Q/14.pkl.gz | Bin 0 -> 539 bytes .../from_se3cnn/cache/trans_Q/15.pkl.gz | Bin 0 -> 658 bytes .../from_se3cnn/cache/trans_Q/16.pkl.gz | Bin 0 -> 779 bytes .../from_se3cnn/cache/trans_Q/17.pkl.gz | Bin 0 -> 950 bytes .../from_se3cnn/cache/trans_Q/18.pkl.gz | Bin 0 -> 1106 bytes .../from_se3cnn/cache/trans_Q/2.pkl.gz | Bin 0 -> 422 bytes .../from_se3cnn/cache/trans_Q/3.pkl.gz | Bin 0 -> 427 bytes .../from_se3cnn/cache/trans_Q/4.pkl.gz | Bin 0 -> 565 bytes .../from_se3cnn/cache/trans_Q/5.pkl.gz | Bin 0 -> 699 bytes .../from_se3cnn/cache/trans_Q/6.pkl.gz | Bin 0 -> 545 bytes .../from_se3cnn/cache/trans_Q/7.pkl.gz | Bin 0 -> 697 bytes .../from_se3cnn/cache/trans_Q/8.pkl.gz | Bin 0 -> 662 bytes .../from_se3cnn/cache/trans_Q/9.pkl.gz | Bin 0 -> 783 bytes .../from_se3cnn/cache/trans_Q/index.pkl | Bin 0 -> 557 bytes .../from_se3cnn/cache/trans_Q/mutex | 1 + .../from_se3cnn/cache_file.py | 112 +++ .../from_se3cnn/license.txt | 25 + .../from_se3cnn/representations.py | 249 +++++ .../from_se3cnn/utils_steerable.py | 326 +++++++ .../network/equivariant_attention/modules.py | 905 ++++++++++++++++++ .../RoseTTAFold/network/ffindex.py | 91 ++ .../RoseTTAFold/network/kinematics.py | 216 +++++ .../RoseTTAFold/network/parsers.py | 255 +++++ .../RoseTTAFold/network/performer_pytorch.py | 284 ++++++ .../RoseTTAFold/network/predict_complex.py | 317 ++++++ .../RoseTTAFold/network/predict_e2e.py | 324 +++++++ .../RoseTTAFold/network/predict_pyRosetta.py | 200 ++++ .../RoseTTAFold/network/resnet.py | 95 ++ .../RoseTTAFold/network/trFold.py | 252 +++++ .../DrugDiscovery/RoseTTAFold/network/util.py | 253 +++++ .../RoseTTAFold/network/utils/utils_data.py | 64 ++ .../network/utils/utils_logging.py | 123 +++ .../network/utils/utils_profiling.py | 5 + .../RoseTTAFold/notebooks/run_inference.ipynb | 267 ++++++ .../RoseTTAFold/pipeline_utils.py | 42 + .../RoseTTAFold/requirements.txt | 4 + .../DrugDiscovery/RoseTTAFold/run_e2e_ver.sh | 78 ++ .../RoseTTAFold/run_inference_pipeline.py | 17 + .../RoseTTAFold/run_inference_pipeline.sh | 76 ++ .../RoseTTAFold/run_pyrosetta_ver.sh | 123 +++ .../RoseTTAFold/scripts/download_databases.sh | 33 + .../RoseTTAFold/start_jupyter.sh | 11 + 83 files changed, 8446 insertions(+) create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/Dockerfile create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/LICENSE create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/README-ROSETTAFOLD.md create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/README.md create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux-cu101.yml create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux.yml create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/example/input.fa create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding-linux.yml create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/README create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/RosettaTR.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/arguments.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/params.json create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round1.txt create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round2.txt create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn.wts create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn1.wts create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_cart.wts create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_vdw.wts create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/utils.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/funding.md create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/images/NetworkArchitecture.jpg create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_msa.sh create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_ss.sh create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/prepare_templates.sh create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/install_dependencies.sh create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Attention_module_w_str.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/DistancePredictor.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Embeddings.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/InitStrGenerator.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Refine_module.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/RoseTTAFoldModel.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/SE3_network.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Transformer.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/fibers.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/SO3.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/0.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/1.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/10.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/11.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/12.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/13.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/14.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/15.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/16.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/17.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/18.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/2.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/3.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/4.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/5.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/6.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/7.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/8.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/9.pkl.gz create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/index.pkl create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/mutex create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache_file.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/license.txt create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/representations.py create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/utils_steerable.py create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/modules.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/ffindex.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/kinematics.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/parsers.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/performer_pytorch.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_complex.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_e2e.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_pyRosetta.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/resnet.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/trFold.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/util.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_data.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_logging.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_profiling.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/notebooks/run_inference.ipynb create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/pipeline_utils.py create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/requirements.txt create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/run_e2e_ver.sh create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.py create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.sh create mode 100755 DGLPyTorch/DrugDiscovery/RoseTTAFold/run_pyrosetta_ver.sh create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/scripts/download_databases.sh create mode 100644 DGLPyTorch/DrugDiscovery/RoseTTAFold/start_jupyter.sh diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/Dockerfile b/DGLPyTorch/DrugDiscovery/RoseTTAFold/Dockerfile new file mode 100644 index 00000000..b7c202e1 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/Dockerfile @@ -0,0 +1,73 @@ +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:21.09-py3 + +FROM ${FROM_IMAGE_NAME} AS dgl_builder +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y git build-essential python3-dev make cmake \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /dgl +RUN git clone --branch v0.7.0 --recurse-submodules --depth 1 https://github.com/dmlc/dgl.git . +RUN sed -i 's/"35 50 60 70"/"60 70 80"/g' cmake/modules/CUDA.cmake +WORKDIR build +RUN cmake -DUSE_CUDA=ON -DUSE_FP16=ON .. +RUN make -j8 + + +FROM ${FROM_IMAGE_NAME} + +# VERY IMPORTANT, DO NOT REMOVE: +ENV FORCE_CUDA 1 +RUN pip install -v torch-geometric +RUN pip install -v torch-scatter +RUN pip install -v torch-sparse +RUN pip install -v torch-cluster +RUN pip install -v torch-spline-conv + + +# copy built DGL and install it +COPY --from=dgl_builder /dgl ./dgl +RUN cd dgl/python && python setup.py install && cd ../.. && rm -rf dgl +ENV DGLBACKEND=pytorch +#RUN pip install dgl-cu111 -f https://data.dgl.ai/wheels/repo.html + + +# HH-Suite +RUN git clone https://github.com/soedinglab/hh-suite.git && \ + mkdir -p hh-suite/build +WORKDIR hh-suite/build +RUN cmake .. && \ + make && \ + make install + + +# PSIPRED +WORKDIR /workspace +RUN wget http://wwwuser.gwdg.de/~compbiol/data/csblast/releases/csblast-2.2.3_linux64.tar.gz -O csblast-2.2.3.tar.gz && \ +mkdir -p csblast-2.2.3 && \ +tar xf csblast-2.2.3.tar.gz -C csblast-2.2.3 --strip-components=1 && \ +rm csblast-2.2.3.tar.gz + +RUN wget https://ftp.ncbi.nlm.nih.gov/blast/executables/legacy.NOTSUPPORTED/2.2.26/blast-2.2.26-x64-linux.tar.gz && \ + tar xf blast-2.2.26-x64-linux.tar.gz && \ + rm blast-2.2.26-x64-linux.tar.gz + +RUN wget http://bioinfadmin.cs.ucl.ac.uk/downloads/psipred/psipred.4.02.tar.gz && \ + tar xf psipred.4.02.tar.gz && \ + rm psipred.4.02.tar.gz + + +ADD . /workspace/rf +WORKDIR /workspace/rf + +RUN wget https://openstructure.org/static/lddt-linux.zip -O lddt.zip && unzip -d lddt -j lddt.zip + +RUN pip install --upgrade pip +RUN pip install -r requirements.txt diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/LICENSE b/DGLPyTorch/DrugDiscovery/RoseTTAFold/LICENSE new file mode 100644 index 00000000..b967a3bc --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 RosettaCommons + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/README-ROSETTAFOLD.md b/DGLPyTorch/DrugDiscovery/RoseTTAFold/README-ROSETTAFOLD.md new file mode 100644 index 00000000..2872606d --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/README-ROSETTAFOLD.md @@ -0,0 +1,94 @@ +# *RoseTTAFold* +This package contains deep learning models and related scripts to run RoseTTAFold. +This repository is the official implementation of RoseTTAFold: Accurate prediction of protein structures and interactions using a 3-track network. + +## Installation + +1. Clone the package +``` +git clone https://github.com/RosettaCommons/RoseTTAFold.git +cd RoseTTAFold +``` + +2. Create conda environment using `RoseTTAFold-linux.yml` file and `folding-linux.yml` file. The latter is required to run a pyrosetta version only (run_pyrosetta_ver.sh). +``` +# create conda environment for RoseTTAFold +# If your NVIDIA driver compatible with cuda11 +conda env create -f RoseTTAFold-linux.yml +# If not (but compatible with cuda10) +conda env create -f RoseTTAFold-linux-cu101.yml + +# create conda environment for pyRosetta folding & running DeepAccNet +conda env create -f folding-linux.yml +``` + +3. Download network weights (under Rosetta-DL Software license -- please see below) +While the code is licensed under the MIT License, the trained weights and data for RoseTTAFold are made available for non-commercial use only under the terms of the Rosetta-DL Software license. You can find details at https://files.ipd.uw.edu/pub/RoseTTAFold/Rosetta-DL_LICENSE.txt + +``` +wget https://files.ipd.uw.edu/pub/RoseTTAFold/weights.tar.gz +tar xfz weights.tar.gz +``` + +4. Download and install third-party software. +``` +./install_dependencies.sh +``` + +5. Download sequence and structure databases +``` +# uniref30 [46G] +wget http://wwwuser.gwdg.de/~compbiol/uniclust/2020_06/UniRef30_2020_06_hhsuite.tar.gz +mkdir -p UniRef30_2020_06 +tar xfz UniRef30_2020_06_hhsuite.tar.gz -C ./UniRef30_2020_06 + +# BFD [272G] +wget https://bfd.mmseqs.com/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt.tar.gz +mkdir -p bfd +tar xfz bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt.tar.gz -C ./bfd + +# structure templates (including *_a3m.ffdata, *_a3m.ffindex) [over 100G] +wget https://files.ipd.uw.edu/pub/RoseTTAFold/pdb100_2021Mar03.tar.gz +tar xfz pdb100_2021Mar03.tar.gz +# for CASP14 benchmarks, we used this one: https://files.ipd.uw.edu/pub/RoseTTAFold/pdb100_2020Mar11.tar.gz +``` + +6. Obtain a [PyRosetta licence](https://els2.comotion.uw.edu/product/pyrosetta) and install the package in the newly created `folding` conda environment ([link](http://www.pyrosetta.org/downloads)). + +## Usage + +``` +# For monomer structure prediction +cd example +../run_[pyrosetta, e2e]_ver.sh input.fa . + +# For complex modeling +# please see README file under example/complex_modeling/README for details. +python network/predict_complex.py -i paired.a3m -o complex -Ls 218 310 +``` + +## Expected outputs +For the pyrosetta version, user will get five final models having estimated CA rms error at the B-factor column (model/model_[1-5].crderr.pdb). +For the end-to-end version, there will be a single PDB output having estimated residue-wise CA-lddt at the B-factor column (t000_.e2e.pdb). + +## FAQ +1. Segmentation fault while running hhblits/hhsearch +For easy install, we used a statically compiled version of hhsuite (installed through conda). Currently, we're not sure what exactly causes segmentation fault error in some cases, but we found that it might be resolved if you compile hhsuite from source and use this compiled version instead of conda version. For installation of hhsuite, please see [here](https://github.com/soedinglab/hh-suite). + +2. Submitting jobs to computing nodes +The modeling pipeline provided here (run_pyrosetta_ver.sh/run_e2e_ver.sh) is a kind of guidelines to show how RoseTTAFold works. For more efficient use of computing resources, you might want to modify the provided bash script to submit separate jobs with proper dependencies for each of steps (more cpus/memory for hhblits/hhsearch, using gpus only for running the networks, etc). + +## Links: + +* [Robetta server](https://robetta.bakerlab.org/) (RoseTTAFold option) +* [RoseTTAFold models for CASP14 targets](https://files.ipd.uw.edu/pub/RoseTTAFold/casp14_models.tar.gz) [input MSA and hhsearch files are included] + +## Credit to performer-pytorch and SE(3)-Transformer codes +The code in the network/performer_pytorch.py is strongly based on [this repo](https://github.com/lucidrains/performer-pytorch) which is pytorch implementation of [Performer architecture](https://arxiv.org/abs/2009.14794). +The codes in network/equivariant_attention is from the original SE(3)-Transformer [repo](https://github.com/FabianFuchsML/se3-transformer-public) which accompanies [the paper](https://arxiv.org/abs/2006.10503) 'SE(3)-Transformers: 3D Roto-Translation Equivariant Attention Networks' by Fabian et al. + + +## References + +M Baek, et al., Accurate prediction of protein structures and interactions using a 3-track network, bioRxiv (2021). [link](https://www.biorxiv.org/content/10.1101/2021.06.14.448402v1) + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/README.md b/DGLPyTorch/DrugDiscovery/RoseTTAFold/README.md new file mode 100644 index 00000000..676a900d --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/README.md @@ -0,0 +1,138 @@ +# RoseTTAFold for PyTorch + +This repository provides a script to run inference using the RoseTTAFold model. The content of this repository is tested and maintained by NVIDIA. + +## Table Of Contents + +- [Model overview](#model-overview) + * [Model architecture](#model-architecture) +- [Setup](#setup) + * [Requirements](#requirements) +- [Quick Start Guide](#quick-start-guide) +- [Release notes](#release-notes) + * [Changelog](#changelog) + * [Known issues](#known-issues) + + + +## Model overview + +The RoseTTAFold is a model designed to provide accurate protein structure from its amino acid sequence. This model is +based on [Accurate prediction of protein structures and interactions using a 3-track network](https://www.biorxiv.org/content/10.1101/2021.06.14.448402v1) by Minkyung Baek et al. + +This implementation is a dockerized version of the official [RoseTTAFold repository](https://github.com/RosettaCommons/RoseTTAFold/). +Here you can find the [original RoseTTAFold guide](README-ROSETTAFOLD.md). + +### Model architecture + +The RoseTTAFold model is based on a 3-track architecture fusing 1D, 2D, and 3D information about the protein structure. +All information is exchanged between tracks to learn the sequence and coordinate patterns at the same time. The final prediction +is refined using an SE(3)-Transformer. + + + +*Figure 1: The RoseTTAFold architecture. Image comes from the [original paper](https://www.biorxiv.org/content/10.1101/2021.06.14.448402v1).* + +## Setup + +The following section lists the requirements that you need to meet in order to run inference using the RoseTTAFold model. + +### Requirements + +This repository contains a Dockerfile that extends the PyTorch NGC container and encapsulates necessary dependencies. Aside from these dependencies, ensure you have the following components: +- [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker) +- PyTorch 21.09-py3 NGC container +- Supported GPUs: + - [NVIDIA Volta architecture](https://www.nvidia.com/en-us/data-center/volta-gpu-architecture/) + - [NVIDIA Turing architecture](https://www.nvidia.com/en-us/design-visualization/technologies/turing-architecture/) + - [NVIDIA Ampere architecture](https://www.nvidia.com/en-us/data-center/nvidia-ampere-gpu-architecture/) + +For more information about how to get started with NGC containers, refer to the following sections from the NVIDIA GPU Cloud Documentation and the Deep Learning Documentation: +- [Getting Started Using NVIDIA GPU Cloud](https://docs.nvidia.com/ngc/ngc-getting-started-guide/index.html) +- [Accessing And Pulling From The NGC Container Registry](https://docs.nvidia.com/deeplearning/frameworks/user-guide/index.html#accessing_registry) +- [Running PyTorch](https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/running.html#running) + +For those unable to use the PyTorch NGC container, to set up the required environment or create your own container, refer to the versioned [NVIDIA Container Support Matrix](https://docs.nvidia.com/deeplearning/frameworks/support-matrix/index.html). + +In addition, 1 TB of disk space is required to unpack the required databases. + +## Quick Start Guide + +To run inference using the RoseTTAFold model, perform the following steps using the default parameters. + +1. Clone the repository. + ``` + git clone https://github.com/NVIDIA/DeepLearningExamples + cd DeepLearningExamples/DGLPyTorch/ + ``` + +2. Download the pre-trained weights and databases needed for inference. + The following command downloads the pre-trained weights and two databases needed to create derived features to the input to the model. + The script will download the `UniRef30` (~50 GB) and `pdb100_2021Mar03` (~115 GB) databases, which might take a considerable amount + of time. Additionally, unpacking those databases requires approximately 1 TB of free disk space. + + By default, the data will be downloaded to `./weights` and `./databases` folders in the current directory. + ``` + bash scripts/download_databases.sh + ``` + If you would like to specify the download location you can pass the following parameters + ``` + bash scripts/download_databases.sh PATH-TO-WEIGHTS PATH-TO-DATABASES + ``` + +3. Build the RoseTTAFold PyTorch NGC container. This step builds the PyTorch dependencies on your machine and can take between 30 minutes and 1 hour to complete. + ``` + docker build -t rosettafold . + ``` + +4. Start an interactive session in the NGC container to run inference. + + The following command launches the container and mount the `PATH-TO-WEIGHTS` directory as a volume to the `/weights` directory in the container, the `PATH-TO-DATABASES` directory as a volume to the `/databases` directory in the container, and `./results` directory to the `/results` directory in the container. + ``` + mkdir data results + docker run --ipc=host -it --rm --runtime=nvidia -p6006:6006 -v PATH-TO-WEIGHTS:/weights -v PATH-TO-DATABASES:/databases -v ${PWD}/results:/results rosettafold:latest /bin/bash + ``` + +5. Start inference/predictions. + + To run inference you have to prepare a FASTA file and pass a path to it or pass a sequence directly. + ``` + python run_inference_pipeline.py [Sequence] + ``` + There is an example FASTA file at `example/input.fa` for you to try. Running the inference pipeline consists of four steps: + 1. Preparing the Multiple Sequence Alignments (MSAs) + 2. Preparing the secondary structures + 3. Preparing the templates + 4. Iteratively refining the prediction + + The first three steps can take between a couple of minutes and an hour, depending on the sequence. + The output will be stored at the `/results` directory as an `output.e2e.pdb` file + +6. Start Jupyter Notebook to run inference interactively. + + To launch the application, copy the Notebook to the root folder. + ``` + cp notebooks/run_inference.ipynb . + + ``` + To start Jupyter Notebook, run: + ``` + jupyter notebook run_inference.ipynb + ``` + + For more information about Jupyter Notebook, refer to the Jupyter Notebook documentation. + + +## Release notes + +### Changelog + +October 2021 +- Initial release + +### Known issues + +There are no known issues with this model. + + + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux-cu101.yml b/DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux-cu101.yml new file mode 100644 index 00000000..c88791db --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux-cu101.yml @@ -0,0 +1,107 @@ +name: RoseTTAFold +channels: + - rusty1s + - pytorch + - nvidia + - conda-forge + - defaults + - bioconda + - biocore +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=4.5=1_gnu + - biopython=1.78=py38h497a2fe_2 + - blas=1.0=mkl + - hhsuite + - blast-legacy=2.2.26=2 + - brotlipy=0.7.0=py38h497a2fe_1001 + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2021.7.5=h06a4308_1 + - certifi=2021.5.30=py38h06a4308_0 + - cffi=1.14.5=py38ha65f79e_0 + - chardet=4.0.0=py38h578d9bd_1 + - cryptography=3.4.7=py38ha5dfef3_0 + - cudatoolkit=10.2.89=h8f6ccaa_8 + - ffmpeg=4.3=hf484d3e_0 + - freetype=2.10.4=h5ab3b9f_0 + - gmp=6.2.1=h2531618_2 + - gnutls=3.6.15=he1e5248_0 + - googledrivedownloader=0.4=pyhd3deb0d_1 + - idna=2.10=pyh9f0ad1d_0 + - intel-openmp=2021.2.0=h06a4308_610 + - jinja2=3.0.1=pyhd8ed1ab_0 + - joblib=1.0.1=pyhd8ed1ab_0 + - jpeg=9b=h024ee3a_2 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.35.1=h7274673_9 + - libffi=3.3=he6710b0_2 + - libgcc-ng=9.3.0=h5101ec6_17 + - libgfortran-ng=7.5.0=h14aa051_19 + - libgfortran4=7.5.0=h14aa051_19 + - libgomp=9.3.0=h5101ec6_17 + - libiconv=1.15=h63c8f33_5 + - libidn2=2.3.1=h27cfd23_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.3.0=hd4cf53a_17 + - libtasn1=4.16.0=h27cfd23_0 + - libtiff=4.2.0=h85742a9_0 + - libunistring=0.9.10=h27cfd23_0 + - libuv=1.40.0=h7b6447c_0 + - libwebp-base=1.2.0=h27cfd23_0 + - lz4-c=1.9.3=h2531618_0 + - markupsafe=2.0.1=py38h497a2fe_0 + - mkl=2021.2.0=h06a4308_296 + - mkl-service=2.3.0=py38h27cfd23_1 + - mkl_fft=1.3.0=py38h42c9631_2 + - mkl_random=1.2.1=py38ha9443f7_2 + - ncurses=6.2=he6710b0_1 + - nettle=3.7.3=hbbd107a_1 + - networkx=2.5=py_0 + - ninja=1.10.2=hff7bd54_1 + - numpy=1.20.2=py38h2d18471_0 + - numpy-base=1.20.2=py38hfae3a4d_0 + - olefile=0.46=py_0 + - openh264=2.1.0=hd408876_0 + - openssl=1.1.1k=h27cfd23_0 + - packaging=20.9=pyhd3eb1b0_0 + - pandas=1.2.5=py38h1abd341_0 + - pillow=8.2.0=py38he98fc37_0 + - pip=21.1.3=py38h06a4308_0 + - psipred=4.01=1 + - pycparser=2.20=pyh9f0ad1d_2 + - pyopenssl=20.0.1=pyhd8ed1ab_0 + - pyparsing=2.4.7=pyh9f0ad1d_0 + - pysocks=1.7.1=py38h578d9bd_3 + - python=3.8.10=h12debd9_8 + - python-dateutil=2.8.1=py_0 + - python-louvain=0.15=pyhd3deb0d_0 + - python_abi=3.8=2_cp38 + - pytorch=1.8.1=py3.8_cuda10.2_cudnn7.6.5_0 + - pytorch-cluster=1.5.9=py38_torch_1.8.0_cu102 + - pytorch-geometric=1.7.2=py38_torch_1.8.0_cu102 + - pytorch-scatter=2.0.7=py38_torch_1.8.0_cu102 + - pytorch-sparse=0.6.10=py38_torch_1.8.0_cu102 + - pytorch-spline-conv=1.2.1=py38_torch_1.8.0_cu102 + - pytz=2021.1=pyhd8ed1ab_0 + - readline=8.1=h27cfd23_0 + - requests=2.25.1=pyhd3deb0d_0 + - scikit-learn=0.24.2=py38ha9443f7_0 + - setuptools=52.0.0=py38h06a4308_0 + - six=1.16.0=pyhd3eb1b0_0 + - sqlite=3.36.0=hc218d9a_0 + - threadpoolctl=2.1.0=pyh5ca1d4c_0 + - tk=8.6.10=hbc83047_0 + - torchvision=0.9.1=py38_cu102 + - tqdm=4.61.1=pyhd8ed1ab_0 + - typing_extensions=3.10.0.0=pyh06a4308_0 + - urllib3=1.26.6=pyhd8ed1ab_0 + - wheel=0.36.2=pyhd3eb1b0_0 + - xz=5.2.5=h7b6447c_0 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.4.9=haebb681_0 + - pip: + - decorator==4.4.2 + - dgl-cu102==0.6.1 + - lie-learn==0.0.1.post1 + - scipy==1.7.0 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux.yml b/DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux.yml new file mode 100644 index 00000000..4d4794b4 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/RoseTTAFold-linux.yml @@ -0,0 +1,108 @@ +name: RoseTTAFold +channels: + - rusty1s + - pytorch + - nvidia + - conda-forge + - defaults + - bioconda + - biocore +dependencies: + - biopython=1.78 + - biocore::blast-legacy=2.2.26 + - hhsuite + - psipred=4.01 + - _libgcc_mutex=0.1=main + - _openmp_mutex=4.5=1_gnu + - blas=1.0=mkl + - brotlipy=0.7.0=py38h497a2fe_1001 + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2021.5.25=h06a4308_1 + - certifi=2021.5.30=py38h06a4308_0 + - cffi=1.14.5=py38ha65f79e_0 + - chardet=4.0.0=py38h578d9bd_1 + - cryptography=3.4.7=py38ha5dfef3_0 + - cudatoolkit=11.1.74=h6bb024c_0 + - ffmpeg=4.3=hf484d3e_0 + - freetype=2.10.4=h5ab3b9f_0 + - gmp=6.2.1=h2531618_2 + - gnutls=3.6.15=he1e5248_0 + - googledrivedownloader=0.4=pyhd3deb0d_1 + - idna=2.10=pyh9f0ad1d_0 + - intel-openmp=2021.2.0=h06a4308_610 + - jinja2=3.0.1=pyhd8ed1ab_0 + - joblib=1.0.1=pyhd8ed1ab_0 + - jpeg=9b=h024ee3a_2 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.35.1=h7274673_9 + - libffi=3.3=he6710b0_2 + - libgcc-ng=9.3.0=h5101ec6_17 + - libgfortran-ng=7.5.0=h14aa051_19 + - libgfortran4=7.5.0=h14aa051_19 + - libgomp=9.3.0=h5101ec6_17 + - libiconv=1.15=h63c8f33_5 + - libidn2=2.3.1=h27cfd23_0 + - libpng=1.6.37=hbc83047_0 + - libstdcxx-ng=9.3.0=hd4cf53a_17 + - libtasn1=4.16.0=h27cfd23_0 + - libtiff=4.2.0=h85742a9_0 + - libunistring=0.9.10=h27cfd23_0 + - libuv=1.40.0=h7b6447c_0 + - libwebp-base=1.2.0=h27cfd23_0 + - lz4-c=1.9.3=h2531618_0 + - markupsafe=2.0.1=py38h497a2fe_0 + - mkl=2021.2.0=h06a4308_296 + - mkl-service=2.3.0=py38h27cfd23_1 + - mkl_fft=1.3.0=py38h42c9631_2 + - mkl_random=1.2.1=py38ha9443f7_2 + - ncurses=6.2=he6710b0_1 + - nettle=3.7.3=hbbd107a_1 + - networkx=2.5=py_0 + - ninja=1.10.2=hff7bd54_1 + - numpy=1.20.2=py38h2d18471_0 + - numpy-base=1.20.2=py38hfae3a4d_0 + - olefile=0.46=py_0 + - openh264=2.1.0=hd408876_0 + - openssl=1.1.1k=h27cfd23_0 + - packaging=20.9=pyhd3eb1b0_0 + - pandas=1.2.5=py38h1abd341_0 + - pillow=8.2.0=py38he98fc37_0 + - pip=21.1.3=py38h06a4308_0 + - pycparser=2.20=pyh9f0ad1d_2 + - pyopenssl=20.0.1=pyhd8ed1ab_0 + - pyparsing=2.4.7=pyh9f0ad1d_0 + - pysocks=1.7.1=py38h578d9bd_3 + - python=3.8.10=h12debd9_8 + - python-dateutil=2.8.1=py_0 + - python-louvain=0.15=pyhd3deb0d_0 + - python_abi=3.8=2_cp38 + - pytorch=1.9.0=py3.8_cuda11.1_cudnn8.0.5_0 + - pytorch-cluster=1.5.9=py38_torch_1.9.0_cu111 + - pytorch-geometric=1.7.2=py38_torch_1.9.0_cu111 + - pytorch-scatter=2.0.7=py38_torch_1.9.0_cu111 + - pytorch-sparse=0.6.10=py38_torch_1.9.0_cu111 + - pytorch-spline-conv=1.2.1=py38_torch_1.9.0_cu111 + - pytz=2021.1=pyhd8ed1ab_0 + - readline=8.1=h27cfd23_0 + - requests=2.25.1=pyhd3deb0d_0 + - scikit-learn=0.24.2=py38ha9443f7_0 + - setuptools=52.0.0=py38h06a4308_0 + - six=1.16.0=pyhd3eb1b0_0 + - sqlite=3.36.0=hc218d9a_0 + - threadpoolctl=2.1.0=pyh5ca1d4c_0 + - tk=8.6.10=hbc83047_0 + - torchaudio=0.9.0=py38 + - torchvision=0.10.0=py38_cu111 + - tqdm=4.61.1=pyhd8ed1ab_0 + - typing_extensions=3.10.0.0=pyh06a4308_0 + - urllib3=1.26.6=pyhd8ed1ab_0 + - wheel=0.36.2=pyhd3eb1b0_0 + - xz=5.2.5=h7b6447c_0 + - zlib=1.2.11=h7b6447c_3 + - zstd=1.4.9=haebb681_0 + - pip: + - decorator==4.4.2 + - dgl-cu110==0.6.1 + - scipy==1.7.0 + - lie-learn==0.0.1.post1 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/example/input.fa b/DGLPyTorch/DrugDiscovery/RoseTTAFold/example/input.fa new file mode 100644 index 00000000..d9c41e34 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/example/input.fa @@ -0,0 +1,2 @@ +>T1078 Tsp1, Trichoderma virens, 138 residues| +MAAPTPADKSMMAAVPEWTITNLKRVCNAGNTSCTWTFGVDTHLATATSCTYVVKANANASQASGGPVTCGPYTITSSWSGQFGPNNGFTTFAVTDFSKKLIVWPAYTDVQVQAGKVVSPNQSYAPANLPLEHHHHHH diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding-linux.yml b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding-linux.yml new file mode 100644 index 00000000..4ce1df27 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding-linux.yml @@ -0,0 +1,9 @@ +name: folding +channels: + - defaults + - conda-forge +dependencies: + - tensorflow-gpu=1.14 + - pandas + - scikit-learn=0.24 + - parallel diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/README b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/README new file mode 100644 index 00000000..ce91ad30 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/README @@ -0,0 +1,43 @@ +############################################################################### +RosettaTR.py: Relax on dualspace (2 rounds of relax w/ different set of + restraints) +############################################################################### + + +usage: RosettaTR.py [-h] [-r NRESTARTS] [-pd PCUT] [-m {0,1,2}] [-bb BB] + [-sg SG] [-n STEPS] [--save_chk] [--orient] [--no-orient] + [--fastrelax] [--no-fastrelax] [--roll] [--no-roll] + NPZ FASTA OUT + +positional arguments: + NPZ input distograms and anglegrams (NN predictions) + FASTA input sequence + OUT output model (in PDB format) + +optional arguments: + -h, --help show this help message and exit + -r NRESTARTS number of noisy restrarts (default: 3) + -pd PCUT min probability of distance restraints (default: 0.05) + -m {0,1,2} 0: sh+m+l, 1: (sh+m)+l, 2: (sh+m+l) (default: 2) + -bb BB predicted backbone torsions (default: ) + -sg SG window size and order for a Savitzky-Golay filter (comma- + separated) (default: ) + -n STEPS number of minimization steps (default: 1000) + --save_chk save checkpoint files to restart (default: False) + --orient use orientations (default: True) + --no-orient + --fastrelax perform FastRelax (default: True) + --no-fastrelax + --roll circularly shift 6d coordinate arrays by 1 (default: False) + --no-roll + + +# USAGE +conda activate folding + +# try: -m 0,1,2 +# -pd 0.05, 0.15, 0.25, 0.35, 0.45 +# repeat ~3-5 times for every combination of -m and -pd +# !!! use '--roll' option if no-contact bin is the last one !!! +python ./RosettaTR.py -m 0 -pd 0.15 fake.npz fake.fa model.pdb + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/RosettaTR.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/RosettaTR.py new file mode 100644 index 00000000..7e9d4303 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/RosettaTR.py @@ -0,0 +1,340 @@ +import sys,os,json +import tempfile +import numpy as np + +from arguments import * +from utils import * +from pyrosetta import * +from pyrosetta.rosetta.protocols.minimization_packing import MinMover + +vdw_weight = {0: 3.0, 1: 5.0, 2: 10.0} +rsr_dist_weight = {0: 3.0, 1: 2.0, 3: 1.0} +rsr_orient_weight = {0: 1.0, 1: 1.0, 3: 0.5} + +def main(): + + ######################################################## + # process inputs + ######################################################## + + # read params + scriptdir = os.path.dirname(os.path.realpath(__file__)) + with open(scriptdir + '/data/params.json') as jsonfile: + params = json.load(jsonfile) + + # get command line arguments + args = get_args(params) + print(args) + if os.path.exists(args.OUT): + return + + # init PyRosetta + init_cmd = list() + init_cmd.append("-multithreading:interaction_graph_threads 1 -multithreading:total_threads 1") + init_cmd.append("-hb_cen_soft") + init_cmd.append("-detect_disulf -detect_disulf_tolerance 2.0") # detect disulfide bonds based on Cb-Cb distance (CEN mode) or SG-SG distance (FA mode) + init_cmd.append("-relax:dualspace true -relax::minimize_bond_angles -default_max_cycles 200") + init_cmd.append("-mute all") + init(" ".join(init_cmd)) + + # read and process restraints & sequence + seq = read_fasta(args.FASTA) + L = len(seq) + params['seq'] = seq + rst = gen_rst(params) + + ######################################################## + # Scoring functions and movers + ######################################################## + sf = ScoreFunction() + sf.add_weights_from_file(scriptdir + '/data/scorefxn.wts') + + sf1 = ScoreFunction() + sf1.add_weights_from_file(scriptdir + '/data/scorefxn1.wts') + + sf_vdw = ScoreFunction() + sf_vdw.add_weights_from_file(scriptdir + '/data/scorefxn_vdw.wts') + + sf_cart = ScoreFunction() + sf_cart.add_weights_from_file(scriptdir + '/data/scorefxn_cart.wts') + + mmap = MoveMap() + mmap.set_bb(True) + mmap.set_chi(False) + mmap.set_jump(True) + + min_mover1 = MinMover(mmap, sf1, 'lbfgs_armijo_nonmonotone', 0.001, True) + min_mover1.max_iter(1000) + + min_mover_vdw = MinMover(mmap, sf_vdw, 'lbfgs_armijo_nonmonotone', 0.001, True) + min_mover_vdw.max_iter(500) + + min_mover_cart = MinMover(mmap, sf_cart, 'lbfgs_armijo_nonmonotone', 0.000001, True) + min_mover_cart.max_iter(300) + min_mover_cart.cartesian(True) + + + if not os.path.exists("%s_before_relax.pdb"%('.'.join(args.OUT.split('.')[:-1]))): + ######################################################## + # initialize pose + ######################################################## + pose0 = pose_from_sequence(seq, 'centroid') + + # mutate GLY to ALA + for i,a in enumerate(seq): + if a == 'G': + mutator = rosetta.protocols.simple_moves.MutateResidue(i+1,'ALA') + mutator.apply(pose0) + print('mutation: G%dA'%(i+1)) + + if (args.bb == ''): + print('setting random (phi,psi,omega)...') + set_random_dihedral(pose0) + else: + print('setting predicted (phi,psi,omega)...') + bb = np.load(args.bb) + set_predicted_dihedral(pose0,bb['phi'],bb['psi'],bb['omega']) + + remove_clash(sf_vdw, min_mover_vdw, pose0) + + Emin = 99999.9 + + ######################################################## + # minimization + ######################################################## + + for run in range(params['NRUNS']): + # define repeat_mover here!! (update vdw weights: weak (1.0) -> strong (10.0) + sf.set_weight(rosetta.core.scoring.vdw, vdw_weight.setdefault(run, 10.0)) + sf.set_weight(rosetta.core.scoring.atom_pair_constraint, rsr_dist_weight.setdefault(run, 1.0)) + sf.set_weight(rosetta.core.scoring.dihedral_constraint, rsr_orient_weight.setdefault(run, 0.5)) + sf.set_weight(rosetta.core.scoring.angle_constraint, rsr_orient_weight.setdefault(run, 0.5)) + + min_mover = MinMover(mmap, sf, 'lbfgs_armijo_nonmonotone', 0.001, True) + min_mover.max_iter(1000) + + repeat_mover = RepeatMover(min_mover, 3) + + # + pose = Pose() + pose.assign(pose0) + pose.remove_constraints() + + if run > 0: + + # diversify backbone + dphi = np.random.uniform(-10,10,L) + dpsi = np.random.uniform(-10,10,L) + for i in range(1,L+1): + pose.set_phi(i,pose.phi(i)+dphi[i-1]) + pose.set_psi(i,pose.psi(i)+dpsi[i-1]) + + # remove clashes + remove_clash(sf_vdw, min_mover_vdw, pose) + + # Save checkpoint + if args.save_chk: + pose.dump_pdb("%s_run%d_init.pdb"%('.'.join(args.OUT.split('.')[:-1]), run)) + + if args.mode == 0: + + # short + print('short') + add_rst(pose, rst, 3, 12, params) + repeat_mover.apply(pose) + remove_clash(sf_vdw, min_mover1, pose) + min_mover_cart.apply(pose) + if args.save_chk: + pose.dump_pdb("%s_run%d_mode%d_step%d.pdb"%('.'.join(args.OUT.split('.')[:-1]), run, args.mode, 0)) + + # medium + print('medium') + add_rst(pose, rst, 12, 24, params) + repeat_mover.apply(pose) + remove_clash(sf_vdw, min_mover1, pose) + min_mover_cart.apply(pose) + if args.save_chk: + pose.dump_pdb("%s_run%d_mode%d_step%d.pdb"%('.'.join(args.OUT.split('.')[:-1]), run, args.mode, 1)) + + # long + print('long') + add_rst(pose, rst, 24, len(seq), params) + repeat_mover.apply(pose) + remove_clash(sf_vdw, min_mover1, pose) + min_mover_cart.apply(pose) + if args.save_chk: + pose.dump_pdb("%s_run%d_mode%d_step%d.pdb"%('.'.join(args.OUT.split('.')[:-1]), run, args.mode, 2)) + + elif args.mode == 1: + + # short + medium + print('short + medium') + add_rst(pose, rst, 3, 24, params) + repeat_mover.apply(pose) + remove_clash(sf_vdw, min_mover1, pose) + min_mover_cart.apply(pose) + if args.save_chk: + pose.dump_pdb("%s_run%d_mode%d_step%d.pdb"%('.'.join(args.OUT.split('.')[:-1]), run, args.mode, 0)) + + # long + print('long') + add_rst(pose, rst, 24, len(seq), params) + repeat_mover.apply(pose) + remove_clash(sf_vdw, min_mover1, pose) + min_mover_cart.apply(pose) + if args.save_chk: + pose.dump_pdb("%s_run%d_mode%d_step%d.pdb"%('.'.join(args.OUT.split('.')[:-1]), run, args.mode, 1)) + + elif args.mode == 2: + + # short + medium + long + print('short + medium + long') + add_rst(pose, rst, 3, len(seq), params) + repeat_mover.apply(pose) + remove_clash(sf_vdw, min_mover1, pose) + min_mover_cart.apply(pose) + if args.save_chk: + pose.dump_pdb("%s_run%d_mode%d_step%d.pdb"%('.'.join(args.OUT.split('.')[:-1]), run, args.mode, 0)) + + # check whether energy has decreased + pose.conformation().detect_disulfides() # detect disulfide bonds + E = sf_cart(pose) + if E < Emin: + print("Energy(iter=%d): %.1f --> %.1f (accept)"%(run, Emin, E)) + Emin = E + pose0.assign(pose) + else: + print("Energy(iter=%d): %.1f --> %.1f (reject)"%(run, Emin, E)) + + # mutate ALA back to GLY + for i,a in enumerate(seq): + if a == 'G': + mutator = rosetta.protocols.simple_moves.MutateResidue(i+1,'GLY') + mutator.apply(pose0) + print('mutation: A%dG'%(i+1)) + + ######################################################## + # fix backbone geometry + ######################################################## + pose0.remove_constraints() + + # apply more strict criteria to detect disulfide bond + # Set options for disulfide tolerance -> 1.0A + print (rosetta.basic.options.get_real_option('in:detect_disulf_tolerance')) + rosetta.basic.options.set_real_option('in:detect_disulf_tolerance', 1.0) + print (rosetta.basic.options.get_real_option('in:detect_disulf_tolerance')) + pose0.conformation().detect_disulfides() + + # Converto to all atom representation + switch = SwitchResidueTypeSetMover("fa_standard") + switch.apply(pose0) + + # idealize problematic local regions if exists + idealize = rosetta.protocols.idealize.IdealizeMover() + poslist = rosetta.utility.vector1_unsigned_long() + + scorefxn=create_score_function('empty') + scorefxn.set_weight(rosetta.core.scoring.cart_bonded, 1.0) + scorefxn.score(pose0) + + emap = pose0.energies() + print("idealize...") + for res in range(1,L+1): + cart = emap.residue_total_energy(res) + if cart > 50: + poslist.append(res) + print( "idealize %d %8.3f"%(res,cart) ) + + if len(poslist) > 0: + idealize.set_pos_list(poslist) + try: + idealize.apply(pose0) + + except: + print('!!! idealization failed !!!') + + # Save checkpoint + if args.save_chk: + pose0.dump_pdb("%s_before_relax.pdb"%'.'.join(args.OUT.split('.')[:-1])) + + else: # checkpoint exists + pose0 = pose_from_file("%s_before_relax.pdb"%('.'.join(args.OUT.split('.')[:-1]))) + # + print ("to centroid") + switch_cen = SwitchResidueTypeSetMover("centroid") + switch_cen.apply(pose0) + # + print ("detect disulfide bonds") + # Set options for disulfide tolerance -> 1.0A + print (rosetta.basic.options.get_real_option('in:detect_disulf_tolerance')) + rosetta.basic.options.set_real_option('in:detect_disulf_tolerance', 1.0) + print (rosetta.basic.options.get_real_option('in:detect_disulf_tolerance')) + pose0.conformation().detect_disulfides() + # + print ("to all atom") + switch = SwitchResidueTypeSetMover("fa_standard") + switch.apply(pose0) + + + ######################################################## + # full-atom refinement + ######################################################## + + if args.fastrelax == True: + mmap = MoveMap() + mmap.set_bb(True) + mmap.set_chi(True) + mmap.set_jump(True) + + # First round: Repeat 2 torsion space relax w/ strong disto/anglogram constraints + sf_fa_round1 = create_score_function('ref2015_cart') + sf_fa_round1.set_weight(rosetta.core.scoring.atom_pair_constraint, 3.0) + sf_fa_round1.set_weight(rosetta.core.scoring.dihedral_constraint, 1.0) + sf_fa_round1.set_weight(rosetta.core.scoring.angle_constraint, 1.0) + sf_fa_round1.set_weight(rosetta.core.scoring.pro_close, 0.0) + + relax_round1 = rosetta.protocols.relax.FastRelax(sf_fa_round1, "%s/data/relax_round1.txt"%scriptdir) + relax_round1.set_movemap(mmap) + + print('relax: First round... (focused on torsion space relaxation)') + params['PCUT'] = 0.15 + pose0.remove_constraints() + add_rst(pose0, rst, 3, len(seq), params, nogly=True, use_orient=True) + relax_round1.apply(pose0) + + # Set options for disulfide tolerance -> 0.5A + print (rosetta.basic.options.get_real_option('in:detect_disulf_tolerance')) + rosetta.basic.options.set_real_option('in:detect_disulf_tolerance', 0.5) + print (rosetta.basic.options.get_real_option('in:detect_disulf_tolerance')) + + sf_fa = create_score_function('ref2015_cart') + sf_fa.set_weight(rosetta.core.scoring.atom_pair_constraint, 0.1) + sf_fa.set_weight(rosetta.core.scoring.dihedral_constraint, 0.0) + sf_fa.set_weight(rosetta.core.scoring.angle_constraint, 0.0) + + relax_round2 = rosetta.protocols.relax.FastRelax(sf_fa, "%s/data/relax_round2.txt"%scriptdir) + relax_round2.set_movemap(mmap) + relax_round2.cartesian(True) + relax_round2.dualspace(True) + + print('relax: Second round... (cartesian space)') + params['PCUT'] = 0.30 # To reduce the number of pair restraints.. + pose0.remove_constraints() + pose0.conformation().detect_disulfides() # detect disulfide bond again w/ stricter cutoffs + # To reduce the number of constraints, only pair distances are considered w/ higher prob cutoffs + add_rst(pose0, rst, 3, len(seq), params, nogly=True, use_orient=False, p12_cut=params['PCUT']) + # Instead, apply CA coordinate constraints to prevent drifting away too much (focus on local refinement?) + add_crd_rst(pose0, L, std=1.0, tol=2.0) + relax_round2.apply(pose0) + + # Re-evaluate score w/o any constraints + scorefxn_min=create_score_function('ref2015_cart') + scorefxn_min.score(pose0) + + ######################################################## + # save final model + ######################################################## + pose0.dump_pdb(args.OUT) + +if __name__ == '__main__': + main() diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/arguments.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/arguments.py new file mode 100644 index 00000000..44c1eac2 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/arguments.py @@ -0,0 +1,38 @@ +import argparse + +def get_args(params): + + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("NPZ", type=str, help="input distograms and anglegrams (NN predictions)") + parser.add_argument("FASTA", type=str, help="input sequence") + parser.add_argument("OUT", type=str, help="output model (in PDB format)") + + parser.add_argument('-r', type=int, dest='nrestarts', default=params['NRUNS'], help='number of noisy restrarts') + parser.add_argument('-pd', type=float, dest='pcut', default=params['PCUT'], help='min probability of distance restraints') + parser.add_argument('-m', type=int, dest='mode', default=2, choices=[0,1,2], help='0: sh+m+l, 1: (sh+m)+l, 2: (sh+m+l)') + parser.add_argument('-bb', type=str, dest='bb', default='', help='predicted backbone torsions') + parser.add_argument('-sg', type=str, dest='sg', default='', help='window size and order for a Savitzky-Golay filter (comma-separated)') + parser.add_argument('-n', type=int, dest='steps', default=1000, help='number of minimization steps') + parser.add_argument('--save_chk', dest='save_chk', default=False, action='store_true', help='save checkpoint files to restart') + parser.add_argument('--orient', dest='use_orient', action='store_true', help='use orientations') + parser.add_argument('--no-orient', dest='use_orient', action='store_false') + parser.add_argument('--fastrelax', dest='fastrelax', action='store_true', help='perform FastRelax') + parser.add_argument('--no-fastrelax', dest='fastrelax', action='store_false') + parser.add_argument('--roll', dest='roll', action='store_true', help='circularly shift 6d coordinate arrays by 1') + parser.add_argument('--no-roll', dest='roll', action='store_false') + parser.set_defaults(use_orient=True) + parser.set_defaults(fastrelax=True) + parser.set_defaults(roll=False) + + args = parser.parse_args() + + params['PCUT'] = args.pcut + params['USE_ORIENT'] = args.use_orient + params['NRUNS'] = args.nrestarts + params['ROLL'] = args.roll + + params['NPZ'] = args.NPZ + params['BB'] = args.bb + params['SG'] = args.sg + + return args diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/params.json b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/params.json new file mode 100644 index 00000000..ec1eb3c3 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/params.json @@ -0,0 +1,17 @@ +{ + "PCUT" : 0.05, + "PCUT1" : 0.5, + "EBASE" : -0.5, + "EREP" : [10.0,3.0,0.5], + "DREP" : [0.0,2.0,3.5], + "PREP" : 0.1, + "SIGD" : 10.0, + "SIGM" : 1.0, + "MEFF" : 0.0001, + "DCUT" : 19.5, + "ALPHA" : 1.57, + "DSTEP" : 0.5, + "ASTEP" : 15.0, + "BBWGHT" : 10.0, + "NRUNS" : 3 +} diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round1.txt b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round1.txt new file mode 100644 index 00000000..8250e69d --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round1.txt @@ -0,0 +1,17 @@ +switch:torsion +repeat 2 +ramp_repack_min 0.02 0.01 1.0 100 +ramp_repack_min 0.250 0.01 0.5 100 +ramp_repack_min 0.550 0.01 0.1 100 +ramp_repack_min 1 0.00001 0.1 100 +accept_to_best +endrepeat + +switch:cartesian +repeat 1 +ramp_repack_min 0.02 0.01 1.0 50 +ramp_repack_min 0.250 0.01 0.5 50 +ramp_repack_min 0.550 0.01 0.1 100 +ramp_repack_min 1 0.00001 0.1 200 +accept_to_best +endrepeat diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round2.txt b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round2.txt new file mode 100644 index 00000000..714c1011 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/relax_round2.txt @@ -0,0 +1,8 @@ +switch:cartesian +repeat 2 +ramp_repack_min 0.02 0.01 1.0 50 +ramp_repack_min 0.250 0.01 0.5 50 +ramp_repack_min 0.550 0.01 0.1 100 +ramp_repack_min 1 0.00001 0.1 200 +accept_to_best +endrepeat diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn.wts b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn.wts new file mode 100644 index 00000000..e2a9296b --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn.wts @@ -0,0 +1,7 @@ +cen_hb 5.0 +rama 1.0 +omega 0.5 +vdw 1.0 +atom_pair_constraint 5.0 +dihedral_constraint 4.0 +angle_constraint 4.0 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn1.wts b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn1.wts new file mode 100644 index 00000000..ef9e0e23 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn1.wts @@ -0,0 +1,7 @@ +cen_hb 5.0 +rama 1.0 +omega 0.5 +vdw 5.0 +atom_pair_constraint 1.0 +dihedral_constraint 0.5 +angle_constraint 0.5 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_cart.wts b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_cart.wts new file mode 100644 index 00000000..b4a87309 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_cart.wts @@ -0,0 +1,9 @@ +hbond_sr_bb 3.0 +hbond_lr_bb 3.0 +rama 1.0 +omega 0.5 +vdw 5.0 +cart_bonded 5.0 +atom_pair_constraint 1.0 +dihedral_constraint 0.5 +angle_constraint 0.5 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_vdw.wts b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_vdw.wts new file mode 100644 index 00000000..a3a82920 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/data/scorefxn_vdw.wts @@ -0,0 +1,2 @@ +rama 1.0 +vdw 1.0 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/utils.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/utils.py new file mode 100644 index 00000000..e92ee29d --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/folding/utils.py @@ -0,0 +1,373 @@ +import numpy as np +import random +import scipy +from scipy.signal import * +from pyrosetta import * + +eps = 1e-9 +P_ADD_OMEGA = 0.5 +P_ADD_THETA = 0.5 +P_ADD_PHI = 0.6 + +def gen_rst(params): + + npz = np.load(params['NPZ']) + + dist,omega,theta,phi = npz['dist'],npz['omega'],npz['theta'],npz['phi'] + + if params['ROLL']==True: + print("Apply circular shift...") + dist = np.roll(dist,1,axis=-1) + omega = np.roll(omega,1,axis=-1) + theta = np.roll(theta,1,axis=-1) + phi = np.roll(phi,1,axis=-1) + + dist = dist.astype(np.float32) + eps + omega = omega.astype(np.float32) + eps + theta = theta.astype(np.float32) + eps + phi = phi.astype(np.float32) + eps + + # dictionary to store Rosetta restraints + rst = {'dist' : [], 'omega' : [], 'theta' : [], 'phi' : []} + + ######################################################## + # assign parameters + ######################################################## + PCUT = 0.05 #params['PCUT'] + EBASE = params['EBASE'] + EREP = params['EREP'] + DREP = params['DREP'] + PREP = params['PREP'] + SIGD = params['SIGD'] + SIGM = params['SIGM'] + MEFF = params['MEFF'] + DCUT = params['DCUT'] + ALPHA = params['ALPHA'] + BBWGHT = params['BBWGHT'] + + DSTEP = params['DSTEP'] + ASTEP = np.deg2rad(params['ASTEP']) + + seq = params['seq'] + + sg_flag = False + if params['SG'] != '': + sg_flag = True + sg_w,sg_n = [int(v) for v in params['SG'].split(",")] + print("Savitzky-Golay: %d,%d"%(sg_w,sg_n)) + + ######################################################## + # dist: 0..20A + ######################################################## + nres = dist.shape[0] + bins = np.array([4.25+DSTEP*i for i in range(32)]) + prob = np.sum(dist[:,:,5:], axis=-1) # prob of dist within 20A + prob_12 = np.sum(dist[:,:,5:21], axis=-1) # prob of dist within 12A + bkgr = np.array((bins/DCUT)**ALPHA) + attr = -np.log((dist[:,:,5:]+MEFF)/(dist[:,:,-1][:,:,None]*bkgr[None,None,:]))+EBASE + repul = np.maximum(attr[:,:,0],np.zeros((nres,nres)))[:,:,None]+np.array(EREP)[None,None,:] + dist = np.concatenate([repul,attr], axis=-1) + bins = np.concatenate([DREP,bins]) + x = pyrosetta.rosetta.utility.vector1_double() + _ = [x.append(v) for v in bins] + # + prob = np.triu(prob, k=1) # fill zeros to diagonal and lower (for speed-up) + i,j = np.where(prob>PCUT) + prob = prob[i,j] + prob_12 = prob_12[i,j] + #nbins = 35 + step = 0.5 + for a,b,p,p_12 in zip(i,j,prob,prob_12): + y = pyrosetta.rosetta.utility.vector1_double() + if sg_flag == True: + _ = [y.append(v) for v in savgol_filter(dist[a,b],sg_w,sg_n)] + else: + _ = [y.append(v) for v in dist[a,b]] + spline = rosetta.core.scoring.func.SplineFunc("", 1.0, 0.0, step, x,y) + ida = rosetta.core.id.AtomID(5,a+1) + idb = rosetta.core.id.AtomID(5,b+1) + rst['dist'].append([a,b,p,p_12,rosetta.core.scoring.constraints.AtomPairConstraint(ida, idb, spline)]) + print("dist restraints: %d"%(len(rst['dist']))) + + + ######################################################## + # omega: -pi..pi + ######################################################## + nbins = omega.shape[2]-1 + ASTEP = 2.0*np.pi/nbins + nbins += 4 + bins = np.linspace(-np.pi-1.5*ASTEP, np.pi+1.5*ASTEP, nbins) + x = pyrosetta.rosetta.utility.vector1_double() + _ = [x.append(v) for v in bins] + prob = np.sum(omega[:,:,1:], axis=-1) + prob = np.triu(prob, k=1) # fill zeros to diagonal and lower (for speed-up) + i,j = np.where(prob>PCUT+P_ADD_OMEGA) + prob = prob[i,j] + omega = -np.log((omega+MEFF)/(omega[:,:,-1]+MEFF)[:,:,None]) + #if sg_flag == True: + # omega = savgol_filter(omega,sg_w,sg_n,axis=-1,mode='wrap') + omega = np.concatenate([omega[:,:,-2:],omega[:,:,1:],omega[:,:,1:3]],axis=-1) + for a,b,p in zip(i,j,prob): + y = pyrosetta.rosetta.utility.vector1_double() + _ = [y.append(v) for v in omega[a,b]] + spline = rosetta.core.scoring.func.SplineFunc("", 1.0, 0.0, ASTEP, x,y) + id1 = rosetta.core.id.AtomID(2,a+1) # CA-i + id2 = rosetta.core.id.AtomID(5,a+1) # CB-i + id3 = rosetta.core.id.AtomID(5,b+1) # CB-j + id4 = rosetta.core.id.AtomID(2,b+1) # CA-j + rst['omega'].append([a,b,p,rosetta.core.scoring.constraints.DihedralConstraint(id1,id2,id3,id4, spline)]) + print("omega restraints: %d"%(len(rst['omega']))) + + + ######################################################## + # theta: -pi..pi + ######################################################## + prob = np.sum(theta[:,:,1:], axis=-1) + np.fill_diagonal(prob, 0.0) + i,j = np.where(prob>PCUT+P_ADD_THETA) + prob = prob[i,j] + theta = -np.log((theta+MEFF)/(theta[:,:,-1]+MEFF)[:,:,None]) + #if sg_flag == True: + # theta = savgol_filter(theta,sg_w,sg_n,axis=-1,mode='wrap') + theta = np.concatenate([theta[:,:,-2:],theta[:,:,1:],theta[:,:,1:3]],axis=-1) + for a,b,p in zip(i,j,prob): + y = pyrosetta.rosetta.utility.vector1_double() + _ = [y.append(v) for v in theta[a,b]] + spline = rosetta.core.scoring.func.SplineFunc("", 1.0, 0.0, ASTEP, x,y) + id1 = rosetta.core.id.AtomID(1,a+1) # N-i + id2 = rosetta.core.id.AtomID(2,a+1) # CA-i + id3 = rosetta.core.id.AtomID(5,a+1) # CB-i + id4 = rosetta.core.id.AtomID(5,b+1) # CB-j + rst['theta'].append([a,b,p,rosetta.core.scoring.constraints.DihedralConstraint(id1,id2,id3,id4, spline)]) + + print("theta restraints: %d"%(len(rst['theta']))) + + + ######################################################## + # phi: 0..pi + ######################################################## + nbins = phi.shape[2]-1+4 + bins = np.linspace(-1.5*ASTEP, np.pi+1.5*ASTEP, nbins) + x = pyrosetta.rosetta.utility.vector1_double() + _ = [x.append(v) for v in bins] + prob = np.sum(phi[:,:,1:], axis=-1) + np.fill_diagonal(prob, 0.0) + i,j = np.where(prob>PCUT+P_ADD_PHI) + prob = prob[i,j] + phi = -np.log((phi+MEFF)/(phi[:,:,-1]+MEFF)[:,:,None]) + #if sg_flag == True: + # phi = savgol_filter(phi,sg_w,sg_n,axis=-1,mode='mirror') + phi = np.concatenate([np.flip(phi[:,:,1:3],axis=-1),phi[:,:,1:],np.flip(phi[:,:,-2:],axis=-1)], axis=-1) + for a,b,p in zip(i,j,prob): + y = pyrosetta.rosetta.utility.vector1_double() + _ = [y.append(v) for v in phi[a,b]] + spline = rosetta.core.scoring.func.SplineFunc("", 1.0, 0.0, ASTEP, x,y) + id1 = rosetta.core.id.AtomID(2,a+1) # CA-i + id2 = rosetta.core.id.AtomID(5,a+1) # CB-i + id3 = rosetta.core.id.AtomID(5,b+1) # CB-j + rst['phi'].append([a,b,p,rosetta.core.scoring.constraints.AngleConstraint(id1,id2,id3, spline)]) + print("phi restraints: %d"%(len(rst['phi']))) + + ######################################################## + # backbone torsions + ######################################################## + if (params['BB'] != ''): + bbnpz = np.load(params['BB']) + bbphi,bbpsi = bbnpz['phi'],bbnpz['psi'] + rst['bbphi'] = [] + rst['bbpsi'] = [] + nbins = bbphi.shape[1]+4 + step = 2.*np.pi/bbphi.shape[1] + bins = np.linspace(-1.5*step-np.pi, np.pi+1.5*step, nbins) + x = pyrosetta.rosetta.utility.vector1_double() + _ = [x.append(v) for v in bins] + + bbphi = -np.log(bbphi) + bbphi = np.concatenate([bbphi[:,-2:],bbphi,bbphi[:,:2]],axis=-1).copy() + + bbpsi = -np.log(bbpsi) + bbpsi = np.concatenate([bbpsi[:,-2:],bbpsi,bbpsi[:,:2]],axis=-1).copy() + + for i in range(1,nres): + N1 = rosetta.core.id.AtomID(1,i) + Ca1 = rosetta.core.id.AtomID(2,i) + C1 = rosetta.core.id.AtomID(3,i) + N2 = rosetta.core.id.AtomID(1,i+1) + Ca2 = rosetta.core.id.AtomID(2,i+1) + C2 = rosetta.core.id.AtomID(3,i+1) + + # psi(i) + ypsi = pyrosetta.rosetta.utility.vector1_double() + _ = [ypsi.append(v) for v in bbpsi[i-1]] + spsi = rosetta.core.scoring.func.SplineFunc("", BBWGHT, 0.0, step, x,ypsi) + rst['bbpsi'].append(rosetta.core.scoring.constraints.DihedralConstraint(N1,Ca1,C1,N2, spsi)) + + # phi(i+1) + yphi = pyrosetta.rosetta.utility.vector1_double() + _ = [yphi.append(v) for v in bbphi[i]] + sphi = rosetta.core.scoring.func.SplineFunc("", BBWGHT, 0.0, step, x,yphi) + rst['bbphi'].append(rosetta.core.scoring.constraints.DihedralConstraint(C1,N2,Ca2,C2, sphi)) + + print("bbbtor restraints: %d"%(len(rst['bbphi'])+len(rst['bbpsi']))) + + return rst + +def set_predicted_dihedral(pose, phi, psi, omega): + + nbins = phi.shape[1] + bins = np.linspace(-180.,180.,nbins+1)[:-1] + 180./nbins + + nres = pose.total_residue() + for i in range(nres): + pose.set_phi(i+1,np.random.choice(bins,p=phi[i])) + pose.set_psi(i+1,np.random.choice(bins,p=psi[i])) + + if np.random.uniform() < omega[i,0]: + pose.set_omega(i+1,0) + else: + pose.set_omega(i+1,180) + +def set_random_dihedral(pose): + nres = pose.total_residue() + for i in range(1, nres+1): + phi,psi=random_dihedral() + pose.set_phi(i,phi) + pose.set_psi(i,psi) + pose.set_omega(i,180) + + return(pose) + + +#pick phi/psi randomly from: +#-140 153 180 0.135 B +# -72 145 180 0.155 B +#-122 117 180 0.073 B +# -82 -14 180 0.122 A +# -61 -41 180 0.497 A +# 57 39 180 0.018 L +def random_dihedral(): + phi=0 + psi=0 + r=random.random() + if(r<=0.135): + phi=-140 + psi=153 + elif(r>0.135 and r<=0.29): + phi=-72 + psi=145 + elif(r>0.29 and r<=0.363): + phi=-122 + psi=117 + elif(r>0.363 and r<=0.485): + phi=-82 + psi=-14 + elif(r>0.485 and r<=0.982): + phi=-61 + psi=-41 + else: + phi=57 + psi=39 + return(phi, psi) + + +def read_fasta(file): + fasta="" + first = True + with open(file, "r") as f: + for line in f: + if(line[0] == ">"): + if first: + first = False + continue + else: + break + else: + line=line.rstrip() + fasta = fasta + line; + return fasta + + +def remove_clash(scorefxn, mover, pose): + for _ in range(0, 5): + if float(scorefxn(pose)) < 10: + break + mover.apply(pose) + + +def add_rst(pose, rst, sep1, sep2, params, nogly=False, use_orient=None, pcut=None, p12_cut=0.0): + if use_orient == None: + use_orient = params['USE_ORIENT'] + if pcut == None: + pcut=params['PCUT'] + + seq = params['seq'] + + # collect restraints + array = [] + + if nogly==True: + dist_r = [r for a,b,p,p_12,r in rst['dist'] if abs(a-b)>=sep1 and abs(a-b)=pcut and p_12>=p12_cut] + if use_orient: + omega_r = [r for a,b,p,r in rst['omega'] if abs(a-b)>=sep1 and abs(a-b)=pcut+P_ADD_OMEGA] #0.5 + theta_r = [r for a,b,p,r in rst['theta'] if abs(a-b)>=sep1 and abs(a-b)=pcut+P_ADD_THETA] #0.5 + phi_r = [r for a,b,p,r in rst['phi'] if abs(a-b)>=sep1 and abs(a-b)=pcut+P_ADD_PHI] #0.6 + else: + dist_r = [r for a,b,p,p_12,r in rst['dist'] if abs(a-b)>=sep1 and abs(a-b)=pcut and p_12>=p12_cut] + if use_orient: + omega_r = [r for a,b,p,r in rst['omega'] if abs(a-b)>=sep1 and abs(a-b)=pcut+P_ADD_OMEGA] + theta_r = [r for a,b,p,r in rst['theta'] if abs(a-b)>=sep1 and abs(a-b)=pcut+P_ADD_THETA] + phi_r = [r for a,b,p,r in rst['phi'] if abs(a-b)>=sep1 and abs(a-b)=pcut+P_ADD_PHI] #0.6 + + #if params['BB'] != '': + # array += [r for r in rst['bbphi']] + # array += [r for r in rst['bbpsi']] + array += dist_r + if use_orient: + array += omega_r + array += theta_r + array += phi_r + + if len(array) < 1: + return + + print ("Number of applied pair restraints:", len(array)) + print (" - Distance restraints:", len(dist_r)) + if use_orient: + print (" - Omega restraints:", len(omega_r)) + print (" - Theta restraints:", len(theta_r)) + print (" - Phi restraints: ", len(phi_r)) + + #random.shuffle(array) + + cset = rosetta.core.scoring.constraints.ConstraintSet() + [cset.add_constraint(a) for a in array] + + # add to pose + constraints = rosetta.protocols.constraint_movers.ConstraintSetMover() + constraints.constraint_set(cset) + constraints.add_constraints(True) + constraints.apply(pose) + +def add_crd_rst(pose, nres, std=1.0, tol=1.0): + flat_har = rosetta.core.scoring.func.FlatHarmonicFunc(0.0, std, tol) + rst = list() + for i in range(1, nres+1): + xyz = pose.residue(i).atom("CA").xyz() # xyz coord of CA atom + ida = rosetta.core.id.AtomID(2,i) # CA idx for residue i + rst.append(rosetta.core.scoring.constraints.CoordinateConstraint(ida, ida, xyz, flat_har)) + + if len(rst) < 1: + return + + print ("Number of applied coordinate restraints:", len(rst)) + #random.shuffle(rst) + + cset = rosetta.core.scoring.constraints.ConstraintSet() + [cset.add_constraint(a) for a in rst] + + # add to pose + constraints = rosetta.protocols.constraint_movers.ConstraintSetMover() + constraints.constraint_set(cset) + constraints.add_constraints(True) + constraints.apply(pose) + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/funding.md b/DGLPyTorch/DrugDiscovery/RoseTTAFold/funding.md new file mode 100644 index 00000000..fa780963 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/funding.md @@ -0,0 +1 @@ +This work was supported by Microsoft (MB, DB, and generous gifts of Azure compute time and expertise), Eric and Wendy Schmidt by recommendation of the Schmidt Futures program (FD, HP), Open Philanthropy (DB, GRL), The Washington Research Foundation (MB, GRL, JW), National Science Foundation Cyberinfrastructure for Biological Research, Award # DBI 1937533 (IA), Wellcome Trust, grant number 209407/Z/17/Z (RJR), National Institute of Health, grant numbers P01GM063210 (PDA, RJR), DP5OD026389 (SO), Global Challenges Research Fund (GCRF) through Science & Technology Facilities Council (STFC), grant number ST/R002754/1: Synchrotron Techniques for African Research and Technology (START) (DJO), Austrian Science Fund (FWF) projects P29432 and DOC50 (doc.fund Molecular Metabolism) (TS, CB, TP). diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/images/NetworkArchitecture.jpg b/DGLPyTorch/DrugDiscovery/RoseTTAFold/images/NetworkArchitecture.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e80cf5a1880bd4fc47f39306cdacc87c00c74802 GIT binary patch literal 117715 zcmeFZ1#nzV@+a7`SQc8$7K6pive;slEVh`b#cW&57PH!7W@d|-WihvynOjkX4nhU*=!d0q8Q4(vkqE*T4Y9006u!0mJ}rprHO- z|2;s%LjMVHFfh=t@Nn?(e?AC^ZxIm?kPzVEkx-D3-XgySctlh*6lByt-~SN#Q~Xb< z*9#c|9^p@p|F!V)3xI*}<}J)BG!!}D4F(i62GmOrfCK;qcqRM_7x0e&j{t}G1_l-y z=})>JIsgFs2Ko&Q5(*YF0umw=BH#@)3@jWx6v8`9azqR)Hf$VRG7469Q597l5ivC* zpp$cKTzvHu5*|J!2Pdzxx~8Gyx9^EHb(7PZR9v5pU0h=lvU6%@#5LUfld`J12&lRF zBm(omTW7R%k|w4Bo!u8SQn_Y58|TvIy{`a>UXek;{u4X^01oOE9y~O{YpOiPt8k&A zV4-1=5ug!X^8o-*ulO)v;ogz4iYOa8Vq$%Zse;GGA!p-MQ3GamPU3QKsXE1OoQcL| zQ?RQWIe)J{r=+6h7B>-lSp*x~aOWJZFNmDlS%%@%lum`ZbJP zc&=xg&mzOv4qPOf2K98?jO-2{ym3_MMGY5J4oPKpmwUWEtKM3pF?pd~g2S{Qo9=&e zYOl#P#*qxdu7Tu*_*v;YnBe#UaZT?i-8LAB-82XqA%FL#?b;f<36sxS2Jw_3yC1+1MjRCjO*RDU#)bBH#aYBZssg_yR@yDQUUW@C8t*WH6%WCnpS|hGqe6V&Pt5u1%v9{A!bW zHrSCQ?YLQaE=`l2jGkt`Ky^>6cmZ4!RNb@S2fP5#e|98iJ}3M6%AS0wdI8MSGVjoI z! zd^n-g!^MV?O!ZU#0{rwO3aE)3&q!?`J9OP?|Kia-(ypZDR@<9`Kf>GYv&WX{!hv(S z30N@;^c7s<)G$XBloZTv0syG}Z}dR)XCS-3+*a5aL2dD8HDIYjjODvi9KNM=bAax||AxT+SBgMm1el?fnbDIl^MWA$ z+&zZ)fiHlV)Zkh37r>XuW51DG;qIiWltueP=k}?aY@R_@UzLnUB%0^DOxFJB&cm?D zgftM8{k$vD_>IRNI&G0x`BsRt3BT7O>b4L`Fh&-IeYrT;K;Ak*$ zP*qNmCH!qQ@%grga~wW8v(xzZJ^w5F`Bru#_1 z8V#Ke5obX~%B_@fV1tmG=Zu*cL+MdEKYevt3H>Fza}7a8EaQR8nN59R+%ZCaw^OKm z!}RiTC_HVGVL}J5tNz&9NTm|j!~9gQ>C_RS^rl>cgY%gXL3GWpR8&y#+A$m>=oz^w z3~6P4ZbA1aof@$g-wPmrQ$^BcVkb6N3p%^OOqd1Y4HMGM->@5)dA!s*t5VMwJv^M+ zzB>~i#wnZV{+pJ(3zJBV7gxeuo5Mm;3J3mlJR03xHqh>DYa9QdSclmbv%c0hkTREe z_9$)m6V3e1FuUovQfT@MV3)&q5uYk;*mgF3LoK2`!Iqs+q4$%vMkQNW>JY{pW#YDu z1R-X4rl`x~`<(lWb)OOJk<>T0oFC!STSOU^=j|A>58Q;&KF3+7C+xhXyRXeU&B(~v509jL zUX+?}(PO3HoS3fBE2iB8hOLZ>x(Y%Qj2ggXEN#$__>>k^h)Zz8<# zU?0E|+vdz!?}QPftUR0Rpu=@ugu9gxf(>PO8~vc6Zb&7m$S-MZ!^7h56IW8q~q!}IZrFJ$Kv%<&<{ zX=fI4*>|3`9d)Og5eUyD*_4^!-{d`%rl0lfRInOy99>@Z`R50(&A=92^t zDvZS#gL|`3yyP!D(QcxfTLpgSSj(;6(Itfiq?oxdecIH(n_={19CUOU^dy^j_U*RZ z>n}j-WLKWTFRzkcSWj!Qv_BfDYfVhprX=41?gdlh#gO<4T)rAlKArHj>(67zvyxg{ zZQEpaDT_CD(@`qm4enY!cO$`VorsqFwN`-khpMh7UA2Y8;<9#wqnwgobPcyMKSu#OKmpzZZq?T*-RuSsGs6z z>J_>%e&f1T*FIi`4MKsilt+IL6%aQB3eF+M1A&SAjQ0hwsBCbDKoFyY zqH#!X4oP9&bijcQ2p$k|?D#f*8ulI)D$engcBaf)$e$xgO;qamPN{dKF$5!e(JD6E*p< z2g)7$xEH|Buse=MoK)fU$NatY0ntGQNT3*nQqRZAVoHM8WRcm(G)7xZMg8xXDXT7L z=#3Xw)uJXTVH@&IH}-)$s8hX*A0kyJc5;c@_&){7#p@f+(SOW9|9HTpn4mQG6Y^Yh z3Fmwoq%Wk%&z{JZP?MxzU1Z5?CTGPxS*D#4K&VNjNTOYJv)pBU7zdgkqi7wS_dKNW zCevWs(Qc7Y(>;JoX*gvZI*(Ba;J*VBJ7Y;07w$dGP{SyOYt8gPsxnQuncdzj>~`}e zIM)f(ViE>^KsreBy?jR-4haB;f>Ks7#NyDlas9VHbjz%MYjOq?O!cmYk@%O;W9_>) zPZ@@*u2|`khKi4UIy!_z4P4`_BN8?~-P9M}%W>!q_tWmwx)fhps(k0b@d}Xn) zcPZ3ggG)^oWZ1~<)a$pk_lU;oY01CAv6vf>5FBGOhtQw=QykTQrRq9{&Eoh6)Gqf1 zYXMt97iTI#b8T#K;&%^t@eMf@@m`7CEfU@+vVq?vf*z zHw43bl_iR3$Yg7J|KSjc{xE_sq7>s2e_zAQ+rDz6xc z=p=!nuxPhX(q_w8@SZD`7i?mwmS74Qa1LMa9)WGL?%eBagY1=VM zQp9yIU{?OxT^EUdDsFEo zL@Nj$N#~UWtAx-IjIkDPxy+H9&r131d=uWW_cz-#6S^@kri#!BNlvQ~o5qOG?XDBj zjB_3=e-L<{on%A519byrX1#*xGs;d(+e1qkQC90}R~GfS;?)&Z``@SAEcR0)VXz;~ z@=ow2JF%#uHlB{3&4|;v7t_i}j;?ah6t=LE2s%E@mLN*|3%XJvT4yq;! zSjK+HEx07^GG|XZfUB!inFUhMsAF8q>nML}x5}XjlZtahxS)t?A1GMtHsik+yW6_s zFe`avSKMAT+LD;TMH0;}C{f#4;tK6eyPj5hs?op4l1qJ_htB)rE}K>DlKixkqtR0G zA@tsQ=o0~3beL+A^gy}mC*vxs(3y!;uch5(GPo(rhT(xucHn;Y^zM4}6HYIng%s%a7ZI`2*EqS^G+bXy`7gx*t({VW0Pj5z*F$D|n1Dj7jqOGENZo4#U17s~A*?9t!94PeJ2p*++W z6s4=-^55rX$9R&FZ;3k^sFtByj|Us1!__a;B}I(F(hxlZC56(OFOapH;(ze(hM~tC z^F+Bsw>yO$cN%os+B@lWQb_BUON27V}|;{<8wn~I(Y9SNGUT7gbNoi4(Ui$EEH2x<}3C^Z>l)M~3lJ*_G>y7>qfXx*w+%TLQyR>h0 zb>7nJ+EiCgT@iWkn33S3M7en1nir|2;E* zo1@xHHc4(}%QLu))>LtX5dHO1s75ot=X08rrDY!{G55!*Jwqq@erux0FStT&0bWKP zH8q#xtJI3uP<5N6?8_xImy9t6>ieHI^Bt5njEmYS5f)gaTiZPiUs*6~6P}5uHnQAwILICI$ zsQ;7s|48Hi_idw9PJrO#heBiSssz1LZrqHccpr^Tx-m#G#`kYM0{TvSlcDix%3FAZ z?o5*LcN^bTo@BGayU&)_gD$*7n5xkn;&)q_!l=tNN5Vh+SOx{$e!F!He@(2apx}A7 z_|vJ#{xl=3*G`47OPaM3duRdDZ}TSL9=383uOevgqPiOdx0kd%>q3KRq8`d`?;YN) zDLVSP|Gnt4!QYEk>^YeT<+s^i(w&v4K~5_}h1lFri%WlPPX*B z>k&30lqC|3l8_p|01)XVk~hY1!9a&V(J_)Y)D)xb{zM)D$2+C4F4=q(Y?;_1ZeAzz zX6UXz?C4+F0bIf6izDehBDxCMUdx|<^t=Ep1{nq4|I6G4KmP7IOvwK}ib$dd3VWA& zK}J8KMuVf2PDw={$TOOnV9I4O7aG3A9nabE=}c!wKfuiF%zmM#PU%qs!&1p;g%j=8 z-%i)v(X}yHLhA-bRXT(0TYKV!y>n!4^K_9+AS_Pq#YMH%sdK8EifWoQaZqWyl&j4P zpilh;fTTt`9ZsEmh5Hwd2v?ctb z$p!+`sG#4P3M#o{WM|ZGkic(<*pB)5s5Wr&v$xM5a<`DEMQi!NIBAh7JyHIhjA&P# z(Owa$7K<{+fY>?V@G)|C{^uu$`>Ku_qGpT8&HE}eGtTC|Nj)p{$PWRr(F_buY^Bbr zuGLQd_JRd|QaVRbV&nVS{NsgQEPsz!Nh4HrQHu5emk$deR`9>2>@F_qsjg?Abex+CWRL>ly9^ zu!iihq7u1va{N^{f?wdLhn&pW+L*_bzE~0HO6IFcx)42uu#8^&o`{K-Eff$ zxfCB$c9c897l1b@NZ~Ndmc6QGe)3H`q5^L=$Cowje&8c|&s2GlMgKq4(VlF`p zO6b$b9;Rdm@~1V*QP6*9(|IAc0Qm>|54*E30C~?zMKaaBn3a?D@se_Gm^!jA6bFrF z>Vni=cwszR*jS+{%CP~wi;XnGv1K{$jDziKSxTrn$ut#KWSe-M#%B(HZIm!Fngx9^`8 zfsn(-8yh!j11bC#Sli%>|H&&5ljHvBSU^$^zh26=bXREp8PDUoV_LsfAW$*;^8^4* zxZ%NiHuy);)kgq4XbHl3C^n$>@E!pD7ejOa)b7n6xldeR3xxmxmyiFD1OVUx`3ETg zK;U_8ze1Ztv|thdAn~)mnkz$l)zjC1;ed4iPX#o%;|W(H=1Te0U&%uK@c5tfK)oQ< zS4fUG$Ugy4O-!9X{vWu%NsGk?06sL4#RCA`RP$`FcS^Re03^FlCdip{PmmYDw&Ljk z>$lh?=r7t|*JNWsYd^S6*v!|7+oA28T7zM@MRf{#NL*gv9a+Qj*r_t8XqAlc-Z`u=iB7Ib5Xs9>f3JnkD zr-plZ68ZLz2W!c0dX8v#z8&w>_!-(ta-m4Nm}Rvw(eoZg;d3tun%e8vZUfVE5Sk1H z%wZ$_$6bi&edu{)Ns*7|$cxsg`HTvUHM<&oh?~7~rOSfzndK$->J$_*|t3d`$NY{ji~=oW0pjzy%DzA46{tefY#J> zQFWwU!%F(Rb6AX876&C72kSaTeC?6G&+_m{ts*dPH$94tjO}7SG92K3eYIx)t{PAk zNu2mr{o}IuHbJw@*oH&u%&C5i^UkD%OPpust;rJE{G&JIXC2oxe}2UsKVL^j&K8+A z-Rn6xMu6TVYSUs2C$5;*qd4z09oilzXZnYo0pl_h!XWHA_1*1KDeK=tSApZX)1As(``k~r(*J%YgITpOJGx18!_X2++I;I z@tkP&-0p6Orkzii)iSnL7R$(XKVW%d{ExXDt_HYdb%KjjC+&73mvP&S1w%>f#3((f zbQM0uYL64(xrC}#g$goP+U?fATlkT!z{6$l64|PW4e@U~bH+wczvLY4o5mPtfi*NfrHLnq}10B@B4bgKO4t=1(AY*0DGW zKOuVH=_eHHTF@Sz)rdLnsXn27>*Je=oj}Ms4Mp7g@qjFB4|uc0X5+fGV-L<52EXHM zSl`&w%{PiIr4OVZXzgS*C4O&%A5&6I9bq6g$9v@}n4KEs(`EQIywnetbIewI5l++S zLSioQqLbObnU7v5vyb=kcMNrBaja~Lag6PEpG^}QU8v~hjn*=Xb@LCqiHu!1W|3vH zSqDgQ1TjQ<+4b)zjfY}vb0YRIBoHequKHa!9r~8twU!pBgM3m)NJ4Q-5 z$~E9unltB59}`}Fyf;!KyS*^e8hsn%)miIwi^U|tfu_QIJKBYkC`;9RBHYq-(ITQo zE_0pGF=88wr#y`hV9J(dMo!L!cn`VxaJ7pqpbTu8wGzLINsJz{>H8&%1uU01V!JA` z>%A*dIgcdLZ>}m)?W%PyKjZ6=!3XmL1N%J9&w=}CF~o5|E(urjr5{_fDTv<#gh_~_ z@7eh4I=!4JDtgM?ESP#ZMOrIpXVoID7JBK1emD+F*H9E*4K$lrd1KCsqCX5Mu)te? zvua4mTvuqQQKY0K9&xUP*r^;lmA6;Zwnv%^C(Zs+Z-*JfA}F3Qh1+CKCTP~q_fVWd zSfRzekUb`r?y4w*&K7&VIcCFr0VKblClRmyd`OG`CjY2=;fhA@S5H&8KnkG^rh$zM z>s-!huX>R$O*>adHNH_E^#?Yii@vOhu)tx-Flw203fu2bAzO3J@=XY~&q52ouup#J zl`iYrRPXerSRFLm9neMcs#HaW@$%{kjfuVC^BiIcUMAnK75eUbmt2WMA)Hvl^hCP3%jKCd?wHwv`XZ6;Y~}5k}!1!hzf`-MPBJttPBvJvwc5 zjM#)$Z*>9py_Ar~9xRGan-ulD4+o@Lug0a&du0D_Z*FyIk=hUDTRL^GAi7F-{WG@I z3U^Tv=K)~^W1>VEo2UpyLVNRt`HA3f&jG{K&V}J+ruaQe$v3P(mIRv*Dh$pCk|jrD z!8T9gd>`Tr@S487jvE4tf0^?v2F;t45C8Ircy&fB+&&}88?JVolU2>Uj>;1(IE%NvU@cYQ;+cRGLM`2Mf_Q$3d zK#K5)H^)qi-&9D;U;Ic3mtFuR^N-L^th8SX_GHa`U;TytBSpyL60{*!W`6eki#Ac+43n7#q^=6i}ygU`kr`FTsv@e*r) zRf6R&b%u(=`ib(hU(x*0HtYWl$_rYHB#ych1zt^hzWMa6xs4OVpAw9Y;E~wOLcbnV zw+0=={G;pD6a(2+m5d(pOX^G9-bpOo2CP|B$;hZJqqt}xv`r;@ZT$3=*J3h$M>VV5Y8b7ni+~E)4~B5V8gGh zru8nE)}ux3!cq^JAh`8swldE3QIBi-G0^XQP#414fqI7UKQhyujTQVZdI2zBlfrF0 zK4jMD`^eN3Tzi_IeHY+MFjqXd8sC6%pZ@~i%?HvbcIA`R!~3b|)=smLIJH+#{ww%;&Bsc>6hLeOcH~b=G?PFJ8fa5hg?i+?i?~ zyzU4kJw*FEKp=`*6+(Y8BmS$_HZ0H^2?oW}&jtrVDBZW$f@ZA`e+lAM=6@IBdF3y( zAa&mo#e2WPJ8rbKnd{WczvQ~{SEc*90Yd*JY*30pF8@UB+4$e+0qGx-llhw>{8;}@ z9bkR`h3$Rhe@*|Ok$)>YbJc&A-Oj(&EJx9`)vIP@p8hnHe~)hsxfr7He{Crs&3s`5 z%^h5ceJJX z0yyU09Cd%Pzu$c*{^wBs!b8z6gEY(s6;H+RTjb;LbnuAWUF zs_55fk`mr{3Y924V^rT6o_Cn%9Mqm}5HbgGMZM{GTQ2M<`|292=uYQ*;WuKnhTLHe zI;h?Kla9Cj0uY$L*?x6PLs;-;t^M}IC8UXq*RbK^Jbi&xb0b`+jUtplb;xn5vC=wV zx(b=R8uif{t8!wwPF+rpwcwK%SJ(@^1^IGgDF~=ClMs755VUMKF-4*4=o{p&3SWF( zB=~$zsmNR^Sf8J3t-YvAE6EnG@20^X)D%DUet)(|y8b-Mo@N37!?X4k0Pr3UI-wZT zm!O^R=Ft@#^dMi1z7e*#QhjmqJkgpgMa`+8E>P%5(MW0#SSsgUD?L z%yj?u#*vv6eV{sv;1+9d^rT3d;l>n9E%;kYHFp;_u@T%5Ob*}dEuc{!mj1&WX=O!p zm1pPUIEx-vNV&IwLK_$|0@K}j--Y=W^kC#GNdhN|hUB1G~~)+fZI?TxCcXyrmEt=iU_ znt(4jhnKqyJgWu~G0oT%@HT}zC6$;+WKBw3OLNsDg9(d*-T9 zHAS4rscu;#T~B86&6DURlV%Y$jI}eL!e@1*dM;LtK4L0Qqnz&r5K5K}y!z9QhAtN^ z$sS)DHm2I1Z7l{1kQJhBCXlM9OpL(T4WYsu1MYI4DIWHWO) z2981-kA8>mX4>5^aYaQNK}q?=8s~C|Wkb=HR2U-6imy)6o~+ovxw5%%UCIuZW|YI- zW3|F%i9q6RD*+2bk0Ld*so72W#p>UI5qT|C0t}_@7tCB%OyH%rNuJHs;UW4y*Zf2| zGkE0kGCaHjrNWonNOW;akE)t%dMXDut#uLKdpspVfMg9ik{cYP$)bo4!Q%or7Hs~X z+ZU-&cmmk-Pjs+#u@(xQ&&f~~5rTOuH+z(qCzMC`JdogEaSv=3#nCS!#@oLK3$r`* zqvD?o)?O4j74&sg;4DvVPC53mC2=PO#=2B5Vvo{a*B+*N)mWpnhi5v?k8kG=jv5mu z7pJcOc0$^ z<;@dbIki=`qmQOpSfxl?|osYx-b)hwdMySJL?#!V+x2gF_QaZr8OFxD8#JE zt~WEYt#^D_VcXU>8*gQ{#MCtG)JcWAYcKoe$1Or}<}FV2OG8hMAmB+(P%> zx4LiY!e{VDlTuty!Yf62u#0r#N!Bq)K?h#A%92X!J!aUOmiYuGh+4hlO8=&ZNC*3a zY#@}Vyy|>;Q;6@%wKr_c4b}zCkL{jLeggd}qogF7sV^%!E%>_#O1wW+{QxnLT+}u+ zD37oIthW}*fwNBAL2T7#q;I7-^<1}@UVCfS8R{4!9Dyu)Y-E^PScR)){9||4DMQrn zePX#e+}5_6PZ_aBq!dn^j*FeiUcsjn=tI|*{Gy){gL^C!OrHT9r(KAhE=nRM!8Eg6 zf=}@o9?;BFstbi&^gywmX2ly-8$Z^Mdq2|8Yd$O_)gG4qRM)5n@$;xr(MfU4I7Wen zjs{#Cf69*ol8u9Pju1Zsu$mXo%xBcF?Gr53=|8y)Cq3BCP{x|t6&z*FTBQ<-dy|;^ zlJ|L|XwolEv$()^G+}K~&19c&T${eRUIvEu@t8Bnryl8Hf<{gUxIO_;96Hv&l$BG( zG{)As423x+)326?loQO9QYb{o5e4W+;FE_5TxR9y!-M;yiSva&xtQJ&pY)m1mo%~Z z*#$By2{b33BT)tvkVQet=`&U&M57y8lK`~(o}*AwB+xzD<8Pq{da zH1`*1Oq>*bx<_IP?!&)dF3S>ZNodulq<)}0v_6E8xd3~uh~}4@mEvXRKDE0@|Ednr z>XJU1c5p@~_Z$vhMTGeuR;WinthyT$rU=$}xC@tg&u&k~4*Z!94)in$zZBJ%+Nr*s z{C4=~JQnygLEz`9Y1^D!qW(JfW6W~#vGo<_!EtUD<1n-0Z*q_9npjNmn}@hL1IVaO zY*>)7$TX`wIzG)~Fi8ye2PyPNjvM|5w7ve_C4qQ+MmT$#?9>;)#ICc%6=}2mk1gt& zOMd)5u*8&Up?U&dD<>|q%HY1Fe^J|fQeEC1D3_C6{i9?23jf5i)Q;igme>1-52<{J zBrdk%G&k~m!JmCCE4Fkzferb5@1n`fR)7}b0gFJV6YDEToVVGWHUVb|!mRoB6mzDy zbVWT-9{zfp=e}wTm|Gm|xYZB3`LN|rb-AJM?KRI{3MLrIuUECzmT1)CC zjwPLyl#M}x(ey}0?eVewVK;Q3byY!-Jq7ZRGL>WXeplp@yC8v_H%h7}yccRFtuM#f zjNihHWF&h(+7vbLWQtLt1qZ`vREv`rae0D_H+6i!*DUw9c1|%N%LkNiL}UZeKOr%S ziFLPNb1A8dSf_|fSggy{Iqbd8ZAVGRx=2qy{2^&KM@z?RP`(Yjw8&;+c}Ag+aQ6V( z_xsX@UU|kmN71-=H{pdV?*3iRtODg6+YFZTxnY!C^fmLJp@nOryJQ@s9Y^0O+S|gI z<^o$)+joFt!uoP6*bxm>U5{{g$Goy)9-gwG*h@IUEBg z>!&u3g_Z8ifPr)hzUj21sos~)wj`Cqn^qio7&g`P+zt!-oY^fd^+iQD!Mmclz6%f> zh`&#JWYF?1jj^K0oLI7Bhg`+}0leLn6CaMEJ9%aHQ$UHlkxXt@)LvojFa-aEWQ|%Wir9{ zU@zpKkO>zFWiy^U5V9i`p|V5Sja{JIGx6!K3TFe9efrtGvY8cUg#3;N)oYk=iL>k+ zjn)Y(;n+CIHuD&WNW~%_<5nI?B*}j2k=7|F(PE)cfbdeevV5D%226$-m*3##=+d)OsOsI7nU!g0Io!&rBbm`Z_{7uY1X^kgB`AE5LWy%EVo7PHG?_0@`wyHHW?c zW;=7hiuMjar0s$! zgCWqz9bA)AZ68O!lpSC@#dQICGPs6Ua0<%8GY`Yeg~eKU0h}w&U}YYd45c+J%oq~P z);K>9b09di@($MQctI_6q}bVXkv&WR74s2GQ&B^?(iFdN%%_RO%k@q6VAyVY=QuNW^Nu%JBe ztXxEF<8sX9IlzGpZ@5I6HF6+5tzK)N?Z?iuI;*po-_gu!>UNU~MhJ z%;j;sd+a^fK45*>-jczqd68DcByIeA(d>&5GieUa=!f-Jz zp1Ou&&)Gl2d?IR8-h}r)BSNY{EbyY>hr(b-gKJoMvaP;45NqbjdQ8Uu#$JoOeUT@1 zJjd>Jm)?L~^vm0$H-(Mk9D*^$%%#^F&yC)*aq-`c0tH+PnR2)*v~S>)x2W&OvV+$a z-JZXCs(}zAJd2lUi>FpJ>USy|u9^<1*3Gz(=;-j|eKN3Pcn4PybEHzfGOQzYCXQh? zE_do7$z*!iSw9eG^n_^J{Yc0Zfmxb?`53^rMTX#G!sj`-!vC{obMJf|ho=eHiMNY% z)jWJ0oKUn-`WrT9OdcL6pY64a^`>^iyVX{eZ^B}lDXq}=mv69}s28h^y8$tm@sEc( zbG?2YgZ>{%sX~!?*4`D<@C8e>c?$Xd3mU;kq;WTaIBC`W2-qV^67n20jRUvDPj9@z-!iFz8lHZExKed4Y{~bFS1uqQBGa)YLpi+-)aviF4K7d z2&&O*#)C73GXnmU#H3O=)K$dw7t%!s!A~DYZ63sBrl%2; zQYL~rLf?0#c~H(=i#@-0sBn!>7>jGUPG)+xdEhtN%u(26wS56pdqYS)J^%nm&g~-T zuU-;G4gkPs`*-UtUeIGF7}0t|mqk7`X@xSQ(oFu<{N-|DNB5FZXG+80y4R+NnJ%Pt zH1@@6sN3Zy1K@X5B^n`>vUHPx>>EYMi<3@}>C^x%wOU}B~;u;9dBoKeYA^;A~SX&h7OwibL_ zrQoo4)o(Pfu_C7_mgx8GO10@UJsm>w)!Xu!1jbBl!rH=_oAc=wBBFj6K0USVPeuBs zv+Uz`%HYq)=vs&T=XOIup5XfrMO)V$Hm#}&+%w`^3rBsAVpRo>9lP-jPSoIj-tazB zE8eapL76-n4Ym~(ZxURXJ1|;!zZEk?xqEdsT`uUrzF}?h^7I-e3>>&pt!&Y>&u&>O zW}lNCa3=VEMw~b%tndK$so29cch>20fNj>ofcLVnou7zTaNnX@YQP`$2vwntAA*vw>8sDT1)b6Ux!^g*fewn zK94x9HXx6FRNBy(8AT{ra!q0JZJN8x$w~cfZ+FnUZts>=k2JpMjdh7fpQ^I#*Zu#p z7l4xUW}^>ATnQSEj!}Uzbz6hP>u>6d`kg9u?V8)xy!3TE#epfaWMtP_3minL+`N!G ze^uu?col|oe^bhk6b{VW&eAE0JcH(|&dVkd@wcW_;dy z?Ewg2fj{AQ?DX_+VTvt5A`>ayH2vwg2dKD^B(jU(l)>LEz+!&bbWw7rxZY3;@p8*% z7b1c_{OQ^Sb=UY6^OTd)yX)7JtRz)dpf9Y`hQ8VIaHUC)6@`koR$t&-jFY6wN90-+D+k= zIy=-nXq?#mny++}@o*S5D%Rbg+Wo%jK|QA~U*3dlh54+PUWKVr(?DJ+OoCWpqp4;Q z-SAqztsu{;&%13X1hZr96lQ2zuCfG{PJ(*@H(~JAeA?=)UHov66c#z^+MwYGlj?qK zKW~#icQz+KdnGKMsX7y<$?p_rBvn9!xlvx|>Aro5^9wh?(*yGD+4SCY}U2AyROo;9)PhI{p9zA3~SD z_M&D~b^1Cpq)I(LX*Tcd%RZmG$`pg#LcZ)mU>8`oCT-eZMf8Hag4~Xqo(J-q-)g8EBmTDDRtYxwxk?k-^n2ndI(4ZM#}3N~%9y zdAQ6o?*=6(#VUyF&`dkZ)tY|nY(e9^OhS;6P$Gkt5ahU^+- z35Jh0zWcn}EZh29aH>9m_=EhCXTpq_X2^y5j02FEjg0tb@T{AT*yVl3kGHa2ZwfZL zdJ*MRl6b9#zLwG!XU|PRcm=j05}t{$Tc1>fYQ?w-6K7n&!V%5gAhaA zp57RXRq-5|pW*3`shsIs)~lq+j-c6e#@cHNI&S)ewu-GX7iR9Mt2XzAB0hrA+2s&Y zC;jC;>Z~hblj#XS{a1QM@P=JjZNr<*?z%cR>f*54!@B))Pr70xmT;S97**SLAM+#g z9h!DVXQE-ND3tU#)nojD_fnro`_Q*$Kf*hxlb|Et2oKF{fQ=1$-=BYK*Z$6oK3B_A z*R3PKp17MI{;TsmMqZTr+nP{^B%3X*a)}0O1r2sWFYwHlQr6}gQEQ%W($-P83H*VF zJfxj4d{j`4f}m4I+9lly;nV3HNr^A+$geyV;+lNtA=|hYz{Xnn2Us-8%5TZGlq>aP z(jBaZnF3$-HGJMV)+mL!ZF+ghOhu~WsYV^j_V-T#iOp9Cb|Ot>MjMb_cG670MQ;Q* z$5>gX3$p!gaDO#8pGQ75)C79kBI(22oEW7i$2wk4q$gPp-U@%Bz4RtKqzPy74GYI; zfBtr`+HA)a=^vmSXge_%g?6z&z7z0xjyY3X%y~`lF6fECezVk0h;(yYV&Y=jb?liX z1F4QQF_C5GEcRF;`<|NnSnn-m7f89<>kyQ+GLkvss_#&~9IR~3NKnVqK9UJMg(T<3 zCBYpi3YbN*v!pt<4!$FmGI>Av{zQ-4mTF0Lddb&-rQ-`Ztt}=#H=m1)wXb8iC4(el zLsZi^@*Q4Mb7sTV&$5&B14mnlT|{HLTUQ*ovJKl2WL^HGqj^<&cKe+bREXn|lJ|$X z>gwEujAo9_OH{bL|1nzaZ*jHbfGIy1{e4-SO2l}%>8;T9WFL_ zs+8#~sMtQ0`#k^vY|;bxf)$WT!r6k`ExccY$9~6O|IA14p#}f8Q3v&FrB!HehbOHe zcm?Y~`GV7}>+njX%&#ox1wf0EO|=|;6I6bu=NrOVab4@T4OJ`N65~6O`1*G&+@B#S zUdq{19Xd_t`rSVO2r2Ea6xYiIO#4HWpGiN|X)c_+9U9Y!a6jOZwv!y4u5WWmW!I8_ zo8EYvmhSbe)JoG|1@2!WxVp=JT52B}hj&V>ve9aBanMUlojoS5wjj@@Cl=c~yF!=Y zB3V?we&n;FH#nx~Y^AR07g`em18bynbcu)r@Mg{VJVd)RiaN%NWG1D;VrE$N-lw@k zh&(xC6b5{}A)ZfFER)`tSQ?_%2X?Xr|BbS@4vORJ_Pr+w36dXf2^!oXI5SuX4jJ4v zxVw82EXd#zU~qRI9D@4{?(QDkA$OkVo_kK6dr!SpZ~xU@Q&YREd-vW;zU#AQiTNnJ zoY{u)zJz6Gzit>`0zPyaIH#Wp^Goy3L0m`8R`6*_ zzJqh1jrKrT=18@>9JYXqDR57w3hX5sP~Qk^NC~oV(&LmS!b2|~2zR|a|KYT!ZK}AA zbAL+7qfx2=a!u!pw<}m7-MHnAk2yx}q_Bxal{N()`r@Wd!SaIm75UWl=rhePPDNFek}9Rh87Jt%u*CqE>jNZZMFCC3%!n-zM> z^G~6un;ox~K88wRawiZ(VNX^5Zq3e@WuLx3J4Ot(h^} z@iW9HRYA75rm_tF zLm#iAn5zHF)J1YT8)EeFq!A)#9Fj{7_d<1wf9K9cmUfRT>K|H&t6^Ema;Tj`Vx4cH z+&}f@U`M7I++zO+qZ%_S=<;DpphKxiTbGAx#K>MPZ?fReHN5E+ArPBZBK`LvPKG6O zyKPOWeOZ#jhY!ZP24DvCfAX4(+uWsn)B7gPQ-o1JZl>B^L9c{Cf2{xHq=LIJIDS|a zEcrN}^~VXGAkhwx=d5BqHK*c`T%vddfvv^OPNEmH|J=hsAsR2;)8dO+lB&0zr-}%| zvQ7(k?3^}_jLGXL33BdGuSkeVVvXM9$|Pgc070P)jLs4YIWm-no`po~kJV^9Hi0$g zTKj`Y2TrS_kW%&I5N-2qzo4>7`uE$GQL-+lut~NQ{A%b&m_d4`_tT}YzBIXIE(bwD zXV;NT6MU|a>TZ8e^gZ^q6Zxk3AJ})bTnjH&wqDwWK0^~f1A)RN+bc|dqjFYj4cA2d zeYX8x*m}1RkrN{DjEXGkQMuj~W<)l$oIEV2qOu1KiU_wX_3@&Yn znLVFXb#oJHXQ$eQvFkv4Cu~*wD?V7VqqNq2Uc4r_zi!5o&wkWeyYs~yo)ryo8;(NV ze8}llnIe#^$<6-yRXh7VE{SVeo%K>Gz9g4>SO}$~>nK<7GVP|0&%DD}VZpYmk+x56RZ_@%?K|X0r z8Z}yS-2DMq!npj~*LLdy@ABtl6ZzyD{M%ns3+x$2#B(4Ryhh$71B5N;&{HwUTRg8a zxSixu2$Q~~!bd`>5S8~R@AV@lIe*X7AYnTc%G4X`Ls;aLHH_J45fI8J6szA}^z%X^ zN=Jk)jNLg2t8@$1z)2Cv+sq6_8Q~KXfqn0?gH!7R%=}?!Vd}f_sROwU~xHrF?VrX_ku5;n$^eIxS4bcbn!Y*uT2}*s=sMFE372ST0xKFOLi*_?-Aso{2=V-e@^I+H^cule>aN zRKDpESscfe{5U9lZ}iQR=%D1ObkKWV(d%a<@k!%PYV)@u>2F z@;}bGLU~7{#{L4>&1&bszAv;c#0zg0A`x8dKT8EB2=)>Zvn|LcHut(2jE@|NPzzE0F_UdAroI zxz6v?w^Pv2=TNFUk$!_+(T@3xht57eP=fVC*1;70LWEk)tS{#CZ1<3DJ>y_eK|#%f z)9+a~FGqXTJISQ)xl@S+1mjh4lEqVC@q4H6Mc7+ClBZ$oabBjr1*|5-(zT7}t2T=Z zHYY20@V=&AFotk@ZG08JRb?iEyzE&WCkAK|RAkA|)Lywohr5@<0Gv{k3OrmAH91AK z%+|O8SCVsW{4h}7;AJXUPz-l=NCIWDsE(Icu&7p7EkVFNOKDvWWWI{H%+9zvu@Bs$ z^R@PVXl#|u=pdC&l?Dq%PVVe;%<~~KE^M-{5sHiQs`7N)o6-KeVUtc)8iA_rrOb7iL?ku+7jkp6IMjPRB zf42=U+&dqV1kRopW6#}*p*$t~lpL(;0vQWTty?T2pu3>uF<(7hc?$!vrU2!#4k?-e z|4>W4{hQ1@N>T?2z5ttTv`#{TL{&pN`u061u%TLAdyV@|LjIYaTAG%gci~)h@`f$1 zumk*P6Qt@0T_H0ZG@uYdG%yF1A4+%wX@hLs__3TREov!fMi#(2x?|q-nHGd}68-Ty zc%?WGl330*yBC641IEYsbeJ|XI_(Qm)3O(X4WitwJO8L}@z(hxBy@X;{R=;)xj_8t zVd-9~9)ohO>z(wqWR+(Oa}k$A?lT`#n<7eo)h%etwk!6Wmqh${dfdDV3mK5=8=%y| zQpgzP<%MjSm5-B~okVKW9HZrltH%5Vc=A{oRP7hUOW`SHRL(O&2ROe!g9PcbQD9AY;rL%C8K&Nhv6sodt#T+-XR^uhFg?fGlXwSY}`SX9`O zLRR%kJvYs?MRi0`W{}5;GWY9MF)k1;;l4-qH1b1vtEuMVMCYl(VGWgTG&$3G_Shu= zAZa)Rn*EL|TW ztRH;@phjCe3)6izkY`h+?F!7-()E;`+pS9IRv$Bh#Q?F9vued283gJ3hooxltg%|{ ziIYxNQH5DuO@AS&(N0$`)?Vzx!lzIMI^bZtctOhDewg(CI1Y-Bmw-UYV%^5!2%U=R z=@p%k%4DK1#^S4r3E`v9Bs{DNPp=7PZvxZB;slEWtRs@Om%^IKA?hl+$#p3I*a=C? zNlC}f)TU4L>-z30gDDEcGSBtqy>8h2XZcTG{nwCy3*&QTp{}*dGctA?KOS?z}gAJLU?CA zt_5!QKJL?y8k1{JjiBjXKK7u}Y~x{OQ*O%qt0+zij@r7CKF>6Vby_2ArUdWc_d81^ zyyCg#lpQ1+Fc;R0t}B!HXH4E>K!3SCUK_ z*C^>hH!18hLNc_o1LbJViSH+4GWDe5Dmj*6udD4-I(qFF@hr z_0B|O-K|Dba&O*E340Wi(bf%Yl35XBf%4d_m%W{(6~df^r*8K@H1cOuJ)gx4_kCQsk##Llbv*9No$`J?eCh$uw<7xz=OJUn zuYiefepV^rigvzSR6qtP3`e5wCA*W+LB91uoJ_-yMjBvWY1R-A#H2~+Jz_+kqPuKYv30?Q!UkG5GqiYV zH0)>3?M1svKE5+B#F4}(9xqKdUOk>I62Pr>#ZJ!|Ufb^K^H6ghB3Br=4;Mz!=Rot2 z0N-0yj)+7(riGrEP&k+{yH<#!BVm2xGB7)3F;DS&vq1{7pL;;KUsRhlQ^yrUX zOefBO(jY@OpDc5tbEqp=TrKi4i>5dH0*%8S-Yo|Or|C?Z8W_h_mJSX*My?ZmR`FH~ z5?lOXc}e+fqR5}a9+yMGD)-v14U~!^QVTtBN0M4I=zMsKg#Q9~!>u~b*yYJ7qZ9&Q zL%GV3yasFeAAVsM*A(>n+mvHo10vI~t~mRFdpZpU6Qz6f)yUnC_@nuB6heX8K+Py} zhn#5BrFUiI%J@uni!?*3Dkh`24`<)U9S+gn7SrxOn2r@pQB7LIqW1W!Cj;&(?wvO& zBbK|Pl;Oaw7>?;I$UD;y?oDGBzyf=!9Ew^Z}c3P0YX#LCYoSMfZ?^G7e# z3--0(;q0EIrJ|ZQ>@6t$vy?*7wdouBuG^DDBv)+wfEVUK8n4TCQP}Ypkp9X_r;OdQ zSk10qbbU7?_~d3)tIy4Nv-z|^pgyFjlA+F-_@b{nvd3r6yvLxeP#jLB&4_ z`Ie)#eMLEvJUE-vH5bPvdp8VaMAUe)&>&MJezEXbjojvE7az0*SHWCy&}cCx)U|r* z#FU`ePeI*GDv!h|y4wC|*z1Q|qDq;(>}Na`daz(6zucdi!o~ zq;|hP6B3afuZY+E1o71?LU9`bvt(%cPwb#dxv}6vB^H= zxD9R~+Ubn|b0CBAqH|IElrc9oI~rQ3GqdweEIA=q!sCVyGT)!;3K14cv4gw1SO2L%@vmLZiN=Ar7t#ZOAnq@7dM6pI8eEqyuXl3?(pzU}+* zJczRR;gZOrZGfk6WXj^(N#P`K^=%Fht5oOL64-+#dSg9t%I$Y|hjw9=bLhc}^NoK2 zxDRNL{_dG`j}dX}D!le)iAbZL%La(pF)4qcTvci`9zgUI8IMZdJIp)K;wDTrz;dN+ zc7avSz*2Ij!t$60TW1>-q+h=gu_ozX6JJ||L!gzMMH6L%mgP%`q0haW;!lMYq%Kb|diI-c=IlS1hFPrHULV~@w=__lYrIxLjPB}t`6+d!lg{T(oY zu8M6s=0sU=xdh)fIIVe>+69^(nL4=sbXzt5JHWwkNw!AW7RY+x<)Kfk`_=VM>Mvj) zh1=$Y$JWG)*6g3=2&Tr6oiW!^($wT6g)$?M!@FfaXqo{7&NPdGN^7y32{B{;k5Hc4 z&Q>=+ReGC6SEhqOzNN(H*pQeVg`Xb(LMub=N_@X4&_dV z%Y6zej@|z9xRlD5_8H5OkWBUMF9QW|`kNH~_j`YyKFli^$TcJfk!5iACsV*LoXkMi zS0x6El6TTXi-z>bsOle!&!5YSsfzfl8t3T4{l5jy#khLO86gRs(&0Fv0@r^ebb<2z z5d#VCwoB~b(3D7fr0Equ<%34zSW2bZySEnx4;dAeZs*)zZjKE+SLAXK*xu*8PS`FU z(wyB@9qqJB)kcrRd*gfO-*TeJWnx$j;Oy2%VFY5V-LEQr- zSYts#-io2x4DiOCctlO@mMP2r>9qN3d%=qi$l^#5o8bI0euA@nb6rXNGn;~h0@z{~ zdVdv0WQ;I>y;K@s)i&iKWzFuA6?VK%pxh+bWwWijSx}xf66)h;8F1;W_(wcVq`UfD zYj<_w^Nd4a#<3Pf7Dvv39DCNU>aEnmpRzQ~TYo-eWSWQE@!zSSSzf+6b`#*T8Lk&! zWqz^MOFUuLQ-aX@-3oeJ^aKs5!;-2{M*9DwPnnVUFOwX&JU3hXEJgo4PN;J0h@Wry zpL?OdfL;s&>MsKP%;7cN%Lhii*kA3Amc8UNGEC^C>wW5w8=b{AgYJm%3zRk`({p1V z9-@h+y)tX3I(<|I8DYDu$Lucn&+ll?-YA#YQA7^h>D%WGqI(yA0R#e96eE8Dvt0{+ z0lA1rrI*}w^MK}<8Fu)Xo5V6&iw%cfAGlu>LX$U@5N+*}`MF1w)9;Bq+R>|00C;lv zR#IbLpt;D42p+0@Q zyP_PGo~@aF22fu9#unZY-Bam1EoT{N%$Rj{;&jB*qZa)*?1Xy;WpV=76Alv^%- zg9ep(;RY5m<}b(sb?w2Z$mFS1;Gb`N;k>vth3$3`n5w^Q4RkKXl5*^z2MFC zm2OnV(zkdiw{91sfG38D5`H0ZgQh6Ef_e3}xq%-q+f5q6n@BrDJ4>H)M<%XOKT&DV z+wwWw!vgpob*p#rI?$Ej9wLQ5aHUES@Mp%KMi}2OxR~Vx>I4@uxJs8Ilc$S?&IZQY(x5h%i$PnM!k+VwCrTh z{xhdaWKD+dPNQ>YXvN}qItJE>9526TFRKu3b+5}Hxj^~7&>8;kbk8YOQ40o_1$64n zlpK{hx3~AH84AWF=HeObGRUI-_xIvAfzxHIz;ya>wiT%?50vq;Ao4d}LW#L{&{f7F z6GMBlhaiEosVjZ(JtNLcS8=CZu)|XENoKqVj8(;WI7RwX^LqMl_*AK3zz-Y zOVe#gd&d-rz0Ci{&u+DENauEzb^eI5cxORy)C5pR)V6gs0Lg1Iw!~dWwEOvHrd0mo ze)w*)G{+^>O#S6^P`?t;fXnomRbx0)eOWH z$soLTrqm{4__*;PK6Asta*<$0r`)SXY#bLuH4m!LwZ#u0g4OPAmxt|;Vv@`#cKmsh ztJ9Dx2RnWpG~+aPpZVF=j#=$;$SHrA%qLTQj};A@Ikb|BU7vlyKK&O28tSz~@lh|L z!D$~$>;NLh@`?fZA;MmHPdz^dzWc9KCcq-=-vRo6CoaiC@!d}!a?<5Qr2gMr<@D;? z%JDb#&#>snEWV$~&KwJP%Kx0$SPWD^IIfB<_EGtM)W*kZBRcQ zfq{>6fT*;FL-kM7X^OD=C!()%xhk0QzMYQi_0siCVn=>PoQ1~ukvP`HYy zUa{7PP%rj5Qcu z&uoFS*Rg4H(jHFIY^dO2*a%XL+Z5e$!)4MYXS$Q>Lh?adM-`^ap?7)fCL4vrdM6!} zE`+ZFGevl3UnCBfS%D4^^ez3vXKmVN)n6}rq+P_#qQDsR!j1lf7?nJFRGds>YBmc- zi2>|sYIEs#@W)@3jV^}_UaQenuvwYL8AeSFKG*%lAq)c-VuIbE+KwpUR!|IE;l;pY zunma0ESq)xv`b(082-@~5p=`%4@Z3QA!}!(X&(MHE!W8`a1iD!$-5`<2Of)+mw#VPMaP${|dd#gbhW%dMlDBA^Zt>hhfm2!xmw}j* zOb`KH#H~>Cq*^d@;K62r)i0GzX}rSUo0YuylmH@7i;8GoeXx&5rZ+cwIPWTuNUkjcM&J ze5nv1y?{fET4y@?J6?yEq>qapgDl;mGL>d{>=Ffdw`tx$AJf;CO;4rjT8l5wrJaF= zyPI1_PdgjBKyp{r`p)Cc_^5#SJyCS3bu{Ng%4@#2_d-fzB1Sjr^LtArZu@gyGu*#+ z-xd4Y)q2|DCJwbP?GNLc@O zEJeU`j7C|{w%J6oyQCI_byI2a4z$=bdTGhtj!5D!UT}l*4|ak@-Zi8JIcx4Pr}6)TSv%$u-%T}n9x6Pi zPqG6WR&ZViPsh2&4;7NN=H0KaRn)5L7MAHAq^fQIn*QPGtHvGXxOI`i_}exUO_=+= z@vzv8Bb>RJFUw?bYLPXAL9(Q0%$1YD_Sx|K=PqNlaw|pCL3}PXr>mAg)0+XQUr(bw zs^_>8{0~ngNOtwdk#F1RB;UuQH54T4gM*`OKDcF2eh!0;*DX=q?JiNY41ZTK|07s+ zrRU0NWOcVar&?7`a1=#gI<0UNun)B@b7MjH6iL=t8sdE6IpTsSLQ0h-x z3+!pfn((<|<0$^KcR1#9%ny40Xkg2gl}JBhbOHKNkm$IcmN#ix+uoNQl>wnq_6Yrv z^rS&tPg26QJX+Q(`&op(VFVTypH$Ehx@JRaM?MkksgTx-%GR6 z!q>bHjl|V;J;JaU*RU;xj?qz7#qQI7gSxe(5!U*TS2gbI=R+jGA%U1b3s+GtH))#) zsdS9ET6LP{_>7Aq)u8})U>c-49o-UtaY|lvK)jgon`fH$ywVZGqefX2GVun zQnXuzRQy+ar>MiaZr)z8kQ>GLFa^^K~0(jCBM63dA%u>ISyi3)_(7kyAG`Fj%THTGl#{^9@(|=TT_ZhMy{kK$7OoOB z;$f!ng0}9=@$gPd0a|W43H?6%3|-A3>WsS?nw=J&LD+)5Ll|?ZgWT*;Z~2{=G&vFV z+5Rz2>J{5@Qgb%AK{F_qqH?p0wz<{G^yS7I*>Yl%@G?Hj0i0;19fmq8?DhbukHcBT zJKBrVY|6@9&XdK}qNd8iXWpeNZzDTLbQcKuTwm@F{_X;q)(B-+Z1e*Wh}mli+vry( zrl#>Zu8U#5L|?-uyd!@0M+&3hks0f^EWzHdLLD>{Owel^Yp45*0?-nn;Cq zotARM+G<~9W92`&^|o9O+ADn1w010Vg9@S%*5Z`|V@KgQyYzvXb7T%2?6Jd=^WK=K z=;XX<)xX0Dp4Klhk~>FDY>sLS7Cu<&{eEaE_Uj$q^*{u{HpdR6_VV@&@R@0je+IA2TF|= zQt9(@K9FW3J=M^M*nVo9$zB|wLdZj@g6t~Dl4?f3!-(8rP_-Q-s;&>gn zhvQc!Jp8?prv<2O>P>sr*Ji_(YgzDpRi!2Mm$H=5>&fcMUq zlpkhc#THH~nXx`+BYMfSlYRahgJZ7Sk#3TGx!Rn{{t8p6{KUIYYvKER1mz3RI6E8V zZIn>aqcH4p%oN@emwpTw)J0b1OmVgB2PDG~WIED=trFMe4 zsyG>4pOcV9$0mA|M~$7qn0*8PPVhcBIwLR5;EgRwUjfwGsyw?I4V?MWSX;N8WFnu% zO!=hXnR`8|w7aTv+T@*-JIp%If@pHP`N1V{E@vV;vp|ZML{Pjzyf$9!Zqw0SnBGeJ z{%aX}tvOUj5L-6O>bx$0enQg%)SdjgtT-O2~!`T zn!VLvJ#7Z*5cop=ck)C0=$-cz5(Lo7J^Mxn35CqWZ){wz{I7~xFg0DJtYE<{z;%7p z>B{voX-IGhjWQ?>;$mP%kqxn-8_S21QzrQu+)f==)~gb>+yYQ6lLvG@9%RO2I=&TVnx1Zf_tS0ilQ znicBpxZ-b5*cwmH!4-mtz|FC3kpn)D7}!M#iSL**^zY+UB?o%V&l^%Styi%Y0s?=198VI}?2yC2_@ zzZSNb;UZ+a5|`knNji;>$oP~d9v{mwhcpoQ8)gzqlw6NXjd%&MDCssgk$qQg$fEKw zT(U{z(o*{fnv5!JP=$UglZyy_b;nYI&~TB3QlT%SeqJ3jwK3n>4+L5V-a$;o1_v)h zYOo=cuT(pn%606Wgi{6)rX9cE@!4=asAwigIzRC8UqzM>5sN60 zSR}3`ohN&DZ_h@3X?q#`WD{(rekV}DbaG)6ceMsBf6TMjaqg#k#W;!)CEGOPHjL~r zz(d)(x=KN-e_?(QDy1G|9a~9oVX8)LkN32Rjur3N})Qd^Q{fGd%Va zH`wVU;oQO$BPFZ?-qVe3U;83tvus#wpVKVlHqLhnu63c!+q@ywt5G4tD^KF;l`83? zH&c^7bIWU0se@-lbl4grKu?X~qOf`>E6Tx)}(+ov?oM73NB?T)O}$@xDQ zz>UG7Ql;(Bt#Mw&a)WBi*rL=_M5fI?pDCrD*HQF=n5r|XQKpbD3cNx;c3quuH0s=eCidQguT@ac9A93mxKz4YsheJ$!}M9rpYh zpVHEGGnT&NeWTd7|5FeIKW>(mTmBuhrOaJOHmi-jkM>dO|73+I1$?Ue-C^->8cU13 zeI8n|tKA7ZsC0cYUTn^2roC(Jpx*c*#^2Ji`vR zuxt_aqm01u=eshQ_cpMgvdUMt3!SO<#js-q_oBl;&!^(tyrd8E(w7Sx@)n8=jZy=d zH4d4dhRY1T4vym~5^h$2KZNW?lKPoXoYY6|Eg%G<@DCGaDalNCHF@gM1`TOT#L?*@ z>t+=xF3=ndG@}0q>!Pz>SbY(1)YMEAoy=Kk z4e4C@Z2PX=v1wM{ekUjJRdnMdAz3JWz{lJQsrDZ>qAsKm`k+1CE`n4!mT=#T;G{=w zsygVO4_9vmm=)gQg@c7ET0!jvvUYnQ*!%dz!ZiA{zTKJc*ITM9SlikPQ~YeHaKl*s z5guFOqaZ~`S*KG3p{EoaT96Zkot3tU-9ca{`$bR6G1j?=)AsD8jYQ~?2~kER`Vuo+ zuObpRNJC=K;O18(r|>Z-H@Wp9C|Sr25vqz8T`MzjNt%4}jz0L~s`j!&(;jkF{B+kR+Eu>AlX`W?(H|KYm^@%8{;&ANhLP&Ey=bq0w-{x5 z{IH3IzmPyrNrqq3#6+^?M3Nu2Ql{9OsjJbewozuELH*{4|Wlt$9K-%crC_A7=XRh)?khJ8;6$+l% ztwm@M)dPb+2$ERFsGJX!EhNq$Ypf_FYD%>1{A{gu2IcMd;dti{eV^)nFHaf^-q%Li z^imP3ClBDk*V?n_ElZ?J8L~^K*v!9Imy!^io9UmUU%P*N@l8kYw7x%PJJ^DDHrpoN zad*KECEb~MLJxN6ThMH;CjM12=_s6@m)CWnQZ3g-!z)NtOTT=Zf6e;%>2%zN#b$w7 z8_XNIE+n+uJ!M>ZJbCwvw;lJ(DX){k1 zKZg$J>XdD~JsdODj3Ca2EO+@&S> zwER$sO7d&+6+0)m&-jt{L$Z|073~GT(t|YQSzT{SyqPSNlw?itsH_6_p_kYuOZli= zE2?|~--jlv*YY|~Z&w?$ka32+@`b*R97HfCOgp`j#x;!paWaEOr+2$<*I*@bsgQj6 z=zOD@6!LPYuD($d!Z$;jkT&GY*HJAFoQ3PQSt2?18bi}CtO;vFp#;khk*_>LUPRKvuRSOnaDIa znRJv-OvsV5N}o(Wx!DtWY>>+$D9VHed^FAA?zD(lQ)7}%n&)q^?CVJf?J#rHUdrOf z$2ZO(r2@Mq6rgy05d-PO`=WEAiJ-f*k;$cT#6xG&;+IH0X6JaR$bFIu1zd6HuCu%J zf~UzGbVz@|hIpde8>_^Cyn_=#Eix{aXn_PMsc#>6$&K0%`4ykRfeeZLsBfq9b@C4S zh9Yy(2QM91LMepv^-Fjs)O*TG^5%0t>>`^z*~)Mf>llKB!L`Ft-TpyBK3KhW9%I0t0v+2`i&dz*24pcB=QZ}`Xkfb zSZB2GvPNMZjc8tXz9DyT_3V*vXBtDy+lXNLT9KGTg~7jo?kEv1AfO3j+_H)dRmYwC zNkL$6qOP24a`Urx(#MuqyMnNH3hMNQ^ebGOLGfF&-yH?0Pjq<6M_jPt&G*j*6pqAi zz-Lw8D=E-gOQw>zPDJBU3QN$O1%A%i_=k)Nd-rt*X)5PBD6}Rv3_dEXSGf|B^ADP} z%2u#YWXb}jQ^W$y=Z8kjZt{m-=KV^bZOv0nDpVoL5`mDuipk_U7UDri>>j>XuN;M9 zdU?>w+qa-IZf32Bkt32)*haK53wA^3#6nkXh`~Et^FtHrs%FdLBUDwQ z^vK{d{_2c)tg%DPJ~T+6lfK|9myhAYSN?J&Tb%hwFTZ!_NgO;0PZMd<80V9wS*YI% z0|_J5eIm>IJU9_lwM#P@x9R7dUQ9NB-mK*1GNTQpzTQ;l!+ysu8Q<-j_Uq0nbByc- z&;S0aHtbg@U7g9mi`^&TZj5(G6q?)phnbF<@xOqCH{*CGk`LmVzU&ko8T%Depqq?P%Pv=xn88I~_SrW=!|K00(orPgtrF((uml7mYF=fTUYz8~pp zk^MVv(7uT5o?pt^O+BOaTc{@Rn{WSW$5YH}1<~{)p<{Kc3`?M%O|r(R91Vg$X& ziXt3%IZJ#c8`==#Qr}xUcPS{@jjQ7+ei0s}xi?T$r<)?%+;dQ4x^}x~KhlUFc}TWu zQr@y1fjOzCIsNbd3o*m6Vk=WVM4sbt?PV1#zZB}^t@`qe%lj8_Npfreub<94-cxi| z0W+Bh44TO}NcIabc6*mJ*Ab19et)4Tzm7>X1sT)JV=$G@aDzt+Z2^0nfL`pB82XM7 zL6Wd_6Cl&VITuKjR?)?8VnH-ZtPX!f+;|ej%hbI8<=IzoHP8OD^7bUI8iXu$PMk(j z@_4k^K=udYvB=8zq035V2hOZxwu;>?^P57li)4)!`~z#|eRbN6Y~{SXyfY+KplnQi zy8bS>_C_yhA}`naHI3#tW%$Tu)PCzcn%`)652@AK$YA^-xr1`C4dW$lSzgtWIK|*t z!SV*}DfarAm+^S1vIMlzIwrdR<(;mxzY_wZfa9N!Pq8Mw2V0LD+WEFbDJEW|%}!q7 zYngP58}L{^m`5^^ORZK!(Ihk(Xhd6kAj~ep>SJi@T7tDGs*mHiU%)|mFB{zue-V|J za6!Cl8OK4=D9S{?VS}^bLNLqwgJ#WV4Z!}N+sS`F@tn)q;S*brwO9YSufnukXSqZr zx8Ra?x@Dj`Gog*3f>cRvlW>T3{AY9RoqFN5p-(T~ z>*><3?5K0bowIG>yAdy;Er^?E2;Se^ym5-?N4kOE;+JWr0VrnHtZ6=?G_)r|aSF)^ zK=>x{gpd8>?y2y*O=mMYXQdvgrvSonX@k_}7?kFCde+SWJxcPNUjI zrtb@DsAgQsHt{Wt=IdHcSUxi_ja$6Kf0bt?nppKP`cet-hE03YT9aV%Xa5M{R)yn+ z2wLlWaqoVApr!2spIuAO#+Y}t0i_DGM%pe~=wD;U{(eEEFNgv^lPx$`M;$0iLHhz^ z4M7;7Lw6axE)hRRKdfsQYwbLc$9r~JQo&(kyGvujr^x5{VrSa0-it$*=?^FFkEdB& zhW~bnbMPoiJv;nxEMbzOm_BY$vxCYmOA=rJDq z;-bh*)XNa61-p&{n*n$CbY6abv6MdT-T=ap)tPX1c7lHq5$7Nv%jHovh%O|S?D!ZO z#bAQ=*sA1|ddEhoKvj22^YhmTz!-rS9hb>4a(YuUDY&8-7O9pb6 zMR>oA1ycfIO)1U2OdFIPfb!lXiDHNlnGL%)&aY-Pk(DUW+PGO0>qX}u!@Kc#USEr> zr0&U-FBgynasz=3(Z-Q_-fXEk5~Td}Kd3+3PJRhIdetSMyew&Vw5K$jPf$At0O*;1 z``;6q@DC{2H_-)X!yqeE65lj!Dz#iXKvY^h(S+rPCex;s-iw(n3b%y=s=hJVe{u0U zYXzzd8{U^_mg+dCe}7D>AFDvyyT@^Rp9Mu=QX-qjK}X(vE!L}Cu^3 zq#y6@jc$~{rQ(M-huh|w%@j1fT@>}3%?0jp6qQ!jNx~K($|H?ytEzK z@HBTOq^1>%nSX@tb;8}NRZ$r@wwfPa_EQ1VByKKUCUNX?hqu+!B4WA@4ACk=dc~W! z)p*;q9wew9(cfd8S**MO{`7XT?k!>Q@yy$NFO3I0&%Kqt{=GojtvVkC;8=;>DN?y- z+#QNur(Aqfexcnx1c_WP%CWuh2tDv1{tF;3*>1e=iF3>SoVml#d!G|%nQKVg@WY4Q zjRT#q!-mSrYF1{3;8>cMw_Mt=23w|u{4LpoeSo_o>S9h1Ou7 zU^UZL_T{H=-T&Z+$aOK9dBL zObQc0T)9}jL&cp9=KIV@Q2Lj$dQMb2+Rdax=j~zw$AZw(n6wRtm|?e9+#jUbi&D)z zm@bHfoCuVpphVANAkycm$DGD08cMH*4rpx`$ga%5*aQ^Ed0g&tCTMRj*R701qOpBX7 zh?I|hlyB=4O>7B(%D~P#2mmlPC+cN>*ZUvQ&nAz|E8?=11~reK0|G$b;LCqmU2u9r zQS;45%_Ew#yM+LgbNb%^oPqNzAKVVZ)nW?k1PSUAvYpUSr2`M~^f*cYey-7kqB{kf z=(8;LH({zLQDkF99rwUh~pSyFOwGU^l z8VBi8TcA8JZ=FL-=z=Yr9nurn5ZO40yfZS45*r>`lF(-?@ii=~@v(h|tg18F_!)`Y zR}+9lE3U!CiR50WLU=-gO0gA>nsv}A)>Pn5J~>v2Hzqbgwy>$xgeg*kO01Z`jhJtu?wWdQ%V(Q@v1 z_$MZmprg&?1v-7qIpp0!Yom_h8^8(GOBLdo1Y$oszKB6iZz?q3rKTpYY&ukjEY;#H ztJ*TTnL@$xG&J5idoF2&0t|ZYg8$+7iQP#tca!0uE~y}Mt2!mJd&x`#dgiTYDDgZ0 zr`eDOPHY8+Lt+yvy@rL`60V5Q&sZ6u$Z3FIV~4#8-Q9&AFtxiBF9UM5@1%6=1?&_6 zuiNEBB~I;w$3kn}SG>!MC=2@Dq?y)5%rc0&NFdc5y0?@9%2c+Uuzl<5WXm^ zK9TWPc4ftyd#1ea^;;`He>Fx?@E33%nEf?)aim~NBU0yy0;ZM*p#Dn{lN`5Lr}IcXsg4?Li+xs4Qy1IGd>y3Z!j4J(JyrJi0A~s{q~!iu1tfg9aC9&$i0NX8Iw=MYfaU@q{viAY^1DP8sjms z1PDbb+*mx>(8lrB_|qG?SL8+Pw1iA_SVIa1!VyE?`{^QNySBixuj_2aqA7(IL}ISC zGINOrhJo4y%8LJLP4){Q`1Rip$EDzBZX2Jy=>^<4#WS+rNHzMWAppQ8Ha& zH8C|Nm1&Vnxas{Nn#G|qrCiwiFF@cGGS2<~ef#nn0|H zh#T(}o=;~lOz`?p{WSl`bW-~+YPQc{%I_P%J&xpA>Hofi7on@e6a5@S{@%4oMx1nn zirGUMxg@iR0$_^AErx(v#em1TVLc>V1@5MhYW5{{3P}M}9oe#&iEf;s$hNSqV9^du zc+6!_yVr7++|FHt-BUE%D*OeQXZmNfKfYj719s=wFv-cqeJ=x%(s_K)h|MV>P+F{t z5sBs${dibdY0#s#sI0ppeb<=5BR@nwW5n1fRmJ;6$=tNiGm9GpM3D0h8aw3pT^!Q? z1-O!vS|p`nJ`I?d*bVod*nKadX||#1!(a4bF=VLJ!DGv~xMVMAgL_+CzU-B8g>-(m zTk3G!DQVKAZuH5nWYvoyNp=DzqcS4YTAZXwKm|^ueA?kr7JA$8YJZ6ii`)p`Ti(MPF?8V zkaB@09NDm~;q|{*d+WHkmS$~qNJy|?!CitAAjsenTnBdz?(R--cXxMp2ol`gHMqNb zzR8w%@Ap0XoO`}|@A_lbn*L2scUM<;SJkR|>baYAr6s_W-Zqsen-*Aga~{hf_Y>e< zeKvPCyIZsm#rI^0P^R<=!JH;7Am0Se4+`CRzkY`JB(WA$k%){EnX;<^8gc8sab1HW zdqpuwX1Z}{Y#twN(J{j&%iGMUAr5N4Z_9`z=X%q6kGSJ2Ew%;#bTJ~^$5+MUSC%gJ z(6R0$1lm&wLEj1o5CQi920HEPKLOe1hMqwRN}HPjK3@*q##WRc#0p}$zM6_nLjxl< zFy~}x)ADC*kRG5tw8z@G4vVfpX1nK9#ZPWlufBWz#x(yZp|$=?LW}=PLNhM75K#q7 zXmGz*1lQ&s@vt-pE3sehB5Jf!mRy>6FaZ^~e-m_w+pW~@A@?jFx-Nn@{%8;YcvVuQ zFmU((#uVSlLy%VRyD9!%1f5b9&h_&HINzNWb?E7Yc|rRCDKsbx94ZY+@)B-x8OG>) zgF&*JoBR^8=dwlCTM)8E#X9#R#_ge>gzz_d{rSOXZ(ubh^kYEHl2oh$N4h1PV-d?R zLc|Al4Gd6RCum8p0p#f67&&{ZkOKFxM9t<}MBOKW8cl)?InPXVgPWzB8ie5Gq5Ry9uMp5(paq^CbA>oGV zSdC4sLXufyh96(R$TY^lFwL#%6Mu#IcLfq3J6t#jjumuSW9)Bsw+sk?_!Li!qv3 zRGLQ&gQ034xjHn!KIF1iL37daZ;BH&5hgpDIqgj~8Bb!M$Q=WNc&;$)cicKH!_o3A zWv$|8;bi6C#S~lg@V9cPmX`^uS{%aT-tBeW2AZ>A*zfoAn$SGA`PMP1o3ls?())Z3 z_#CG4$-v+Lcwi053!$~7PX0Km;at9q8etYJ)bM{9NYMf7f*JRxy`RDKf2EO-4A@!P ziFUDuu+nh`15k$6L2Wnid00eET7_p*jv*nwC65wgyc?4CP_qLZo+;i#>>}bFw#Wr!kYgMI+Izb{wi6n-Z-HCrg;U z#N20cnP@+Kj(Y(|@=5LR3&rq&S`oHbbHN7_7Z2%F@fxT22~Jnnt&4s)*88mu7Ila; z1hV7xTS1A=_-`usC9$@!_ppz2vCo{I^RF{&f~%2^W#%rbmv$qUtZh+$bm2cqBx#Ls zp`}twF4ksl$HoL$Wi1v9s-?Y|Zz9(aIwbb^kY5GV{yGp?AR7_I*Z7}EK>s0E*!~aR zwSQ6$081C0kpoJ#RG!qDm&eGIaf^M%HrsURjehCOd^KvY>R)W(pQ|CxV-E2ALe5YF>gpBCUXRjwwiMKH3d%}B{wR>~IUWNtpR?;4tgyHO zHE<0z&-5gbfVnh4(J_nZ&}6QK@f`e@qICDRaj?Jn2B0~!eFyhEL9N3 zhX%>`M=Ctjeu05S3`BOU&q1RWYUye!Fv6Ae5owjSA^v(q(^ApRf;Jd|KLJv@U%B^Q zCCaLlw4t+bs???ynXn6~PA=S0B-O23_ncB#1Z^m36o0G_80Rm_Owr9qx8B^^3_$PU=0rpg{sm^3RNnkE(@e*XU0ItLG^KiEHTFd9^2#~)*BmL7I)SZ?^H+rE3GOvwgD zb4Df-gE)rk^wPo0X2j4N(EF)(-8CV(bS@a;7H@ z7Lu7PoEo!i4pb|)evqk-x7#Q;tpxzuW!Ngn0KInY#nhuDulm+(z2t3`X-B&G;c*Hi z>Y{mqn$W`#WuqMq`T8v{^l}@j+IjN3lJ0&PG$0fTQ22*PGPa}z4c6Pz0}^zVro~bL zsRd*~kRkD2QC}l%SEe0Cesfwfbm8&0SphZ_^}UR$4EW2F zcOV3C$O)Tm#a`d7MO$Q1N%26Ohf9ZS)&x2VjROZThAa#lnB~vT-Tw-I7~i~{R^ArF zv$+0B15R3f$l8=LX@56#+C}Ihw8^wh-kSo9dIv_Laok)Q!9YYgWFxmSFlINv&&}Fn z$A*=QFRvabyfHD>yGJoMUUOq6`TQuzac=N207_Tf-r*p4u(&k8oTSH4;8QkP1jPdC zSrl*4GVTz8;g5{i)W-E~&r%_D9^|&kRgi%++evkfxh+nlIGr2r3C7GiRZ)3#0Y1S;5@^ZF2kl;N*T_ag& z<~DQ7Q1?~0pP_rnQoGO1e!a=6d5&V(=9S|XF-ezdz{ObL&Avoxr(9B56N`-SNk(&% zl}zm8UEKs34H9qqkGgep=G{53*ILfH>(AwkZ(L|5&9#kLrhDeoRFPw;p_6RICyg^y zu(+J|wzf);1;G*m%89F_lUMaA7K`}8W15`>Jh@p37NtZl@atX_^fioy+Qb$TkHOp} zmGLF=jjo=G?pjS@b%wRJsKk$@7{8JN&GG|wZAGeUhY8UwBy zAc3D&yTr#?6>3t$bq?O|Q|#)bgPmBSKAeE|fiaS8KQJx;)FZ3w#={y^rr=a}-(?(H zd+LT}Oq?j+;eWHAaPFO_;KME4TNc=l0=Y~d5*#6v+cr8Lq$nf?zeHv(%9t-LQPfK* zSpxAW?zfe%r)1eXVsvZBDC>_5cZ+_6u*q0S$=Q=848M8rQf*D<>{WzbM52z2dJ`o^ zS#xCUq<6$>`;vHZk^p&>8nzu(UKV!RsOYa+5Nb#uON!v51lnn$g5*WJ`UvVdlSoxP z0|;Wk`5`=Oh^P8-85kv#MzgAtDTyA$C9M#yr`Ms*_UUAJxMOnx)??GrOlNFLh9IQX zP1slY)jsxR-(w08tu|?@!aMZqG$Cy59vMnJNDc46eI6pL(jP|SwN9HsiADk9HSw*m zFU6)skxa9lyH+5waFZW~8mykFepXwgqZtoWV1&zwzn)e7xGTub=5BQ;eQ53}A|1UHCXr6zJ7g!>K^cL{le6kF zz=jPjr%e#KX%!bYJ<*9}%2sch6Us;49;jQ&*)cl0={`w4*FGA1<2I{VJ_Jrof8WHk zU1=zto+bGM8NFDzAUH&)`q)Z+Yw1F_&>k@-qO2CTpplR=Xt+?_RJKfmGu6GnY`Pqe z8&t)7G|N%DVkROv8{YMmI|JzzaP8n*!cFTs>u@cNhJx(mG#)k6aH3FwPm6p6@vuma zEW`J{amh=|iV)SBL|#G&AozVKU{zUy|y}WfCQ62MYhud3k7hiu%et^{a>f|zdu}>R4le6l*QgA26lKH2F9l`eW=eDm0hzN1(DND;d7(nT1?@8FK8gR z47TUB0jD@nyQI8qrk+6{Q65VJxeKJ9>8K0_3z06rr$FXI+EU*=JUD~w`YbDtHfF{EF;_WEG;URZ=aok+@pXBQz-my<7j*>e;S_FWW@u-D1&w^sjZ;uQ-n zKD(fgZRxt+xFEOlsC%+5X`}5lSB96QVo73Xfv$p02m?oS)K*bpbFdMy6Yg$TXdgGH zsvya_e}O_)`J0AfS^=>*2iZh+9x3{&ZmqbUCUz?1TAvs)cT)zCAlM`9#Kh8z^Qlwg zA%i(#^-#YJILHLS&aZqvMi8?doOx1fWH+Egup@ul@A0I4B8zDXQgu@fkdl+6AGa5I4PQ;_$Y6L^$9YX)$NfJ4*nUDO1I1fay#4p*zTNwagop}0Bh zpz%yb;gmh}!#;1+=yG@z_tLln1B$2d7y!i9IPy_p#@Q%~s9dd>eMKGCpYLr}(VB)C zE=#0|WCzBeq*R<_cJl4?xeLcR{!&YeLIdA}LanY^>VslUr49WBlR}TOuiZ1VlxUd~ zmN`y%Pz`wjIH574#J8mo*G_6TL+TKH6sEA|64+SxPGgprqgjwg|o=+xWai26q4Rtidn*2gbL8wD|OHT1vJyUiEN*YQH+7@Wv`UA;FA4 z4TuYzXyY^bag^Gj{qU~^^Yt=;wf8;RHs;BM;v8&731yRRe^Y?kw!#f5 zF`XOpwT1)2!juDkZG9Z6TI@EB29&fIgTCZ2W25N+Yk=M14z22td8V)2>5U0ofW1dh z8hHKA@I-{8cF|rP4=1;xx8bj;wd~6~tR|%EwS44~pGAi5uZgiPK2^`jaund#N_MKU zqT3kyv_3Vo&7B-Vj&!j+eOMY_zUN+}`4eyo4X6f?g#WoT|5}z1|G4_WL?Dj;(;pSx z75gpYJNl#FH;8f=7P>x%?e@g{OWmk+@4&t8xl^5LhSHch< z;=hJ6zUA~SRE!+As%M10$M|zY!ZQ}5~w?1{-YhdSMEK*s{NB=MgTY|eC&s^)S0S`+UTXV`Tz z^up^=CPrf&)+gqOA&KRthRuyB zUeO#8@@)M*s&q2&K7&&g0R{UZe8@wTdN4UJ*TptyD>p*+jl!q^DZ(}%VKNktQ#v^t zSQ+JKP_dxEtz8Lz>yTMxC4b%HX}`P}2R&FGnZ%bmhc_5lL(_Dh!nns1g7zjU=7Bac zW6SS!+YVB@Gl&vVzap=-2@Ed2uo#u9Hjgf`B>Vh{qY#;`-!>-32#msYh4d59nkqTt z$Gy@3ukg0xeQF>_%@nzM@>qMeMtImyKs&29>*pE*QI*gE8J5A7WpQ}L(ljn25jwG% znPOz+Mu7;&72zhf$7<7TI}D=l?{A^p7oA;?)#LceHp5yZrdSVL^UL8V2hPhqQs zJ)N4u#=_d`Lut{rdu?7rR=LJaZlU8`J_2iiy)0gMN0*%l$OzBk(gK{0>ORAXUPW|2 z98Jt8QjS3mbw@pW1vfedzD>c|1_3lGHve=XY|eH_zH;xoj9(+f$Vm^Bi}|!yhZDPJ zKbky-V4Fo`k09w0=DH{N!0qx9sedcSHVur5;L^4rL1|VzN%4!2l#IyfPd<npF6@ z=io>@Jsl@swplQ@rDU=?Q@y(|%r&=gi@vO9cfa>B&#DVj;imK zzF7H`!;d^iEu({!8hZi`qPoF-?x^PebJwOsgt;C|&ap+&rvZbhZxO<3sQ`2((vA!kM}t-o-A+6B@d05;Hw{5&JbBH^oj^8x-yA;6Q&K!U!<{L+~d`a z5dz0!Sw`kyQcy;ub4_Q#Lv}r(~I%)c5^q5w{mS~ z-_5t1QV1h6?&0%?fsy}jr?@dKa05pC261c)!pv z9y4)FO018`V6t5kd!4_|*vf{$Ypj2KJv*Qv$S5as@ty^Jye9B_|5B+cvS=ZHW)KQ5 zX586sK?M4P>0w_7Pb{hGj%J3zj|q=yqpKF@kzVeEHMK9u21>S~;yaVG!V0m?(MBxJ z!M>W@GD!>V5h_5YkHP(vZTyi>`hBgBtU(fjpfvz$3Xii_BPxQn&YE#@(>4cgE;%h2 zL`u*+>*j%T+zYPywQ>gT%ErUKgeObg9mM&_DqVSO-=(XT7hqu?UdPSWG2}3h{%Lh? z4XTgDsjXR#VC2{X@Y?VTFz9c;@-r-J%hB<~Y07UmXX)r+Lki|E-6A>o|bf z;eS0yIGK(nhvJYrbE-GRO>qU9g2QRWw2A^j8A6a<-vKJ0{$1<*AHP9dQa0QLjJpku zp>rFd&2cKf49UG@H?$Kbmn9s(qMqIQ3Hah?+jF^?agEk`4>@DjZiKAaK>i2?zd-O4 zKx9f_knuv1Xwpu<=(%!iARsxv#Trx3hOpO!bOv~p`DfFw7)FHjwHcDWnBe$urZhxF zl1T9zz#L<^G!G#|B9Y-^hfwgj*MwtbwUJld2$dJ<7!Ocgh$ZL_DHdca5fQKn<2qhh z-@5=aj`pyl>ZMV+!s=!q%EJWZR&lF$u+upx#70@-%k#b|hbP#H_|y@N+DQtI@<&;C ztZ+dA#GJo2u^{@=Zlha}1=yn4UgR(@=BY+74Wh)hY6f@DI;iR0!ccg-?cB?6dtxL(-D`XZD?uo_0wWMtIyKKg5+=*ns|1g%dLIpT^5 zEae+^g~YXT<6fU{9?>}Oa^pM+YyyO#=l0KB1WDs@U^h&5)$ixZa|yHalzNPotTCI; z*$~bx&ZBW2nOw6|$EwLb=5MC18omNW=xwrX^FP+|Sy?A`JvmuJzX7Dp25a5n6q?ePw zlq@z0+7!~)nHaiN1s)-htDuSq^z)(YBSEm{2FZ5W<Q*4_kQK^AlHI0gcMCH5jlC;I@pzw>CfNbE6mU6 z0TQKN=-}BSvB9*jTAtf~$8Kv-qDmWn<7v7NyXpHjrgUTfem((Y+=wYZUqp3XG*HQJ z#%CF0B-M^yLjU8^hhh5#RBYKtNQ4`ua66EX%EWMENEVH3*R}WHp_AoRv{<;`Jg_IF z0xj@jvJj&JvxMTH+rIl&f9NSi{zv#{2sf`^qkzJ{{R!X^QK(OsV9Ds@d9H>NUHu8z z-G{B88t}io69+q^^LDi^-0y2$GyMd-*QoY<89Up)Lyth1-(l5W!tJvN7!VKu09t9H}6+tRp>*tqC$xF;ZBctPV&M417S1Q< zUbZglExMn83hWme?Wi+;=QsO2>frM3QLP&4y9#h`c@WbH=0C1mA3k?n6TElmo99O+ z+6%7f^l>U;UeK;6tRpSs9FZf!=$Q=DlFq?TEoME(-Mi(ZNE(&g@tAFWt-OqSuwa^% zQA{j0%sW7+ENtj3-QJ8=^n(tKFq;FIv$k;Drefce6K8qD)cc)Po8kDgp#Dkax4ZcR z^qpl)U#xsX#!pTng(;jEM5xkRUG99u{^Z;dm6UDj0$OtktL#eR13*M#Vr&1aBvomhsUO=koSeVYb(er17x4QB5%z6r~e*FpfaH$nHAAH^gp~<43 zc4+z9nu_`u$3R^UF}-|okvSIBi6WpLlwH#^kAjAfG||IHCV@E1!j6ZmW}`4?FsHT?!${VbF0zF$vTa5Z@WaZ7 zA~eIf!Mw_Jn=E1?>DQRRN5J@|Fz50E?cSwg{@Nrc4NnvjIvkvBz`?ykU{rE=MzS|+ zdajLBy-_OxHQ#1t_y?~^bMJ@&4D;1K842-SzIjf!lFTax6_We&T`eO}waY-Cw|nHq zf#F5J8qP>UkRWZYx7{=i-6Qj2A$7UHL4u9{gQXhXjtR<<_2D53E@EFd;Jb~p9#Qp( z1!_ABmITeSt?I;t#fi8sJGF>>E|Re7n7ny_E@<`zY++d1*g&r1!H_;1R@jrAhS#TD zF5>4Pp0F~_&rzBK*%0g$2<%4Fy|zWZ^q_ejt=w1prmbw%AtfL`hZh=)uUfP#!j@m8 zLUO~Pgs0~mZKHR|+Y9%CcU z*pM!-R&eKwi3x2>#CGnN9k!B#Nm_gO`?&(5QP8_c8B}1D?Vj|Obl@T3f-sb(vJTBP z!^I(I37n1TP~%K$sGnA-s&yl`zBkEMVfK7l$9wXesyp|}x4O}8dBXYzfpJ@<6F!lp zE9f>B?|}njZ?K57oou4}vOU(mPH%EYRFfn9VmXNNcNa1V?~kX?>NT_590GFCID(QU zg{Es?8!IB{BC+E@O4Tk4>H#nB56rm$^LI0+aQI3`KAdagwr_@V4v$@^du4??GmI(6 zXDhHiR4#G3RE^XeVNVfpNke=%6BlZ%oHJ;vv81&xE5XSBfF-UC!!HBd3+hz%Wky43?OlEPrsHA&BEjHS`G&?5X3^CX} z#&#k+q5J$9L8!g=AmSo>?VXR8?w+2ER+j?7_CB(859=D{0~V^%*^KbLi-5Y-pX=&h zcPKQ@tbS`2OT=c%>6|T-Q@^=AAJv}#HQ^U&90BpBgkyM(v*u?cQ>{ZVM)u%qqaa7( z=Pa7J7mKP#hMOG0JdXz3n}kXqM#QLQ)yI_s$xY%Lo4ATS+ZfFIe!~Kj>^;+22g#Tn z+8N1b#=w)Px+#f`i5aVOfNH;^b+fbE3x&q$pjyhz&Kq?1M^!W0wLb zGM`91#QRS-7I^R}PJ7TbITm7)$Y)mvr{{F|BFd5YR5iRm4YBJAMxn%f34z)os}*sR zJ7KjCsyV*y)kxrcN%7_L&NQWD8eOao$a{D#*T9C5>>J7Je)%AXGVXcl1@rkg$~h3P zAp7R%#|c{mn?~-EhZ0sLc$+^jWG_p}p?wg==4w6)zjsd4jBRY1ee}2&=r2sjW4I_%v1RK>k^WpXx@JeJ9D7{_Va%F5!Du=aGTVvFSK4HXT$I`I-Wc!wAmSON%oLi*6ROi!7L{N#BpX;AcJTEjK7pwkv5BCY_+&(+9 zr{^IAOt6RPbm`MS0fl9!{bVKjkDr6Dn|S7l)X&{+3Z4ZmJ$RXx=oTTB$_8FKx*WjL zOTTL0Lcvc!Fd25mutUbJt4h=_orB~!oWGe?fD`Pp{ClGm9}+MaeJeh(xWHz7t0Rrd zJ1XN7(~d^1l9uJuvDF8(r6j!%a2g?NLJ<1!!n%Cd%|g9u%emTGWTg3&s-WP z7^VcnIeb!dP#JjpCIQ5CwSwZ5_))Sg^fQ|GRPv+PPJByx!K9yi(jjq-2zDpQ!3C_x z$L{gwA~M?P2&r)fbqA%s&EY2_RhKs&qE;5cBt7y(pH4S75r zVcOV|v1u@rq_Jl*W!IZcHgWp$E;yoIOeH-?J7V7;V75u3{~Tb zzMSvm%8DNw^*7e7qO~0BP~+Ajs1!rP7CtYVI>QQ)bf3sbM`EqaSX5jV=QY+UDlgWPSF6^!=0Np>=s; zFJ{6;PDGZKk4T0ji_Vd? zMa0C&#iPu%_|xst##3{!E(QO)Qk-`55F?FCrY+^h4F8HRDMB8 z@w#^wB9Ofl9vb{qAe)9I?PS5}_Y-ivsnt`;zzuCl*h+pFkP*S!$NF#OT7-3!zm+Eq z-ROIQO>P^)>z7@RZ&Jw3+ynX>?O)a&0(D5Zkkwh`8DFdD2Yf)AG5pLZ=#owYiELI~ zWF8YvH57%?%mwoQ2j3R6=iwfRWU5C$8$N`@@{ifEkI6)aQC4Vh7#2&pV+rK`b;*|q zhsHarZ`WQ~ul0}A##@yi@1+N%ILj}_muE}o=P3vZjB|_%J`(z9xLrZ#!<^n3vNjQ? zTNy1K)Gw}ZwDhin^S4)*(_3Q<87Hemhw4ylF~gcc2a})9{a_or8t`iT@@s})g-Y&T zJZV@YmcW61_6{azCKGnIr-YpVv5le?{>+`7 zlu})@&xFgc&u(A#iqPPy`LQhaf~h{*;4z8hhv7V_-rcfqYB&mR4u4VQEn{!!N$tr|H&!ST(*{IwFXOU#&oIp@KV$E&5X6iC^#c0nm5mieg8*?_}2_9_naraJVopg2e1`TS`6z9T$Lk?+8Dg*99Tdk$ONE(@~ z%b4pW z2F5UiVBzx^5k}cU>=vuO1mO?{v3K(_VDLg~)diq~W7~ee-qtm`ZSA?RJy#XxQ^76w zHsnRd$9id-DSdF3z(Ia*GZ*sFjRgtBP2==HUmaECf&J7q|Co|Qar=yp<3ocf_WpL3$;Hv-M&0=&o~T$-R8#!o z3Ww2!zc6XZh5KBpky{I;?}`l!5QUJM8j5m`}!p78RX3AuuOsGI+$c6|S1yno$(19S-I z5BlchJ|US$@ZrB=IQ$pg1LGLRqW#i7Vj-apn}+)OTSjeb03iUG(ajBApj?(7m3F+~ z^d0cB69-|GoVgTsq(;$CHic-cO2Bu~!MbvyqU}8o>6YWUSw;kaUw1lt-y^jcQW@(J zvn&h2L-y>Ec|YPK{4iKATjp%wZ1|+-bl{R|XOHwD7l&E>fYplsc8lyQ{uTY`6t5ax zMaDA1$58HwaGT*6kps)lj?E@QbBw4Gib^FkI&yrHC2V?lKC=Z$mmegBj!_m$p;_Ce~feGM~@BI zud*2;er!4rbNO_tuft^1e;qD)n)AbpdZeLxfA*?$`YJZt%DU18dDfvoEB2#@Hp^Sg z9J$;dW;&Err-(UKAm4PC zxMpKX1_2bDJhdD1xuI&GQS4$a0=r&AEUNpRYkqFnJlUyRTI$~6@+{|*Xgr)E*Y$X8BvV-L(fOIC~JV?vctLWd=4M08NUyS)5Pu6R=U%*yR z0x7^|*Zvv+V7-2IhQP@U)>{5v7=Q_Y02^??7duM$yFuiQfy)VjUGim$j+u&FIk|71 zpAG7ijy1(+2VI8+zO>tS%Xs`sju*HKWEceHA*}0~Fu3&dQcPaE?zy<9*v-&J6N0e2 zL4RNo{25s06Koc1+u%}jl6OR=S}`2I3f8zk9DF6+th}ju{&n{(!G}he&x_^X9=>hP z6m|M>&{YhX5cX2)MEItlb>@|LFE}a@Pnx8~@>;h80QKLOaJ$RowwUO4I_+cTo3hze z8!=mes%DiRnWo77snB@klMUr1p=aAnOB-Bm>q1s6ru!?#P>JaxhF=M}6y;v=Ow-0ee|kfmh~SO5MLuqC+gR zjp7PoVc{sSMVCD9E1>V(*E_I(*DjbWZO;Im8gQgXwh-ZbyNlml0EL!e?}VvzZs)41 zSQ()n-b^ipdoIR#g&-2X(R~Gr;5Yq1R&RS3r{KZO3Sr2eIdFI6UTXV447I$F!Gq(W z3Oc*lclcsxt*XAj2K*|C%PS62c4>gDpqSwNb&zYdTPyiXamI#(1ApAu*^f0zcNr2+ zWA{=|m*18`pZDuJ{wR3i|KxfIXSvnin@0GJZ@{=O^~5#T%t)8N5*f<0u^m6863 z1^HhWEl(Ox|J^cIsnchwW~gg8f;;4(^JPn38xH= znJ}tjKJ=Ycet2Rmw+=chafS!GHVmC_60U7uV2uCDMb4Ti|C|zWu2nfMylea%(O<>& zdkg=f-zzM9%?cfs!^dmlsneV_b zhT89wrlcM+@9GyXE*S=;qKIT)N=oaa+^q4)+UEalON1zgKiZTH!M)QmpX~cT%xA;Q zTd6+XW1ZL~Hk^izG$LtPEA*Fw7KdyHjo}&2TH?Sf)eM_!dkl@3Xab>HaEg%rYPXTh z2Hm&AlG^*f5Ke#!U~6o2rNoxW%zG506W)pYFd1@>=3qxJ=nA@Pl?w=+a>1cyDdK$$ zj8}#BO8tC7HxB2f?BfDh%;YKTa=gI3-E!Lqh7}c)2KHOo_h(wlGnlu`Xm~%;FRB_BtILu3U9czLjm}%1gC_VX?<~ zk5sskVAGv@tXzXEUEVHc z%+vA^I5wQB{G1vpH&;;=Gkj$_&io-UM1G0=-cT?NC|7_uF&Z#Uj{<2T44NLdVdpms zdaQBPmP&{ zX{jmBn8kim630*A;umKmWuM#;h%u z42Ocg$W^0CmXV_W@dVHPxu$^J1O&)}DkP%AIG+G-tE`5?)_WLZindlqK$kHIDAf>V?yHgChh2md zt$j3YJpwcHyH0^8@S)2oU(t*UVVaZ8J5&Yaj^ZPo>bVVq^=92Xr+K*4kyBaf1ZBEB zt;kAZ^RXu|CQguzh^wI%zGDVFhtfYh`YzlsJ``ZS`U%)fP#da~Sdq%V3A|;z*swq# z!13EypwKL^O#vd7@!4!R@YZAkOg|<$w-mO3CXBFNq%%@%Hq0U1+E)WCP*ud*r?az4 zDcI%(k3aWvLy3Kx)b5Zew*h1`hmc~Q_XpQh0E&a5JLJ% zaAKSdna3+XZDKERnSZc|s{etR1;fS?NpJ48@nTuHH^#|ViFN(t^!3hly!Iz7hnu0k zDSLH}po1tZ{IDi#bYTIj?6>^7Ia;R8y8FFtaOaY2&3$!>vh6-baDwL=bvW2{a4@9k z$1rc{P~zFCS-5Mb$x*EH!dTC(!$WEoL3wI|?2)T_w|rCP#fwFwIK3S_>nuB`Po{Z1 zZ^seuK=-KLM-8gzL6w?E_BtbCsTfA~>j)q}E^YmfG0 z)hSoT$%v0r-JIA9d?vwJDx>oYa1uNMc(?P26HFa5mu2l!w1AVr$1%glnlE=x za`6^bhq?-ro<2s1q>-8~h4~-0NzerGq7bV4k@?S2S`RlD3Bo!a z->lU=;4;(QMW8%K(IYQ@!*O-7xn z`rmXarINF!5G}gXI^^&_k_v=iT^3ao#;igg{v&3aQfER{3`loJTno!p<_17O8~9-} zGeUxMv-?MBYRycUZlObTkRhiY zI}sjCqSPHFlhY$N!VxR$&e*&FwSQj(5FNrIx#buA;@p z%9j!%y``xYi_iiElF_pyy7|u9iyVn^1Cp4C&gCFdPy2sNZ(rM@-i*@*mGCo%~%GN7lTfT_KJaD6|+3cqRfT z^*l%G@!kvOW zIQM%O!Jr!Cv>o`I5rqg`$ijU@NBwxe-|!_h{2cXcAo{W??zXs+p#GrS>w%%l@q6ppgz1>An9f&A7Ws8a;n4BP2m(T+|Zd?d2S`;^lB%!48p zSG#?Yx75d*<`?WYu86`P*2|9GIwv8(a^R%mcCa+wiXoe9$Z~T=Sx;1KBcz%#$(6GYg@rsEzfuPIN4679W3|`3jzpV|3D!K2{s=H{z z0NfjcGt?W18WyjDO8k`44?Yfsn-qfel*h5-UQVfhSIH0q)>mN!T#I4;*_Qd;QYk z2w!vk)^~{Rl@{*3t>4nJp&rOw03ACIuXxKm%fS3X0iR8}9CG9TUvsp@AN~ZeC_g8; z%4AQrt2I4}&)WbXP*}eIP7Y@vo)V5Y(}y6X;Z14jEs@|R<-#cblv+tfXU{LXaR|C) z&s1Aw+tkT?z{%8Z^$})4{^u?Nh>b%zIr)^0j?{2m4UhQ_#o?JoW1K;S3ZvY`*hS{v zG;?n^j}+yD#*w@V)2|K<_}=6MrW$->h%alm(lKtZJ#03N*NeHwugS8+YzR{2v}e0z zaN{~Y_4(Vf!h9JiA9>3Z@BQT~2f-jYdc|gE9fA6g>z&Zpm3P6mOKJY`E=;-J31pq& z=epdI-q;H(sbrI8!p8>xP~eu>$S=tzrj^%@D+bm&}c>eXzfPFqOg1 zEJ}k-A+dY2U&cNZK?LlyyDiP>Iu6eytTT3+9~#!pdnx6{n?77ucWYcXC&irq!`fR$ zwe`RGqSS!WV#SIVcUqhTEAH;z;v`5RxVC5^xD(vn-CHQ`?k>UI1N7wgpSg2p=FGY; z?mc;twRU#a-dQ`}>^#r&v4dp$4)pn_sMo2aYDfgldgkIZJaO+u-)M?>zam5;;l&6v z87xHNUk}dS%gG(6W}&q;(+J*4%0F`M&1Mz+_pAJG*ZMy{zj)a?qr`lt*4H@rv{7o)4KT58ZJWGYa@UoXtW5+3k?D=7fj=YQmvCVbC+i`*j~hko3)=Oe zgWXay)E4BePFcx?K7S}jCN<>Uy55iw&r8P>n$(U7pkwyXc!#tQp@p{vY zBLO*$6N3jUKhDtTd#SI^fP|9xaZ!NR6*ZKt zs!CthQ@wgrZW6k$lan@X0-%8`z_w ztP*GLUZHHaWjXL(_OkT?J9Ev*y_W-rCpbmZXL_s6rl9qY)SDXviH!fUDgOVd95XcD zk+(;dW0A(M>a~n2MW8=CeD|@qJ+%zM)?Vioq@*~DQ5|!*kZVZ(hL9~O;{K+u;L6nX zX7Wk=KzWTA`>+^$sH~=N%?c>Y^A`mo>_=%@vtTPEl1VAHn*xwlD=udSSU}Ze1?dqa z`;aoDfjSTOdRs+POxCOIJ%Nk3CPv@~xKC|j(5%g#ZvxNF*6I%Wf>23ZIysy*-VpW^ zOdniQRP&TnJG`v!-dB&AT=%u#g7}!+1m_`Lh}^5w?d5w4dq${d;vB!69i63J#vzt# zxeKYVio=*Kj~oHmJrCqrr|-oPMi(Y{)o(F!^p@N*e@c0<{v6ixuE#jHqF+Ovrx2eC zDQdyL%cy24s(nfl9T0KZI##4&_G26EDBE8ewGf;S)N$IV8j!@ZU3uF%*`+>-rKF8s z-V%=m*%g5(_&bwuFWio8IO{i#Wr+_kenUnPevOG2AJN&`laQ(Zm(VM-==sollWh_e;dW0<(^e*+d&OaSR-fk-|q21t(mQ(>QHk#dko z8;S3gl=!lo7smw31vBO6mZ3uL7h_6xvf_IzcLbORdF}5P6aVCjIPR~lnDFaPAozEm z4x2`64&kRg8TWf8yk}<&|JaziBX!)He^JOBau-PDaI53uzF9jj%Bj}IhIa8O{BHsK zyUPEtUwmkA}6xOB`zoE^xE zW|b6JYSyJQm2UI&pYBGIAP_CJPwH#UXGd2AgMei3(EgSl`@J4WY?uMqbCA&BpuxUl z2yk_mRNnuqwaMjJD+^S!SvC;Sj3AP}Ge{qjPc|5R%^2#wUDfmr+OO`6S__c9D$Eh; z$UGGx?gLrJb3JU0}rfTVjxks%6!2y8ZordX8^|pqq@WXPs{l zV6eZ^Cu86cA7mrqyMSFs;pSwy=;T0T8Gd1>;L`_08k`S{kFPE`C#mMh`KK1aQ`~fI zZ^&IPDyo#FB`6v8$jLHJA`Ho+n%!~Tl4dy#x0M12dK_ktd zJ_DqGnOg;HVziIx%MtrtadEn!7G-Xqb+=hDUkd(;EThu{TFJHYUU;gPynaK7<-3FW zZ`%_oKpyh$L}1Dz&WGtM-l;vj1il5v8lkKHUDS~$x^NSgRA_mXO9rc|Oos2RBuCzY zJ_yvK#Dbcm-{6h{8Qvm@M_c!HqkX#O1=rSVw#hY3x*$M!j zeOubQ0=Cvie2Av;*+%4Q<#2yu#MPN;!VcHdH7BW5Y7IoH-g!eIvn2Wr?&F0v=Ls(ai`-Pe}2y=isbLuo)ELibHUZLEi+ zB}1vY)e79;xe5}u`T~&DAA`NZ{M&FaEp6f5tfGNn+#-EdpCjX*Pwyj#+YaY%B=0rX z(uavI7OZ$6P_$ciNoM;~@sUJua?CxmU_QD9e|Wa}e4?`QV~!5HBknua1HDkIlMx_& zaM=O~yMMAEq}$8t5pCM_HU{(gK7qYgIEFE4yUzFV3e~v|I0g1sVgjx8~%yVqq)!7FDgKYmCe1W(==!*`p{V;Bu zOM=r+e)FH;jUyt!3L8qRYu`#$>(bTH0snzPm`OUYj0n6uH&qkyDO+P&BF9_6<}_8|!~`cVs8$n|Qh00Js-YCil(`?{fmE zf3*L+g*e0E4s3g-{B6Cy+O1oUFLcO8)&)CD$@jtdPTS7Ae&;XBkGVgy#&FoUUh#Mu zpXw5`BZcicE`StZTum>D2wH5FwqCL2MxpzL%eFLXm{F-Io+LN0FH+@UQ&s} zmC}PMEz9kAWF;rV?s>0Sp;CQmlajn|ka1glJZav;=k!W$efn^@i4x1#4)WiM(qlB7 zqzvB+lwsoOAR}X+o?WMZdd7hA>(I!G?bsr@ACL)}^C)`ne8;jP<Yamh5 zC+DWg(K*T$3_MCB@tsLlFz=6Y&`TO4JZXvK0rr42;mhDLN4CuSOIvgY@7yP6;1wR? z_g!&gqF?L4gD$E1G_8#~K$b%P+&`PC_Rb2vNlaVM@at?h~P8T>skG>~7XzC;@5*ka3*aIeaUfbC$Xao_@GQIS*xUI{;D!w2!c1OT|M;qX{D>#r{y+QP|nX8v6uq{NXdw z84U-2I4?bpUgjv>N7Kjfz}cb><)Z??XIF5X$nmVo1ml5v;S8jug+P1aM9Gg>rTw)| zjMD+s=?`Xc*>}QZ9BZFZM+tVB8tk?Qcqx_hrH$V@?Jcb>gVgv?9bDNv!Yz_V+R=H_ zo%}qIUO2+x-gl#=CRgW`<>t|P=p*%V)QVMv^x==Bn-ewMHkh6oB*IxmKy68h(Mm^) z7PBdl@4hHbpjx&f(W!WzfYa0JR-zvhbfh|}7onBQI)gg>2a`>|nrod%I4W*or+3Y_rhC5U$tvdebOD_sdh(x#$o8sz7GE4r zjZyzuWUdA=sxdkijPiXKJlb2(j*Kh7qIgf@bZB415~`}`xKY&?PjgY^Nk^+{7e3d- z%(`}uE~dfW{=V9PLN~fa=QLBV*~DYZfQfOI0D}=|@nky@(nuf3hb5k>M&j)cSH`Nf z)?VbIU(^*G(pUIV7b727Jy6C*n_C*&UoT%?HML-$_9Bu^Y{IXhaLVCKw_++sj>nZL zw}xnN%qQ`?;1t>iZ;rXo2H#?k&i^R?dy>%VuSim&>af#?!P7i|3apuss1(G z`Ns(K{{*A{he1z3gYjK+@8RG=IAQ$#^5}w5R}>0*ay#;XSo91NWWl3k0&Z{2=P=Sb z5uP6$5efS~xkWh|@c~NJVW7QtczQ{=HX6!S%CZ$<8U{Uj_gtT5j(N4@c=8m-Y6+d6 zUqs_dW8g^-m4Htko|(D1f6$#Krncx5lLL|y5!e{L2YPs8P72rT&OT-3bJxVw8(vsr z)2V2oYfPE)mVDR?054}_p+hdX-8}t>YW3e3t2Y&tSVJ6|vQs*dl1zC&!=offky@QJ z_*LBGsPJ(oTWjL_mSILp&NdU>-lYZILNgTwCyikznY$p(Cvu$9FH3(W?G_}?HB=@G_J)2WH3{=}{6zQ(XRd*dM@#ld<@C%5IxbiF@3 zG^8*^Fw17n$5iE6YfmUfsoN)x_{}j^KQg0PW*g4{kFn!nVs~(p?Z2T-5LB#B1{KZZ zsRcN$t(JtgHOBY%f`X0qa%p^5xF#j5Z$vTk6h!yhG z@2YGeaA%ss)&@3oB!#LfOxkF6iLvX}QTHEQ`k)+MmG4c|yE^7RF>i;pvYyI1*z8p{ z<`=TSs+M&&TG02VLSyLW3njQO5I@k=zbK=m|I%lj|7O~!gD{+a9-XXKmZRY$!Mli= z640MxRCE7KJ8Q4tin)YP2sRS2EL079J@lbLX4>Y88DnEI%+n!ExF-XD)*$IxyFR|~ zPS1ewn|pt@dPY)4<-22(^Yi;bJ*dx<97{}ku31esTVpoJ{^w43UYa(KKDvu9x1h0! zD5sHtSGYjjEkQnrI9T(IyK89(iL10>i<9gcUMnFeo{UaW@|!JuYS^VrFO0pg6Yro$ zsLA-peOcd+Iy2aq-CvYL11ZXN#+~V&+C~*@TAlHLXOo0e>)E zNicwCK@{xU8oS+niVXFty!hvbv5}ppoQmA~?+^Z`uYHob`MKbQ&VWc{rkhUZKn{J( zDAroDySO0+SRv>Hy8?f-cUQRYg{mn;hc2=fiYJjI^}a2HfvWP@cJ$|GE)*?_ z=?~5Ot}J#P=(F#~PFjwJUm`|CSV8E{igAxp!mN9qeD~QDDs{Tr-dG5l8z~-ZMi)-S z5m$YJL~1NDJkzEeW08lI+u~)LZwPAL>fn52@?aft--3|q@Is*ryIFd<|JI139nwLt z$H&r#%Q@81DTn_eEEX~AzFC)mbv)W%dTL#23bQ^trq`G5JIbrcvmhDhI!fiCnzbDW zslBFnf6n^2`2!MYGvdx)Fn09{G$3hauzU9<#ThvV`Mg<`Ai$?!AvGSIN{*)lEct?X zu?zv{KZ)O*)7*35)Yd6YMiug?@0RN5o=cYoAscd*lN($pNkRkS8nVv6ZOLzap-k*k zlc2wMNdPW1oQh*M=e;jyD)qAYlFa2BgCVyv=BQbRW~lygee=?mTA5nBUE_cZen@+Y zr?3}$OlCjeCy=Z%p~K=vXnQ*>+%qq`7SviKD_^PPup4**Oa51&v%KfREMwb4!j!| ztrN1jjkGIV{)87+yQ{jLoptW^J=vRQjuT+wt}P^%TAjbJMZr^LmYwLf@h8fmm5RX4L4`~P?6b9 zXJ6oT?@!+D{ljg1mq(hz*A;pUOZ4MPzLkV>^II&%JgBli7Y!9gC`>*?B$L;k?=`6F z0=8~nyiu#Kt;-@UuA_UOf4Tz9OKz!*t%7nw!%&2F z17KS1=nj{&84&{?0|8G(n`payev`Wn$nSJRLi$>wCmk!G%wTtFbtv5!d-$C5+F}I1 z6t0FB*N~gh(`&LFgv>Ckw{HeS0!^$_UpkJgHcF;w-GWrjBWo0rJZz*xrbj}M9kI1B z+)4QT8Q@Q}SminmNp%8D&2`QWxi%KIDuV~uq!QUtg?f(52#oVNAK{<%r zICx7P$$$-e@!ycym!?Uw`Hs@u1_+`nf~{(`L9b%|pNs{KJJ{mW35d1q@-Ir%{p2mGID*;#JWIM# zw^X02S-1F*3?~49J#eN9=e5Hfy!!5rr_jl<9Z5vL{DeeSRLoqC!z@$z zITfc~&JXe}%|tv+8goJZq6l!P-81pVQ4TXZnjwBOn6Er2F5LXwbIK3owj-1K=#(3G z%x2o6dG&yaI&Nc{16I3G)TwA)gux?<+|p7z&6|cR$_OXs_Gw1x;ikAp>~S$lgfvFK zsx4PCi zy<^VZj@!218bH#-Mnc~F9QE4ZW9G%4E;ZvL^bsP_hR{*}ELuN%VHUqZnPt#NKy zOSr{LX_(!ux~PZ3^lga5JBAXaJ+oQ;0&R&IWxY&pL+o)KVkL!=_%Z5$@ftmKCON;QQY$NAdf|wdjGo=gJ zIk&FL+EioPLhIP=-Y=D#qQ1r^yOpxm@!1u?7DD|@n1ef<$u8y!3pd!0_!DytegY*a z9x*K$Af)`eo*BT3Lo9M^2wd~=$1jqNp>kaoMEd3@E?g(FKd}IenXuw{F@Qu!kp`#P z53Ic+bnmV-kr!PGE_QTHB@hv(kLG;^M{I`TzZz?>t!5OqP5OHD3S)(9q5^E!sD`MJ zoKjN!|8^35t<~*(kDo2XbnZkF&B2zE6X!O?fhnBS zv{N2d#sT>CZw~~b)-Q!VZK%HY@1RjHPvI&X0$jQ$Jy)WHwNsvCcS9bPxDXgGF290K zUa@%X)=y)Ss&s3NMZPYLq%p*G>V=C3BX+Bhff};Jk0H+(=Glo_2GfbD=PPDr{9>i!X zuF;EK$$HBvB_W#Znd*SW*me3uf??tGhzWD=yq;S6EVg-JMTl}kS8eRYi@5y^_N00} z^)JVOoeg_ZN-LP1~NddoojcNzC(CYJ{QbWq^W0@djoQc z*{1tvD%yI1_2GpBJ9cX#6M!`}Rvwgvp=OJ{hS}yN_qy@;)y_2XUT=y-dMO+VIT|uT zfJpw_o8bo{%G--@tcj-_8tGcfVwq+B|Cm{NHso+}^BOF2=rt6*pA3it^i3VhI0Hoc zM|W^G-}ktu=%3a`Cn)m1jyjHLI1x-V>Ht%cdRFN34HX-4KcG%f24!BXdfOR;V8H943q>@F828D#fhqhYZ6$}cccnOzQ`7>lu47&b zuVX2W`Uq%>M2U?vpoEgbgC?4la)Fw)pun7)9mMm$C{>7iz8gUcxGA}H*rRSAiQT)e z+wS-QwzBoR`mveq!#lt3v2!3pQ~X!#^eDL5$zAnAs^-@0sWcvD{*vZfIyOWLeAF*z zn(kl%#C(4U(nknETpx)ki`b{1aiHk}{5_zAWeJK-i>*3vs*#{J(ombLlke;(%% zGs;EmxENm!M{5>T6`c`94wdY0Bghc)ojO@xOjP4@%b!~v7^V9JOR&-QWlx~1V`P=QZGn0 zrz7O>2C1V|=)ObpuIIpoS}=Q)R+ujQufyFiL?W9 z{L|wHohyS2tu8HiS^N9VoUfgx^o<3d&`j}TFvl0=qD^?Ica&R2AVe+HWb|I8<%Rm0 z)!0c2lrhzA+n6mb0V0A>yl%wri&1(h<#>LNpn}}g^Fogw-d>Sk8yqOaV<#YjFMetb z+7cWmJCMJzGF#R@2MKGwd(aNatC}gtG{Js6Xa z&2$g_+J^_gaGIh{yFEw^b=c<O#G0G=8p0Rm~os7c})Z7G{TJhTJ z=Ec(FtCPE=MeRS`eQc^b)$4N2ZaHvvMl%^m#+Xc$Reb{K3@3GpyqpSXBe@xg2AvF< zb!QflehRN6Vs4e}B!#pVM49EdJbg!4N(fg4d3g~?A458f)}c@L0Bnah+h_9V&)XjU z)2oz}uT&AKC$o(ve^J^)%k#7u)|~&xgul;#yWyA1V}k~z0tkQ$J;GspH7TYqdZec|h5FZRC%i&9b?|_}ekp;g3v(IqvZN;4Q z=0)vO@0Q^QlD&14b%~~VpZ>5mUFn6CDtpVUEnV{0A-lWVH0t!LUQnEf2yYAsz<+kl zc-^alUuN$Lx8&&Muu$J%1UZj19>w=o5&wC zuj{KnBmvjpaO&~9pSJkFEu3Gf>-J!wi;+{nYcBm=eDS-3BHo6Z!!HjoPi868z(VRiJq>s~HIa>I@%V(WJ2S zN#Z{N;jL3|Up9NwGrd!K$RU>{$B8{I35i4!E|0TE?B4 z(t1GHkbdn^f@v8`gyjxB$z3QJtd3tYY@2;xWj-eT7bWepPvJEKQmGefS#w1!tIY1yiW4%`JJ#PQTMXm_x4F>GU#G_TgASy*w0%;3QRgFm;_|u6t3k?wfTW zQ`*^{TOk3}eSaLCMJ#M}R<#h78s9&nXsazMhFRz&G@7P;jfNH6bW!~DYYl_9iz(){ z#T$;uRMm>DWCguGkVjff3`lY|N|jLQXEv>fPkv3Tan5@mwK8wZ#6?7+?mldBdNkVI z5;k7Vt}huq5%-Wx{=qlVOWE`US!@0u#SqCrCabxxDb-;bgv$m5DrWXC1M_UJ>DDdw z*GM4+8?2#ht&6L>Do)G_nX`#v%gZFu`kqLK??WRGi>!zo3QqAj=^h>dHK5aW72oTI zF-U4U$^yf`myG{jc%f7fMvMTm`6bz>WoA-#HThK3q7+qpplTnMH*>KLbom6wxi8;5 z|Br$T+u>Ha3^Q?9mi5S7lS)?0j%HDgleUN#lNP;!OKfR4AI|al)2W;dtQw}L;}CIG)!8R=WrLFRxMK8EJ2w?YCIbv7!`{xl{~_;j0{aV_UN(-cP( z+jSfY@h(;!Ud0XvSsE#RKJC^2qfBC#V8=!;e)yiXeVhkZ#J%FzrH=(8crdhP|4n9S zjp>CpGfmkK-ltfd8UwYb=l_`Uw-5tMliDz`Gjy_qgT z3@08nGI)FOkw-$4&LC~7hG;^u9*w@c8YYyHNyPli^z-nyn(Y;r)lc!Ekx9`OIr$@= z)Yj!87aMOQYxE0?;4IzpOAx=jDxOhJIux$ zDSf4WL@WJrY=y5l_hKpI-KYoFwmK28pvh={F#p6z4;<7lDOR&(q8H4F=RI~~ zRkYr`@w3gcrf@Z7l2YQ@h*ieqZMp}3<6CeEe$(Qr#t8x^P;lCm!-mQN*9xiwEbSAN zcD3Vk)SOfY*xR-3#3++@(JWdA9x*%Ti4>RgjK0bgM|U2X=8Xdo)K;+&v6XP{yWk7a z&nJ@Smd=H8Nd=_9xzk=Mb|6H z`scOun6qAMk`gfb&bXqn}}9aol|GD%3jp{NC*-{Mt!{C zNL-KyUVqHnW74WflEVr9QG|Pl;EGN1TL}Y;jc(-y8LViwsaWeWD9@K)p%%9Oq~0qO zgJ6jtoSD)rH7vV$)I(VeO>B)O-D2dO+XW|%FS}LYadX1Yt^iYWi8M zqa+baTU!h0$#{s)w|HhYYJA|T$SiIlV0>BaS8+D^P4m|JRqj;>s<9(Rv0*vlKM8IZ zm&pWKiKdE6yG*YoR@o?)$FZ7-Gc4A{Jf696HBp&YD9K@~sc|nBdK9O`>Au(nAgHj1 zfSoBij9W%a^c7^d0D0Eq@X@xtZ}AE&u0AL(Gf4wZdS}pK>eUg>2)l`x18Pvv9-e|)MS&;P^~tBj|j#h zmFfKHXK`p^`8Bs^jswDZAidLLs`a+07`^^{+M*E|zpUH(_~&2_a0Ebx2$i z{gruwu2P>O$t6Q3tXq!U6VTbA*S7+Cm(;8!rMAe`igRMV>)M+lCY?ZDW2X(}qg5^s zMIeQ>c{8|lS;z>^UZLVk`u5(@^Q-S4-aeehOlx<2fQ~q&sHon&n=>lZGx10X zkE7lfbPC;4{}TEKp5O7wVm@hIFVU!L=XqfB^?7|QPdBaM2C79UuFUp!lv0Ywci)xfByKs7ZBavZze?;`I=b-ze+E>mTxJrXCHcjnOc;+F z5KVE$m(-iH3}2-2N`3lv5r}{zo3ne9&zX(9xmJiDKJ*$T#7%5#S?fi#?|z3s$0n=8 zbfag4pPtYbzIan;;{@d1Vp%Z4c{eii=-CXmb>+sw3ISUs#UFFV-~FuX)x(Mwx2}~m z0doKz9ZgW0lR1X*>76(`q?R$9`7ljD*9L{*vt8BPD#1W7wQsc~x}*16e9((Tu+_$d zE;A^_D@`zEH$g&n^fR?A5ozes#fHC$>pK&>Q_Elf88Ygzdo@~cq+segjowxuSBnSeO=mDuj|`lQ)`Y_cqUxj`8=D zD)V5_^rVK&kJ&9VWa>?10kN>+!2s68eB*5R9&^#vm#lGvbHyU(W%{LiM0Xq22#p8U zIRK1AIW<7s{Y(aRaa5!djnKC&FrGi&&sg~oj3=FeZ%)pe! zFgEix0)p0b(#>1ES97@;3wN&>qB&|pYE+$itF8XsvVQVZT0vTW5u?Bf{&g7 zxB)>{G$80}TA3u`pn^E_vHidd>ct*@eW{Ht{UIfs0)5pSCGEKMue7o9#6A3V^G8&A z!u=q8kp9Z5e)~I}!OiA7%erfVuY#XuJ7&7}*0(X^wn9(1q6dz!rV9JA0Qk*k-r+GL zcVa^8SI2IaODC^Pe0C;=w+*20f|9^mHK2Zq5MIzhbfdm5deevxycQn9o;e)PVam27 zKj6dc0Qh;;7x31m)bU7bTe>dH%^;623AR^5*Fhuc zDYo`(k2gWLrq1kuLoNzDXa@(T?FZ|m`swGm<RX-65KYCvr~c{ENn+0Y$O4{8L*VLx+hmcEh$L{pX`@4)>^QRlUdn zJGux2{yz}XAEh){{p6-R#2Fsp-Y^XB;^%NN9UPw|2p=X=a!Nl?4)xx<>Ss1QkXKj* zyRyk>sZ?DNaPcSnegJQ|46iVhApZ^pTlpk??$C(E9!F*h?xHzB&{6&!%N^bNP~Uq}JKvPZ)9%UQ4$!$#z50p?%_~IthYqSq z#F3;;>HtsG0K$>YZWrhX<^FvsB&bFztecZuF?E%y1^l>SZTm_ZCO5EWz=L0uJQXWH z*sE6B=Mr7DSLmf&qffC5xjH*|)AiqNSD-fj$b6$R{m#V7mr^-iX(2(C650&ERer}M&h{sC-oBS>VUFT9;qpzx+*w*8+@&a zwiTHjyd{%w9ygwu0_&X?z#XW_{4-w+iQCCO?Z-%|GbC=<(l;6(3`9H|@##rM6`Dw- zOWo?$l^`n^oe`Y#^m}4vJbdTYY%{eae7o1+RGWQlkhG6ykzSM0^BO6d?vXkVOq+#YY}Lp{b+SfU#Dkx+?ju* z*)8(x*n;NcyZVGRSuQT~SxK&AR;8WH-kd#Wk}f3lW6}j@~^eI=^5ntDurZ6B@ zXxJ*Y8RzR=Vq8HHQ;&m6*euA+(MlH%6TCE~A1TvBpKRO|ZhPFm7oIfeSD&;aLYR+E zk4}7zC2T#ru#MKVg|WXL6|QQH3~7)FcT`r7YEkOdWo!3E0!#8n3LWXC@5z4?9m3Fq z=KMWG@RWXAwT4gamMbRTJ32qcKg(Z4`-}3c*Mk>WT(naauWOFrPk2|lmQ_{Prb@?0 zr=awa?$hpxAz8dZwx=YL-Lyo#$cYu)=RSCITcF}Ygft5EApC~L5^4O7S4$-7BJt!* zDf59n@#&J13jT3*BTe+y@Z;mI?k|-8dwV|!d;(LOB9)nR0yOt0kEKzE@B^#-FDO9IJC z8w5Ji!KZ(-xL5G<(gl{LFO%mj?^u8~ArJFzC(r%iUG~o9Nj*_K|Mv3153cyu2+v|Y z`wu+RS1Kvep-Z%VEZU^ptZb6imPo&X_zpHAp!=h4=uk97{wQfkma&QA zQnMO1sY*^7@FA}hu7sQS?Lzoi$2cSyTqqZiWo;aGN8iNVe0v@3!+*BHd-o5wy8$uy zKumdig_=zX1c}J{(VJtIi6z@d1f1d7L(<0URV%II0 zZIpjvwOeYlAmDi-@(#TA50*fBN%H2l5SbzW0=W)x7yhDv3}os}ucquS(NBy-U17Ji zL#D-~KS%})dbhBq=9qHE!y-znA88oW9Iu3(3&nWGE2_b`=j>tQ{aOk=iV`=I0Ku$X zjd+`Qy(TTUZOPFb(GlPu_i9rIOJ|m(*#hm2x^6d$w2$BLbBgMs%_REHI(LieFC;GC zXvRh0tR$J~tGbGs)6+2E3n&?#-cQ<>!d_R5VOZ=n5tro#FA#N63OY;_X;3wE%yHk3 zKy3>r&RuZ_cW3!;Dqi5}%%`x4#UceL?&H_O6N8p1w8N#QKFLLzWc;~Hz^!6j(<6Kf zZ<4#mJM8tJMkDe}(#o*Rt%ofBY)Z?$OS?DHzXy{@VotPj>!xF!S(rH;o^t{y>@Sf9 zgIBlDMc3a{PWi|?&ba}lQj9d^qGyKPwFMs0##g`$B02$cM+TRO+g-hE$HFItM2#lC z)dfi_P+jRpEwpty)*8Uxa~7l~D*&p^t7J3NmH;Zz0+^3l$1%Ng!owfr7ES7gr^l(Z z7JUCWIX~SXsFP;u@!3QJo3-3|l|!Cq^+^(r&zUrC?OE zF@K9bT-1NE+yi#7`q0(4_a89-(H_sPc5`{Zbry-JTzPc3+rBUs6A?GYAH3wid~gc7 zIfxvUOLOX#pU1x8I(rCN87h)nQ4QvmBM^$UaMft(3yc#q*q~h`+Xv5=t;QyOx_49! zWH2D_Ps!hVe|3ZI%T0-2uF>g))V-QKgg3*+=-YP$o%L;1ew$M2#);hh=8E82u{kU~ zHpwGRM$R`Re-8Sh$E%v8;`;R%PP4RUZOUc|opJ9JFkRMLGD*!+F91dx#TFx_``us1-j-e{vry$+SWK#UJ6ujMEekxJ@$m)?otOE)BGB=awi}6`JSzqKL)3N3McCdx1!&dYi*2C-pj{=HK{^M z{n+X_sk{WP?XnkzN+t*ddMW>f4FKT#S7Py!sKI zERNAi^@1c5tFsLDd=U+8li5zA6kiUxDceNn*Zhk9Q>sob#f*v`cUu4~CClWW{SD9mFYeA#tWiGW&{(m6l*fyWxP#ayd;f4r z9O8QHOBsod^Bi3=?!xB^X3l%&KRz^vHb zeIx|b8dA{W?a3~t>4P2a)s53+gAdbUJ1D^NzARLttwk8C*Tk?bkaOJ){N!bH9O)zR z*Aj-G8qvqwMNNPDpfi1p$-srj_j^VM!^%_Kq1h}ewuHf z8NhFGWMAXY5-or!7>Tf$w)mR7GtS0#9dLv!oQ9zFS|QIGmRk2bBt{hy?dF^w8^tfN zia@@H{F7(xme+{;gY-sE`0`9lgbFA$1`A&@;~N7M z&rar6w1h0vaChFX)kqCZcRViK(c8N5^8+%rkSt@FXlbVI())J2igcGXzp0;Hz8mH( zZe9*nQZExMZ3yf)O~7Um1n`$thdWbn6@N%HrTkDe8|fo?eYAq}FpP$4+ZOrlxH^*B zMk8vzT)gmy;F!OSdup53O`z$=W2=GZx>!)js94pZZBNplH8;C>Hyv&)k!TU>d1bZ& zE-WR1IC|#cl!P^Ez+{Q71vW%KAef?s=Gfn{)||~nSniljvWtJj2+#1_Dg~<8tf2st zYRs_AsXYIPn`MLFA@lN#4`}06=eSd^Sa$X*Tj_2gB%dz*jbsT-fH&bJ&&ebX0uY=E z_h7ENLtXURTC}<8wU{-?R!OEUr2tx@eOkzMa%p)71Y*yZLN1U=P?)F7jd=Abxl2g7rk&9ch*NY#fA z?q~q^&K*$Q!FYYbyz`h9dh)2G&w*NF<@25Bmtzp5F|-ue79m1?D6lnWA8VcCrSzE* zou^GTsXSz6DLb;xE4=xQQ=y|Wa6yLB_zTTS*PJIMbhceUJ#OA*nj4x;{shtx$;{o^k7R1(~L2E09Xf|N{0l#am)b~ zR#2BzTDyR-FPE3wVFi|LYy(ECvqc$myw*F3&(V9@P~75hC;PI{+q%u!+6hIHS7j)# zEXB?&Y~}6~ZNpO<(w=#Y$HblhJ%kt7AZKg4KlA!M0eC0aO{MO*t|h$S-&6Z7l`j2E zsOD5zIBY$D$vELv(Q72$?O%Cv^Mq2jhEjy*?ms+$-_2d~wnJZ9ZIt(s#Kto=1P{8Q zF9yW!YZ0!!L5$`K17fXHMGkrZHP@3y|+w;dO|-}4PZ3^sZnaA|xrw=SfU z*pm(ji_Oc7#8Lh8zLF)>n@8}x?yJI+NWzo;GNYj6{TBtr-BGuDh>PSMuS}CxlJlpt z#W-?_9KBw!MF_7aPkn0F!HiEm1xkG;O4*-b&5h0jlDG5UialQRUq4c^sghSNI2z`U znpPM)ABs%kM+P+n^nM})`-8+{jyCZcW8T&8|l&ZM86I8 z*wC}5f$2;_{E6QJXB_S_okP8~w`aVY(r%r52l=8z;q^hrk~lgOCyDjp4H5ZzHX zJ$3Qk9V8fOX2KJ-&#w+Ks`gd>CQe-2e3_2VR+GC6(Aa)Iarm<)@26zc0`TN2Y7<*& zL4V+ev)V~m(Cn@P+A^Vg1kSQ?1~=C|j86{5ZXgKxIS)_2DKD{BT#}@h(YZTQev+0D z6roYJ4Uk5cE=!(JMb_Pi^7Xd&Cy}0 zzI{=?u>Oubq5M%_qaCDT;0wRNtxmdLNu;vsm46_9L(Q9J+9s@IJVl;F;p&f1Hs2zh zL>z4uM?59-a?4E`V4hm~{#v}~ z93h!et%#-CF-jkCDjrSG&RJF;pelCMU-5ZW8=c^_f+ZRQ1n10tZYEd}OJ9DtC+*b| zQ_cfElJzn@`l?4m-ls;-O}3D@7eh2Qo;-t&+LJ3bqkeV>Z~6O;W>8iQq;2}c?o@Ua zBiuCx9>#8eLdScb#;m25-P8p;8oe3YPJZJr(UeE=d91cfll0RW)JU%;jXd@QovO7; z>whu#7GQC8+qNJHArMHA;2xa9-QBGy+$F)SAb5ZP2@>4h0|gXLaEIXTQn%Oo1?XSMtQma<&z1FI^_nLFeF)p-J3qViP;QVHKnGqg2OzIYN9F)bdH1Pt~QBh z@d2uBPvh(iZWUM}vV)cVd3me}6Lz3$>VS$U!543{25ghuEgjd80>e#7gk>+0Q&g3* zkwC3JTZUm;@C>m*#~TjJugVKf8BP2~lZ9o^SQ(1R`*S^hH(W_I zEN^6UvLG|tUn*%qr~K=I+x_4=l&iwVeSIH4pCS11Md z=4?KVW|dc*d}A$mXF7!gY;RhmoVg#A-j>-U3Jj}(7Ut2Il5@8|$!)#yQC8A0cnwlGTXrgs*2&URpyZbN05$u7 z==_nJ#+OCY+np1TcC4MOH$PQb8Q&!h0;*TUGVA3p?MFA4O5=A+*y_G(KmkSyCLf6; z_dqVZ-Ps%58uyJ-Q#U0S1g=FAm>BQh>lVG;>*tlCohzyK=r7rH$-aBGNV!2EhG%E} z`$e3$j+bM`Z5s-L*xM*XR0d<&QR`&=qAA&=ecs*?y+_9WwQ={2=)-N9LZCt>D{EA| z4TPJFCsAiNYpB(n$bh)jj6&G7)6qZZLplk%6 z2=!ZVx{yuGIkG5~QhKE$*b?O1N$GROuK1-pv`V+N;KTWnpm=J8t%_7O*&be{+)p1> z{JI(E^qrHt9kSF%hWg2K0B>_uQaDSgOMlLd_~h|%Ng<<>10etWv;sCct7BKSWHXP< zI@L}Ejxc4AaqQXzVHK!~H$@;XibQGP-m{!&PqoNvWfDahC-rFuxHQu7FFCpA@{hja4XNuT=fhhRdLz;yIB-9 z_+Oxd5}x%$DoLJ+Ot2N!R7+jen$@hgKJ5TZ-<>&#_{Yt|wD8>{HLP3NztZ%E8Qfm1 zsy9{yM9F3`Q@2mzEAXit#F!~(4(-69U?0!3K9j3^7&c(;PfV&uSCVG%GQ%iQIQ30( zJa6vqg_F2U#Ln3z8;5ONrvEY z?x-89(lwKS%tY<4j8TnxfUdqN%+2OUGQ2*xb2^Q)+xPwyPw^Y)0-3Er2fSvqksqHN zjl+3rMXH`(L_ZHW-YMj)Ws0fH#R!`+)*S=b|4ulljY#3PH2;PEu2Ex0LN)C zGb!(`K5bW6!u=Cw)*%PXAcL?Nr4M)b$ojFh0uS)0$1ezOt638XZ&CLlh57IYh)*42KDBdiiU7U#=XNnGF*j0@j=Gp9SUZy9! zAet`QPzPxg_F{#Ff|z$;z2C_vcp~#Ig%%uT9jTX{n=gjoNvjlU+mFDRV?G1~yl;OS zWQ<{L3^?G+q~RG#_!Ri!Uh}W)fvNg>9eek}9^X%53OurbH)K5y*YM1e3w(b-GDgJ$ z7A3a)l*&G~uX96G=ICK$7N9ri`}ilmjS0sWSDMV3)Dz(R?dKh55+1tF7Q8KvqhldF zkbQC)`FQvy)<+j?EhTK>42EbYoq!6uPT`OX?m9dh-E&dgR)?mh<{K7L8@&R*ug`_}qLU^Pl!T`Ul!A-evmZ=&EhF%^xx5%t`&r zQSH!d1zOKCq)bAQX0H)_?&g6bC~xw*;>^DoOW$xhW{W7X7XZxAb@M^u|)k;*^S;*8#bGg(0dHFJL+O_$} zvX3&CCSAcvxC^Cd0hoNKkpPq^t@?zwuWpS#JbuXryWsEf*Bb-Z2)qx^fdEW7i~*)a zhi1MiaIqg7*+s>uQ%A=BWO@Y;6 zpEd_}z$_I&5<51>`C95yF0L83it1)7Q?0~vNo_IzKCKAj0Uv!w;z4w!4}{o>zwkYC zS`^B8^SNlMNT!CHS4YLF9J7)!Mp;=oIPRCHIAZ;D(fov`bjNM+=IGu=n)hz7tV*G> z$hDfHBX0^#cq>U0Ox@}m0+(u>E&qV@RoU)fjE#1)KD{l0d#GEV9DmMOORnLqM)9P9Q@F@w&1M9Qx|tS0 zqVf+|JaeA=8FZa=eyLDd(^+r26aVr$9$H>I^UL|*Jig(o$iCtB{_Ot#IpQerR#8Ao zMzVB<9890{fMNucCj22qft@#j+f$>+w_0b^&G*t|W>3J|62xAJ`UfEj^Uq!4RY_0y z67Np%5!SW{HxSBq)j#*<-Dc!9wQx#7LTi$l*+SOWp|p~5zT$$Np#U33)ch5`Sgtn> z3{g}gDQ5!SF;b(WW7Ny&`;jF%tgN`~_Z*s_`33AEdM7F98kq-sE zC3_U0{4Us{3Bur%G5xMjJwK7EcY_O_?{c-*p6QFc#j38^P+faMNV(UE=Niwr(|aLI z((vx=K3~0&#Hy4J&wc$+Pp@Ohg&UQFr7c|O;0%4X(>>q zvdb}kV_Ky5mS}FDvn8!$9YUrc=xFV5obrVge+C&(Mm+gHcZ z;*azPX9F0sud+s{iIrvMq{LHFtK4&^4fUR#)7YGuP}eSTjn?}T2x7Thq$mW$VOWna31hm!~P zh1#Yx!e6Q11lXiZW1I&DqeXfojU+-Z&37cf9Y70FEDX)K#&X6Y$D# z&(p?>1fb^#V5_*1sG*EcbWyJ_aHo|yF@?cHTZ@Fb-E$G;bbuc=ttp{FL6Wg2<-1pb zx+Cn$eCub4Re_Ni9+zxa?_?T#Q+pXB&F4CU(69BF(m%Uhk<|aX!K-U<*`mTwjE;>A zmaZv-o`~+C(4_x1+BXek=-1#fM^=-U>{mZg-V3+wzP28~Oz+GZ13c-J;IqKyQYpBU zpKHBUIFrNfG!9_``8$|E)56P-fSNc1mrZw054>N5uSxr5_~!d;Ka!*FOkp zB|bwf1>@GnrHfB_hj4=!lD3L~fUzyqe*W0G7sdAV4+2}!w(2E!@0q%>MBd3+KI2Nd zkLBm1s7d^8WpC7^te)g@AIG~4xFgC}-PiV9;Ps%8KAaDq9X|LjurCy--VN}pS&{6_ zX33hK^u|`o$^+LHB3#ZwgH%()^1X?01Vj*S3Atm&OG4e0Eh-q6LOpnYIYtbn=Y_AX zW)aHy*b_X(Jj#VKIMJ$s-$K7|Jh(3dmcmDXYQx}sx&6Al?1EP{)afo73B!#;B z`wg$uH;v0Q=Vot4VNjIY&5g9mK`2@0uHu>x{VjDP;-hO5mp9pJfQ7xGPH9wf0L0TcSJe?QyKwx_o`>4_Yj9R4X4~r?Lx;;BUyL}TwtxoR z=^5M&2?qPTs@S(agpV)Ik7uporde$EZxVJVpmuR8VH-TTnY1kgI-__}yEEE%jd|2X zbb}4fLds}fjvmE~5QEN%U%?{Mo6h-~tTI>mOTu5De8N8x>J56!Ya55A1aQ4$R2`VK z)W7C_C{6Y3Ux5<>^h|MT>ko|Tc!kNBKij7pL3blV+`}(vkG8k(kwv5CZl0~U^<#!= zj0B65-{DBUp{>H)(NAo1F6G)7k#IN={XyeJUd!#uD0n2Zrd}PUssiHg!6)1*!-(zR z!TMZEH0iHu4-j&-Jr~uu2>wh8(+{s}x~ zA&N4%=0coK8n<0pQTlIX?95wwgCGxEv$DK;9o|9RlM(`AeMM>ov4LYIX>GWTv!^oV zJ)=u_+%=rmg|nVz9CGJ}Svju1RO1r3*PZizG|Od?RUKo$eRGYJ^1?%nIH!`?20T}g z={JFOh-Pm_O#^5~aXypgkDhnRd(*O@ZsoJ`L*y>6pSxtF8OOzZBxikpy8_KA|0`6_ zmP>RJC^aG(N8U&?h9Je0MN?bZrSZA4+7%n?1qDdGg`t;WmP6ailFI!Q^9kXFT#*Qpp#ImYY*+i? zSty{%FT$#s<1{>@MkiGAoHh*?#Tm@SB#rSM1Cr7Zq@!@f`x-Z(#9;!TMPW|8teDoe z)+KIo-J96ujh_|yr}9sQ9GI!><1=^!f8TKawMO!7PJG@48hYTzA9g|bw1Fy=1T?f# z<91HCz%pX&I5HuP8Q`<U&~2e2vw>Fiwy{p{S{52RGI0!wFaJkJ|+39s{$Z# zx`EeI8-m;|I|zf5mf#IbT6JY*PJs+zJH_O+pH{UdeJ|YxR-$&9Mn0pHzN?YI<^d`0 zHO|H4VLY8s&>`H|{9=CBY> z)4IN3_rNFZH_9Laz-g64TD*eBZ~WM6E-qY1VMU3tb#;=4x&{zoQaJbB5}2O>4X%Y7 zK{Vo>(!bcdS34~$*P~xxwuAAZdH^h^Z6m}%?255ivT<5oESPlCPSPfwJn1x z9aD~yQdUl-O=CxkM4*({13RN84(Ay^y+18EWkIi{o3-g2<~sg^P$HQk64ZERXQt@9 z)Zmt#TH8%pk-E`UXSQH9>z+R1YoITolf+UR6z0LMxIi4g zU!=5Yizyp#J53~8NxQ~!D@^>p^$Sk1xG~wAOu@_n0zv-t*NJT(zM^s-74c9|D7fr+1fjF;Pyjt${KJ|8)0Lp- z;2O!e6u;`TvO#KED(|HocU+~z?$fCrPaeA}%cOD}t6b%U+C?xuz5VK(?%cxsL_9O3 ziBP`Tl8-;?=WAiR*VQ*d^TpTYGzaFw+mvj^jqB9+(Cc!lf9y7GOFXg-dY5}>%1bLL zQ7%ntNqxfkj^lP_sTlJLMS^6L&V?U|PXxp7=#XH*w){#q;oreV!TZ!VepO2kd&aP% zxtL5tRvQa@=YyGf!H2P)$;PA@oLaLA_s-|h=W`MFa@W#beM4VgnI6}@8I`4Y$8pzw zQ3x925kmxO66*U%=gez;o1Ao+()`Vw3@|?zL^FjI(rCrcMWV_iLWkGElq#sp7~cTb z9)gWm(p5iy8$N5&&Vu_xK54`L`UO(0a?u%WV9^qo$;r#JwZL#Qii@q{oyd1Jp!0A(T z134h{7CnSxVkxGLF1<}zp-)*)n`X}`JmvXG#&)o+BX0j$(?42(;r^d3AU(JgXY&VP&9#L?x`s=Y0J7dzA|$1c6{b+qh(BD8!=THFIxc*@RYa!URLJ z?@+jnK68FIl(_zs>A3UoN9SR1Qz6Z#D=%`_0#bQKjN;jcQt{QNiJIe7Zeu;+ys8=9 z!KM)ULyk^OWn`RQ89 z@bAyja}$E554~r8`oz|p+`?-l|DRJ#3eoeH$qwD5%pmZS{`{^$Bs6|}Age(Y4w`#4 zru^ygMdB58iAcHWaSh_%SIZjI=r$(4Eg-D9Cp|uQKz%`6bMyR4{T$P&_$n@6IIVm# zs84*Ld`3C>+P+Ph#{Tasas5|CuVwd4ty9->XRaViR@WApKse%lbitv0rVT;yr|{S9%0CEr7$8RVY(?icu?y1gG7Oq! zs%p82b>@OJUy$?2z?Eq{a#-O&M49KIhjkx^=m*-pP7_9J@IU~ZX$a*DUMqPnfseUR zMvD2>H1O_&PVh+@v8NMiapQ{8Mf}!zmPFDicynM}vw;&rW6m6bOhdnsc<^N#3yB1f zy$_GWedh;6!zl)V*`>G*D$kLbwx_Sz+Z{Hsv)c|Z1{lL$lDX^eMO;6#<2LsVC&mNc z^HpD8Iq}j4pnSo>nTs2M6>fa`9gl+uQ*PSCVud zllYYgwf4fI7I->H!l(5P z5i-r5i8DrOf|X~CSH-Kp2Crh0x26)RgZ!QAq`koz0*66j_kG?(kHg|0#a`p{H(V#S zz?iSr_7MpShF?@8BpwKm_I*7ij#}C@)oumB|XEHYI36= zb9QyaV@?}4N=zdQoqOHay-L-NGq@GP&w{eBr5X@ccbf=>vl)oO1NNad(0w*Bxi|L5(|5)=vb4;T404wc9JEV*rMqb#V7Xxc4utStDHN>f%w<_m9E>Tk*%2Ztpkd? zN#&KWB@284z&u_L59{+rgWgu<8J{f73W(;zY`ov82qGT_!z_Tr_mAuz?6+~m7?HxB zWlh(YZEQPHT`9AE>!8~?LpKrF;H!ghVYT%P&-jD!`a(iUzJV|UFKvS_$uO=Gn`ze# zEyp;W5sudF2=(oo4=}w&5nU^f_gWhJVAtqV`U4=B@*eQjBCg-8{(Z=k=WrSmdAA&2 z*FC-f9jsxL9D-(hCg38RVGyeBuC>#P%XQ61HHeaP%&}E!Sss z{IDj^(e&zV3XWa7Igh!1O4W*rKdaolR}mXL;K2}<8{6rjn0gGbEnL)}a0_L(-r3lQ zBAbq+&AW)wT2i!?nJn32oRfvx@0tG8P}wY^1rEc>bV-TT9jj>YL5o3LmnGxCe!a{b zO6r*-DbSE;&=4{FF~FbKW+u`0!+LW)Mh53L;d(IZ%E3t=$Gzk?y||x@AvQl~w_p)w zP*q^l#;R5m_ap@uW^$ZtJX_H|UQC@!;aK>>z3SSqkm(uw;c*^VS}lTaK)$Z|)-y>` zCLJqYdc55E$6O8d)+F8lxuL?=I~^MhGDVXK+m&Boelkkm>U z6&D%5b$51YSCSR5=`bpHrJXufI@Ym=s(PtHW@{k5@~Mm z8L6f8-=@ubx0cq)R}cfgJ{g3turoP2CWVEYYPE()%q}DoIaUtxff309oNSq+%v1`X z-`Ut(mt$hrg7El4G}PBJyKJ3S2%SZLBK%W<>Ti?$->h*awm%mR@V8!-B3ap$f;~wW zaMfi>VnV-+hsmJ7&1}}oaUB6RWe{Avx$HVn*q#Z_GzUT$`LX3Sris9A8bb@0L_@rW zPPSC)wXkyVE)wcsruF=(hJ_*rj`vg!ixxl|_o8SfKIMBJ*$9OikEasO>``j;J-*UI zD$bb5wIp-JT>cu$u=$L;{bq{h$ex1T6z4!9yA+9JgB~2r3?)Qqw6%gkk`ZR5g_9S2 zbzt^|R;k;uv4!)WBIg0sMQB1GS;2_+S;qQD=#9ErwF6qidmY+SpNfzb>6ZwWBWl~{ zi!kruFgaK`_Q855$jWdf$Q8bhnrgyC|n30ngbEKX%U2;-!4pX_%27hD7_p6e;cXMMgVo*RA&vn|=4cmr_cRD0$Eb`;5AQ+tf zH6UMXPaD^Gu<8R&*l{ibocqo@Zx0lnwQX301|{Vr8^J;R9dw}Mbt4L;dbI{1M$*8S z(FLk}wKX-RxH*1U75~=QCE2$ZZu0Xa%7v;R)e3+De_S=i(A*YT+^{!0UPo2~vHR&t-9sk^%*K?2kV8xSJ`1lm_qfc}C;mnZg1B9qFv%IcHXc)rGCu9`ac zgPo=HfmN}-v`x@Aa0qE)!QA-Nj0?{_7rF5l$JlLFdf9%5*ERg9Xu<}IL|Pjv@wq?{ z?eG%(frPd|*1|6`wZU$4ANyVY*u&=f7~mUt^%c2~JrF+OrM}QKe?35`P&3UQ0MRgR zQ7u9-ad8V;Q3?_vh)ynql#a5zOfrWB*k5hkd}j|lYrvUe{pNIMKZx0*Y^{mtX|GEIJ4mh*JxMS>)|9hin+O+kbstFoqWLF%_`3Te*Q3UFps zdG+F0=4uB$IuR#ToMN8wbbPqpbYOAO8(9FU;?R2;j@)Q{oB>!&!zs=}T-FfIZ&pK) znyi}2L7#%CDN%bXX-a2JpWmCc-4UN!#7U)Y^>@-l9c!}nb@QdGg>9uL!c94QwAQ~X zy$PN(GgZ&OWF=;oR(A2DIQ`}RV$*$_v{R%mkYw{Uo|!9bkVc`lhbv)VkdP&XCM49e zpDYRQLE{wmeWth8=i^7bW_5#l4_Z#nM_=1Jtl$QT)Gc6Gqd&JtjzvG-KEp`8S;cj+ z)lL@|yzjLUQoeZ^f@qufN0+h+NdMkII1~H-@F)RtDB}FS2ltMa&*78Uzis^gcU1gh z-~C@_gB<&^%0L-UP}QM0^ao}OuSOSSi8TJQloup7Bqdp*?1ASLD!>h0IlMaT&dyyo zoSCY&>oQ_5gP1SSo>eZ^KkOC&t+VRC39Jg+L!FIWEssa1(Er{*@c6)XSikJnkVRt~ zDA2AUAX&6L!68%uCYCLA^Be&VpF4!G7H+~abRVj&5O9MyGL?ra@AvZ`qnV`Kha+0} zk=%wEWyc?>=QO`*RtmL(@?iz(kxgU)CX*Cb$3D+YFnNwMv-XbC6l;gM(es_}H{Vbv z|NYF7aRsqr*Fe{cN9 zN2{i;ZAVIl=kERQrk?*%MbvKhKQaJI|4%aj&-e@|j3#wCW4D5HQj8Qh#De5UTAn-~ zF~m5G-tH2jMp0kt4J}Yr*h|N(YmQ`^>|DP5q<(EL+U?URPM!h17Ns%#5l_3RLH1AKshciXRok;noHsh5-GBym`zuOV3_cfL#8@AK&+J0z z{%(|44(T%Q@gC@JB)}V7N+Yx}i7plIolAkQOrV^fU#xC+_IbcvMIjZU@cEk8U-Pvt z=V%>yq9!UgANe$k=9DdB>=2$Z7r+HT7$J#xVJ+uzROQ9`x9fk~ z5%_<0H2mMVxX_c-`^yEYZY=jPOhR&3+T~I#t#eY`=xEm$K=`|CrwD{_?k@Q9U#wB_ zk2>LRFQfHO?%Lha9z(mHH4F^TSpSP+_4|1S5Rtx=ykurh-e{pW56 zQZ}&%A~p{f^gP-9FtAKg%hER@->g<1*i$Ok8QgwjLP7oH#ixs3Q7zoyz-(uhW~dnN zZioSa6_`$nHv*fg@^ZNkQC7-;$U9j^q~WIQ2K-z#wkGs<10vrFZHf~NHCgFrxTv(j z4f-BDSzJfy2eTjjtan8gqQWEs?mwdKQwcSghb9h8v%?>4DX;KcktC_qg|=AO4IV4@ z7hEq*x3IBq{2*!uf~v79oI0a=dNZ*Z&@wLOy13vNw;wFT&Cy*MY2j&U9JUWZ zKTIjza;y@WAKOr*7Fp`)EBC6f{)?BnPnrvy~ry_+0q&sXw<1bQO?}euwDU#~KcG0+`G(K{9ldT{h0==9lD% zyDBffi2bAg)ys%hsDN$o;OeErRsd2rPFfI$YYlE(qltx7uLKL~mYNd>V@$)$ERT#P z5{|k!8~+W8vdFKOhuGB&tHb&h zJTkB^-NHzDB@=<}GNL@{7mRJ840E1g^Ys6~pD+v#f*fYd6fm1pS$)#$k)5K>)%3jA z%eCnXxYNcik-)-%bp^l{dc~4jLCR{+O%ZL$=538Z{d)t(Fvq%!w{)`@XHe57C!eR;%x1&$l3u6X=Qs>-ELB z&bUZs&w{FG*WQ!>+vuPGU&Ef2A&!+L$<50x)`R+myC3=w1&AYS5`A0lsYRh9AT^*> zQAKH0vXP8hL@`N808qRdTtN}((g!tDByo+?WykEM`Muz1UK!#IcvHyeprGmgvfkWq zl0fY!iJD6=zX=ITYBxPVWE2qsx%Y1Ul_~NcUDj))&b$2&eeLbHKv7^tLx`-ss4C#j>&?M$_H#Q{zO zxM#02zO&Je#+J7_$2p5XNZy@$Iwf5#f*(7XpQir(8Nj)p3mZ_+Z7gn=JTp@H+v~gt%+NW)wdWczka_&BAk0`;-ZM9(`iB*_b(1dibwPGW5fEkFyo7rPd=b;RaIxL32IW1Nnq4Y! z6R_2!D4T2K;gQfZ&Me*QP>)8>>ojrJ`9X~Zp|7;c47P#XI+XL#l~d+sa&r9U8!4AS zAB~)_Lv>LP$Y*~u+(<2OLQjcW0RtMG+hmmL=Vq)g>Re~_rL!PKe@Ks9Pe5=eoPW9| zC_+Z_z-*4?h$olf;AbY^E=APB7|?S41EO3C#}Ljt=??6~#>~Zn43>^njkt=5LxaVn z8YCbK?Bl^s6^$UG(Fo9Nylsd8_|8CeZ(i=cC!_JoCDA(7IDx`swb9BYX9;63U{laY zx?Gl~JHIOZfxUUamx*U4vI{ljahCsP&IzV?a9?qFUE#iN49Yk>zQNAz_mm8XbBdf^5)ZOv~CDw$C^Y1mN^iLtPdKQv}>UUe?15L6)7MT>0q ze9BJD4N;TJ1GvPuEYk1IiDA2yCKotW|3#xO2B0Sm3=f_SbqO=Oo;|=MFgO_Wzg_-EIYGCLt7#hHOjt9b7DiK6)#WqCJ`>j@-w>pR|KT3^ zSM@l4E50LW^~Rf8bmuwiC!yjFZ97Pdtmll_cdDO}ewW+7B`wX(1Zl)!onpr99T|%H zYzFg+WrpwchP@!4HMrD!JEnn=Q+m{uXER5C4o4AKbu>V^W~men_)OD7ETXAsa;`2GN_;&Pnyyv2jt#&n@Oh-s~K8 z8j-OKfjpIfR`Hy9znl_|HvG)Iv_%@mflTz~pDFKiefozstrb$$*FMO|_P$jZ;S63lcgd)=%eBFR3OCF?6r3M^1$dl%>t2S#@#8IU-H1 zJfxq#9zBuyXl>9cE!JGL=}9!7n`R4=&3aD=Y=X&-`FT5FU}Z#TeJ-$gy=oy#f5{P- zcw9PIGAY=rNP$bEQBLw;;apNTC`HLguiU%ao2I)xU)#SmJA{-~$~yf?Mcw6z6Il>V zmkx7AA}=&L+P&YCz|U+fHIy}yG|GAi0UZDO=Mk-`v$&?%cH{IoM-nzPm4hWc9uiZp zY=Tpe>aTf;7-e7cjP!S`wL7r(Lx#<- z%x2xaE-ez8*&Z0y@gKjkx$ySu(wC|V9uxYIxO#PQYn zrIs<6_Ae}WVROOLLR3`!OGrQFy!lzX-4co{GMgFqRdHry&##xk77>s;vSA5ojxbDW-NPB zWUukbH2T)NJzU)K2TViry(7Kmkb1PEwoo>fec{{GNLsx(t0oE8hRqMtwGw5*9Sybm z0Im0N4Y68ocsaQYHmz!&+^;lz8Qdxleuhz_SHPLl8Og_J`8Q3rwRn!p>r{>zcK1uC zszJ~w_TpjgwX@ZzHGO= z^?2{mD&fTBO*E44e@s4fi9x%%-NxpTJ;Lp(u<-ER9kNfL-=Sa_MnGTvM!q&*m7iaA z#?#4aOKRl;2tE1uZWiHyQ#Q7i(t>?z%urx-l<)A{KaJoAp{LmaxT&0bYh9%Ga!T_} zvvH_0Wt&ywSvj${KUI$tVE2Awa>U4`P$_;KGuD+;;D_OrI#c*793eCjZ90}KdlP!P zl4R4Ul~sW`rcq($Oa`Ys59Q9O%fmcEmM?_6Hi0ojebPb{Y$Aa40D*ub@&Ue=QG%=J zh3j@L+3aS_r4IUfH|jb>S;b?X=W%KxDi6v*KYde!Wr7ZV`m6+f8_nlqDB~8Ennx{L z$upPi`Z=IRTBUTKRxJ5FSH-*8uzjT1ymWUBELl1Txaz+a0-_>kx)yFD!W79&7qs1G- zoutQT6Y5}+del27C+T)s`1CpUjtb@&kfUUjEgDxV{&!Wq@;43N-nq=TzDb=CfWu4d zjY0;6G5GaWF#uooJjq1FxknoJcKH`GOzbIkv1utKn+)*P>O%7d8=a~g1N>`h@7+QR zk+jvV6G(is!TSpU}*6*Q? zAFf4BDNR1+*l;^RoX_BLlenP)t7~nVBARBECr$^0Huu975PS$Ey}hfU!$%_-Go-@l z!a5R#(nU_-V>_4#et-{7#E?mb+~rP{O$3Z6O4)9@L&O-W^|))q!~d#% zWUd$n9sBsa2g&4r{ulm(cY0&gOp_|&gg5*4PCqhgp6F(3l%zlO2b3`8t#BA_SCX2h z!?j~SL6TX27CtiEeJ-xzwkvhpE5N&Nz5!%FZ?9$E1Z&!DAgv^^sm%Ym3XOKb%z7jta0Vuc<8g+()@L z9|X^(8v|OWt1HEADsoDVa>tb)PxI;&UHeNo)@Z8$4PMde4K%ncSfrjEhFmZZ2Q2Pf z@Cp*VT+uBfwnc+S7Z?>MC#E^Vv|>>uyXE=6xdfPoymX#(ii?w38N;whUNft?sxj7- zo;*A#>;fq3NpI;+94AG%HL)DeAq6W_6^I*yX~kIr8QUM?izuc!Lb;_YnwS_uV_GpV zdtckdc~e$~fdyq34aqFvG=;s!jJk@=wFd$3q8t3ve}U|_;pf= zS;_j+j*D20Og@HSt-g_ zv{hMc8+hJur4Ddd$RH2C`IR(fPl9qb4Z~%abcyWQd&?tkr-epZ=;C>|X4M>C>z!6r zgIt*jJ*{h=x#{}Ohaw_I!YQJX91a_IH+#@5Pso|WZ3jk#c6+zy@h~p8-l*!CYK`ki z+tgM^TwZLPxXWIj)j2J!vV6$UJB2&T^7481WshbvubWW_lK=@w$Y zVt|8k|3l8gJEDGw_-I>q1B`4bW-aAcLs(PwG`Yf5Z23$nHx!X2moxv26I~9N6CUH_ zWF;8T-&>NYl`d=I&cf~0iZkH13B^mm!b$s;Ll2)MmOZ8Xs zI)(TFT8TZn>V10PoZjA>qJ|1GNR%W|Oi2uSJ~C{McaduG_HGpo^Cs|lKxVmPoIy$1 zG!lD>Av2p=p%<#2c#+6@TkGX>6Sfzn7r7=~i;v86o~VV9U#QoE4zx;ft_?Ieq?n=L zK4aNQ272#I5?xjqpyLLX)T7f2OpL?o7*h7mthZfBTqmi)!v31%PPuq`Y!44Ef^d=N zK2~4kOuhXBEbl6IZguHc{)o={VvtyoETC{;&JINhr&rb><12pW^#Q1SP3|5RkQv<5I)E3toKRDIi}qLC9~z-neA8GG3D(9ViQ& z#mQR#>JwOs_6Xp=(;RF1*I)h_JMG(@hYY!z^Jj8a`i=|*7jv<-`f~!)N4~%2TWNm8 zeE>zzs?pwblV%=NaHVJEoA@3M?6)xWux46;K4BkJg$N=qs;O9(;0?VXrwT5bkUI2+ z+kgpMxYlCQnej8VkEmc<#cd7@zb0ydTs(5kQOChvurSe-eR;2eBO^DUxk0N9Abq5gX)O zN$(`r3TwDopL3OsI)zuY`ZnOT@HCm|Z|`IEAPu-c&5N4`WrRL4(YxyB7(0&zGh?U1Dd{~Se+(?ywsbNux#q7IU7}Cx7qg@)+kmddlLNmz=Exy+x zKcsOVxmBNyV(>}SGGAF&jvEkjN%HPCQVu9+pvB2^ACrO41D?O1;=zi9e7`*`LJ`b9 zupbwkJtLktr3?J4i(X40lKtx>JO3vYqY8O2{r#`s=)EA0B99d6h4&v+#nvHkp|k_T z_6baWCkB3-G1O*d(;Jv*-TcG*$kY#vVgw`~rH#6m0^>2Ct2yqme6I~IFqG<2to38I zt~G{Fi)An2(_#_WsyhOLRLH--_5J75;_S9NH;A*e?WWbh^-CUTex7{ST3GdU053k7 zf6{LelCOuaxD%IAh+ib|^Y5WD1gYjG$m3O0)s>r*iWtWLM#X{*CD&6hcKHG!2uSV? z%>-^nr&}WtHF5Eg6ESt0V)5s4B8JGS1UZ>QX}Epf*}TF8XZoMTwzgUg%u5A$u5w(2 zx?oB{X4;Ae4iq8SbPUp9FR9Vlei%h}4g!*H9^Z$`JMsB}%&?H3D%Kz^bW@|@TE^}v z)%&ykk8Z^zJ0dM#NHV^D?)~c2W`J>x$bJL1nJ>tRSIcvglDq39W|}qsauqOS;JZ>88k(^361{oUr1I_w%UP{F)*p6=@c|Hle~T!*E8zaLVe{+Dcwz zV*PPc5NT{7kD0BQNwyO+Gjn2)nVDi{W`0KJocqq4d*|Mo zs;PHA%%|3FbxFO|>fZa`z1I4z+qH~GWkK5itl%ag;E=|xs}2`@64jWwcEdhsle@r` z9j%%Smv+5MfbM;LvMcN`?Grz{xPZ77Atr~IrL-_AO;IwcTqa0lxJu@*cno&|*w&K*k%%%oEBkd4xqpn$)?LDL z<0^cJ)7IY`$u7<>%KtB@_9q4elolsG3MQP1nrl#7S}uyW{c2_F%z=u3E7}$hjDNm6 zI_aKFEG@^gqudkLWZTSkoR#9Bz+14NQUfY({W%Lpi;}0W1Jmx11Px>P^xbt-R@WdP z1mUco*IO?EuqjGh$K~2fA6<1)Sk#Xbpj9a+H`ZIHPmiAWq3?)SL97B8yaQCGKELQx&qn~8` zKh8B9#?2504Th4kPj-g0PK|{hic3;b4^u8U|3>KJ{~%*yR-^Y>CWC=Q{KAI+e5D5i z*%8lT5z$4E0Q^NYwqOy~FI%NDR6sR}ecGVtK@DK}b_2R;L*|Avx7Lp`J>ENiONuTw zrZ`0EW1n$%R!DRW3cHRnj+D+M0`K3u8mn2zOoP-4$J{7UVyx*ma9yrO!UzS?$$o4i z*qnu)jxSz1emB#tZhIdaU1YkV*egFgFv8SVxb5Fq41E&&0<|V1Q>2U0%Xqvn8X8j- z+h3AT|LqVkklmdFelGWy>=} zW)P+o9hsu#gCR($1AfBVoMjclVB8MM4d_$h;Pla@CO}|-Qhc9cyj&)RndozYIq8Pl zwQYk;l7U%@$E-xF;%=6}&Smf96S<$YhszO=uzwM^;d;3La-c%-H$&`KJ5{S}eyk!| z2V`HaTLu3?Sfo40i=x30ia%-(ZMjBHpdnOMHqSKb0@nr$(g>X&a%1rND375%Up(BA4QGFA@h zce1=jrhUPd%|opnEL2eB@bc}PsKegk^KjLmJ){T@7O?Hp4#Box1v_0pBljo7ZyxTA z`w=EX!-E^|cxFv0J329Cn9(-HDhhWFb#(S*cvb3DStqv9w)X(}0RJCbY4bRC10y%r zzvZ@diClE1-4az*)RHyKD2TYQo_sCp1$_4wH(*LvoXlN6|EF*F$S8J6Aez(2O7L<~dI?)Gj( zn~N-@PX68{0!ljxM}6xXV|9 zV8DiUHyCQ&78<dU<&ieaGy2o2Bq|K7sWtq<6g1_v zIMASBp2coC8XU2v^u7)-7A{kCSsKzaG~tU3m#A!A2oU5pxurG1nq+oqM_mJY)9Xr(tyj@SXAw2`T6(Mu#da?4A!-sp+cbHDzdE7eI~S2LcCpO=gv@AiOy#9@;7iyh+4!S43R z)(D9Mt~E1n;8RK%aR48`Z|F5|h0@s=t;%ET z@PzqR$?i&4m@kIIcw4CcZ-m~3oMRbe;iggQ>vfcy|E#Te`cKXO?wh;1RMc%?p-O`= zTVWf52YoxaW&cSkKfu#{tmb*i1K&w_1z3bJwnWciiP|CmZ-f90?j@0oa$mCg=>0!c z`fwTnIlTAO>mR>YNH~`|7yde2_`U=;!9e`^ue+OOyjXXnum3_<{+K%b_K(VTS2Eeu zM}<#*a5$QJfDFP7f|nl~^9`x|gUSoJ>;L2CPlN00lSeo;iHtF7z+PC7Ytb1WY%gf) zyxKg-A-qvFRdHz|a+M%Sx$*MzcyQ;}$xJN^$GkOR@2Bj#qet;;_WyGqkZ{o{HBWQK z^BYZ;)G$j{bXq-=9*@46I*Lo7825L^`YmizV9l?8d1Ji1X{*dK(aVwHas2>373xgG zeT2|F7r%ZN_@;L#lzN%>`1ND)yP^JEJ@iLP>3{UYSh)i7f{4Dm}9%&`t#W%WNQ~erDf_e_tBD-Y3$i%xfR%MZ0jL@YK9=2Ib*DTVJ zWJ!}6?~{n4p&E&cah#r(#0i)29pjw{&EvTywAC(?Q=XISMcv|?w*aaB-#nanA^dmK zK*a8&ToBLO zk1V{Jyka0M$f8nWczFyFrB<7>^}Kn|pSl-+pXVSnlQq7tu+2}AM^??)Jd3MU$5hxZ z=c=@3)7>sT8BQu3P|ZH0rjKrYcWtGvZhGI&r1TJO8P@Ze z?OHUB&&$tgkLj8-)oEtGHUMN~1@HV?G|kgp1KAZiPk!ak1H76OQ&Yk^C#EJ^b4jH< zJQPpFC#qG7YIeuX%YI>onH_$$m^zwK_je_erd|HGd9TLT*5`3-JXzY7=h6A@A7c^0 zgt<^wGH%x{OwlCtZ_t&G!$WM3iT^Cr-)0ejnk{{!rVXpe?ziIfi4N?X9YhoR$(;iX zonT?%WG^^{FNPu)Ffv0_2a1<>haaE4ddC9l*k1wbmf(L9b9G{vRr&r#2&E5SL*J+{ z(FR}19u1dMCfT3oq<=gcPt`WK%;4>=2TPM{3ot+&Ga&Torsfyslh`-1wdeY z%X_{IwF?VnY;*Upk`BhEe>FT01+0!FP;&e=hsCKj&q;7+uIZd%rbNn{2&fZJWN~X; zAj|2BeB35@bl+$lbSk_M-?$z!oaoV6xxT`q-6TbxZoZ9gtD_aLuzx6dKKXHP9ErZO z8BjAvQsyF)(lb*SXc6`y9O~IN*Nlf{W9#cNChl~XBVz)OkgDJoaNvJ^A^n&&`l8F- z!~lQRY>`y^uyeT)C#4r!EKhb`ApDmbo&319&zt0v*4?cWyf@>TcSgf_av(>Z(Y^Et zu(e9hGqch+kq+6(tEKF2FV&m&8Mhb6UxJzzMcZJsTAQQeZAbC5#OQ`RDL!O8V!q*C z?pyCr!F2T)k?uRA5|Li~a8*~|+p2~bRUL@9DU^40J^rg1z#wGSI0rLNP z^qUC{@xol`iN4?P-&@_JxC%nyDLMCG05{@rwo( z$(O<0(74_Z4QV;zj1gy>p0^jqkm+v8`cY&-x3kPU7fc6Aw)~xV^$ML95mQ>^b!RNn zE6kF>yVv<7VHzb@$lUx?iDELHg99AMIAS{Xh;}x1>Q^rCuZ0}72!`kV{ga>5^?YDW zPgOX6O`dOl=Gsm+wZ^}8$@pIj6P-6=m$d)tX!U>emnjaghj$VvuJX)qK$qfG5%?LD z{=j6hopZLy?bq|>TJ&VTxbIb~+wQ~C>m*eNd<>4PoP3@IufOEmNWHjk zP57)MP)MZh|Gr)I-ri~#VC!fyY~ectFCjR=Jch~M=3wSPjp_yJT*^zD8~%v6u4ZZu zp5W+cuu&lC3sISnN7!TeaErL5PkpwYtK7~ZGmSb(j4cG5^Q?@Cv~n`LmbDePEKemR zc{V0md1f4qYR=8}X_er&2r+f~zYptqGmh3tVtfwL$KK5>x8*zU^{a+y5^KAKp$u=% z5mbYK?ArmJ7d=B;#Gm5$9>gnO4xM7+T8A*O)oAmeunE-&o?#36irzL29AT;$PTCXw zFW<-XM0OApY56>TN8z=uCqy9iOR#cck8ex{PlNSuwV)vghwmD3_&CHS+~M=b3GYfN zzMgvaq3h}5Y3qzEChvsMH3dk+vC}o*{!vWCs`DnB;Jm@`a+&v~IWBSXr?h9DBhJ1J z!rQkuGyICad=1!X@PADc{&{n7GClRhuM7S8K)?4Hkzk-u2Q9zDOj>6QI)JHdAD2i2 zLrOR=3x&8BCOYC4xqtt-;774PGoxRDic}ve$WFOoE}V($kg`MHO5ruQ5hu9w zGW#YQMJn8&K6A!2*NU(?#D8!&2oFuJHhd8pu9OY3c=Xq9K;DEW;SP?dcNpVLSud@r zW)?6X#t%7^UEB~c4$S4LI7nmoSBY*U5hK~1|Nh{C5wyfRH~V?3@#YXCHl{9AO%nTjjek79pj$K3I@jsiGi63^ z(kjV3x`aU8j((si_RmDUt50yxyb8vGNpPWyuMf1w{?uK5V~#T0s$MBz2y>L)c8f)L ztAu>%2b2U)Y%`dN5)9PX&rC)xA1Zgf8&k!Kev4Hvtlu-O!UY-%sc>YV{hjL=%gHv~ z3Se6F>%GD160m@ww{PJebK{7Zm8B0QB-n2{@%0puaek_*16qpNh?+WH93;9#lkO|B zvPPI5&SdFgluVbNoLu}WpFH{|5kV};_i`BA?!tYV_`=ImAn}PbiGr2dr>kqi-sdO!=UGdr(4H zn-M2*el|9%pa>F?Id~A4l~X?eFL6>=YxTMbMa^olLZv;uJuK(LRoH@rM0g?BF_|GS-H4k|B9p#92 zX~S4#X&E2!3!oKv^u+76Uur9=A516?{RW-=%AJl<>z%ja-pa5h&*5N+__AD+eXLl? zI}j-4F_1OJA&1MVQW~GRD@>9v$Jk;V}jGWsm{B0ET{A;`FlE> z07h=*8Fj8zCF+wJg5k{~+Dc00Q{S?;R@B2HUvoXNuo92`INYwDqm=5n<9_4!QCSW7 zP3O~M!dk!W;l%jWT%6s8vZGfGZ{hERwFHp#jT}ZQ_5#fuR=Tr-y*}nMSQN*ZvGs;i z3cbX$&3F2<0D<#(_zz>tr_nn7$uUeB*P~`=fm>|u3z@1i7~FA@MErnEzzfHxn_+&j zqU0EMzVjp>&QWjn{cY>|7gcN=IUI{9=jbFOV&*^N0O!iT{KTjF+LR*+WYca^yyzm? z9?l4=e8aCzm14G^#5r2<+!<)40!x(jQfssL6HO$r=E^#11qOd~xx{!*;B<`rqz^l< zo%_}Cb+*lPa!1*D|9AyY8l!=F4|~k0VICtntS!o{L`@Cn1W$PyTaDmdiQI@>JWgUP zW%Kip%lEy>y4g5H9^qQI8dy1%{b5#z7IhPth1s1$RL#Qvw7%8-Ji(1TcP1&(0hPCh z-yxf`H_N>0Xo*`YO<_8Yfc!H>(3ItqkoE((j>2&Ef|)(u3$c%GrgV1dolaa+6Ce-O_EP2o+zTjw2d8&@AJaFG! zx7Y-Kv1Z*!?@m`lVOG1D9C_gG?$`yFvUTnPILay>=ykCQYv|>)?tfCP_hYB0J9KCK z5Kl*9K?|BhGd%}5czl)&7oI(9>+BKoOGt(X zn;A^#Tw(ymhe2Lr-`~6#F{5Zk`(&n?<5B-nWU8BRld$_5p9Jw;-L%@uhl#3|gpoNA z^Yt@o>meox=H(=pEs82Em=N6v!ao$9^2A7&;~pdH*0Ih8b0Gxla)=IhCFFBiHG-(F*3j~G1r;P=ch zRZ`dymHE;Q$@0vpIy9x0!>X4urTz5eJ3K2C1@DbEA?WbDdv%>m{gd_+S=C`KgSwQ; z)h`j{32DTI6AFC2Z+t3~bts_j>RIjHqrs8xy-dpux+k~vd0j2Ht|bvMb~gukj)9Up z7|gm;4T*i@y<;dCGx@HO&vjXiF>3UJqkb{QjTb&g?|rG~Qsqe3)Nka7c$5lxmHk0} zqgWf|AtGlM$3grhN1fLBrPAKA8+;dIg8bkqwe{rNCI<@=7Js>r?LAdDAa#=ae+k3EqYdR<-@I>veM&6tf*Cq>tC?cbm4>JZ2x_ew z=s~;G&zl*4JfC1-)*DV;1KMZ>@qU>`9WTBWq*@jXiidD`kQiL(!pht`nhbDtUat@Z zi73j3{mGFnn-d*}~1jY?^LIN)lcM2E{M z*VLR)2~+fAmU2}o6%+yolxlX64S}-tnb^=K9q!Ch15#}bi;*?k z+XAW0IqWT6ZVUVZ+KkPMzO6!?eJ=y!D;^J@JO4sSjTVVuGqLxO z`#Ik7idqH9qXXTaI(2B}va2hpp}fXpe` zZ4YU}7?VTNBlcgQX*Ur+k^EMa;&yk1ofwp%AGonY-{4!d+!xDofP>2N#Yo06o=G__ zkTSz5JByC9UK;Mumc|8g85!NRxEk4MZ1Y_@;>vL)&3lzIxiij~#B=QcyR>j;u!(E3k~@>1nd)_i8U+=)w18=>G?5yKbS3%Xg;Tk^jk|CSU-KBK z=qc!QNgkTDlCPF+|Lf|5;+Q`uxcAnY1>)J5gLvS<)s^XT+HQJUQQQVTy#KHm|@q++hzgQ|MdT%q9 ztmRbYhX7#-zlY@8;(TN^gNxfVcJEwN;`E{uck4o0Dz&%8AAXPDpsgKA>#@Z4a67Mw z-v%JlaFMTaU|EV0XFXt(#+rtV9u?q4=NpWSG1t)|0d<9ORp{Jqit>^(K0$bLayJb$EwZrH%AL;6aLh4oJsP|8z z>uqZUU~`J|LSZa?el=g#3Q*Ek?$6Gdi|k6QHLP3b_3A0-h}D!Jq*nD0E^oe;08yr69Og4+e`M4am8B+iql$M#o$L z(nGdi8+Ph6*ppm{MLW`|j=4^g%Z=(8=bkhZ0@7OuMfca#V_|{&TG@7+e#fVLGaD4a zsew6etC-OGi+se)tYkElgTpR^1wRb&U9%9;MS1LH)gwX?G@`Z4`F~h_F~RA!iRGd z^D}(-1@->0caE4}3}4VPx>;D-b`zQuyP=|!Ps!RVBxo0eQOkrJSzq732z@XROP?)o z!m=gXtWgQpF_C6D@bjz;w<9_0?9Opi)-N;~&ctE4U0M!9{GOWTHTj+l*u7({NCgqm zX|smUy~}jBFIlqMTw}OSewPrG1U}O3^zgQEk+#mS@bDc?fP6O5b5F!tYw`lbf)~ zmPw7$7m3x~&x4Hh_K0n_U$Q+)H(rA#GCW26h$LeuF%$__1AdIBFK?phJ>Clc@_Mlk z4=6@+Zj3I{Z<0I?ri~Whp_VAClo$(;zlK$>IyWm6QA9cC46{(N<(G`n%gF}=-~X>F zSz76HfZ6aN;BEnJ{r{pg{MT8}|E@Os(KiHyOL~32p9zmXlEM$YfZyy%1h&U!wPKOM zYGxHM7|aTcwt8usaoN|pO_xy)8ZH{7tDnW58eshG0{pK?}y zh_Y`F8hsD8XN_YiweFuQOy9_A@pM~)QLGHOs+@Y-0Q93=?yGx^bfe0J+giATnlHXt zsEq12(URI$5ln7A6;6h)lDexn!$Kjk$vIiY8Wm-ay$5Ys#VOWO&s7nb5xMaK13%IQ zdI3p-4VZQ|`ZN9PJSa}0Qdx&-8xXMaea>ej@s~=E6@sPPGP&1Rjg8MV_=PRZ+bLJ_ zxelz;OA>CMFiXI{fsuu$%=*Y(|5F$rWuuT$nxclEV%B;C zntE<2-1mN|ZM&_^uV0wp_b^!0=!d`6cPkA#v|3wsiNpMUF&(cY8}^AE_8F=6Bdbv6 z%LDt9fz#(9omPQWSck^5tdsGNW%?9>g$~Qg*j_^nRKgdLzY$g}Sy-)grW_O3!ibdc zZG_3nm*pB0OVym6TuWu0(iI-;+o!T&4UPtO#HQ1K z?ly-g?X2)f=Y+(Uwl0lp}A(|LpU%)eD>S zcN9?VY|Su?|`@f{?!j;Mu)aL(01Af(Pi!{Cd2!O=~Sd9O5nKLv7W{f^l7?f12FSSc!3(DUjB8$kl zPt=i21_BBAt^wD5kLzRB(fU6pd?;;iL-@y^Hf}e;tcwe9NcIydHqb3~;W;|u;gY>P z$+2Z9LnaXmx^P#3P5QEzLkp!K@G1FeeK-cM6EdQ?sjP_IF$eeoq=-5hehaYqkv?E2 z<^9O-reaF{TF-b*>l>^etx_j+>BITGPtBV0Zreci^QGu29Kq&QQ) z6*AHilfAjRH~X}mOfnAWSTP2vMOFaS3b}+(ZOJiII7`H=QYm`hmi5~X4(K`E(I1NI z5*XJr!|y+cC}bv+Sy-IV`21ltNS0^;LEbl6j&<5vC^U0!mh zQir=rvG!Af3e+SAuZ5cG0de!LBJEJzMpV_%G$&v{9M zo^-UN^Jeacwr5x3)~A8?-!mPezM}4f6s+MZ_YwQfezU1(j1|^sbjSsdy5WgeWRbw2A z`(`}v2ZiKogxvfpIN55G>xK7PrJzLyIG*>bg&(!BFC;ZC_+4!k#W$)@34H>3m&iab#n-qCYri$gH zDgqEi0ppgX?Bc;awCk)Rh_XdB|211fAIkSxO+v;DH>*>gH-sv2JH(3@RxXT z+_ZElvIp_{-BFT!n^7&1Q7t)HEbZRw!Eokt5Ax%e`ziZ?Q)o-q2wToi1f}KwKW#zJ ziXJC{JI*3jB##(%sD(`m$PV0wO~9jQk52-|058IC_5Zzh=>OhrMEVP$4z5$ebMWu> zEXir>wRPJ)vxx1zUs91-BFTE*sK~EuibB+0e6izUpd&%oqnfVorXDJ2ZpSkgPMei9ChbiIi)<)r`pOGUH#j>iNOm-z!B3wCx`)haYrZB7Eirf$A& z*W)kLWwI8O@t$9)M0IIQS^c~Xd^1X9VuREt&rDPa>Vbo8cLGmCpQg3H0M;V>Mq%z~nKJB(2s+7qIY)2(%1_H0zlW}p? zV^c?Cv8w_}=lMAwp8iI7hw>W3pq!BHw9uNNI^gkE@~%DUxI>BWHn*;lvYM)x)+KDY zEyXkaTp3zI;ulL|)^i*?Wx8}MvsT_d?UEQ`AM;9=Ri10*g5?@{YhxV#ClTXerPV^n5bU4wa4PB%u_7F;nPzsVveXkTo=Wsd5wIVZ-;F8 zo;*J%!RV3ab~nQHHKSX6|IA`>#H>g1T=2P#tV&JXC%%s|xf>#0(e)#bJn}C1B%pV* z)zSU{77`L^lsjv%7GN&d$GUHhQQZq^^`shUb}y}{aZ(}@7d{LYFHT8y#57+PwuRU@ zKIG}Di6z6buQjFw`^gKf6x$kr!c6YzGPKIK?-3oKr?Ze1c*TH#{Kz8r;KkU_cT@2HjWCUSb2C#_9$Ok16sC)#C6|m_N}Fev#Hm*9&rA*M zyfR9?4p#%<7G|}z*s7fHs|=_m4$%iC$dgAgW^oIDSOgmPo9Rv`U`!v&6DbX<$5&hC z){TkfBDeNPTW(W4ync6nkvlRdQd4ZGEpIwCt<8g;R-UFF2PL#~KW;G0~tfngO z<(Tu^H8D4zb-;9lb_^U@4y2JWg`+~BbBp%MeZ63?<4q1?5G>Gt;g_A+S>frC#(B73 zz4tH)+5(G$UaMt5ey-FXz$(Ew87`l3Ul{&I#N%mymVoPI*}83APKcj9*K9d6uf;`_>u3+;W58HMC{vK7SiYav<`j*A+GI zp%nJpaEdw4zs+f1_`HWy2mz5U5yS4hp>Eze$^}(P^Gf|)0ogYX4tCIlU_~79C6=xD zI>;YCFl~yzs5%y}@ZdmBb^mLf2pw2ZqfG!yk5nXWZXr=@*rUz9NsqMi?V9-3mC~Z4 z(}H%pwo@wcr|XMNg1M!n;%6N(4*jKs@d9qx8B-bjHe+4fl|(zARvbFdV(_pJ<6mR7 zb=B;rJ5<)PIuSID7?s%D&HED1Mod@eVYN=*gT`jkKMXi=yr6Abx5O!C*+Vcn(&j8g zB_Uf}c6<#l>jOag#%WP4UsC*$E-8?AR4UZA0jOy@A61zG!@i(6Kj-nxSwJ6m>#V7p zS55GJ6^==q<~)wzp;9Xu=DjA9_GBJu|UDJR#u|li*xyP>E|3z z+*SKiu`5eoHKw%$c}WT=3XzoG+&tOc`^ zd=BE0&M9N3N9kcd9>w=b)rnKE4xzh%81|_)#%H1|I7DwTa{TFH=BJCAQ{{H)j88*0 zZ{d+_(pm+U+`6+2AS{kM(YDB|LW~%Wc}~0sGhwFac7Nr9WENYC@ukSrOjbJX5XVGZ zK=&w~E9W2kS>f7HlT8P`yJXI#*lp|=+B;0@X++PN?r6CLT_B`C1 z^c+0m9G_gaCjK&9J7S-+w0#K$EeuL=Q_&D_ILUJTQ6$(^#5rP63~Q{J%S|?KAbUsW zuPfz-f)>|WW+%vgQ?f81@vt{hl!3$%TkU#ke2|j#@+&hs?=a1yj52A&#@Si+gWv9Q zKu%Q`CK+-az=dZ?W@e^7WW}sg129|^J}k%i^?p;D(PnR72{n5?j^6i&?(&0`zMM+a zsk2y_eIPES!q#9N`}hcLPtMD?bsv?=L#=s33%M+Tq{J=y4W7s1PE?T(e&Cv=zRp;? zJ@@Kl!dSX?;3w4lsqJ|e2}Td5_ap0j?J=bS-k|(T%XPOe9w=Myu~pE#)94hv5HHMG z_4J2UJr{3i!Ntu{3B+UOX|l{E?@1FYnkdFQ#d89xBVPNKqjQqWx`b}s@M&*Aymj_9 z7~(11Wkdo;>p!`?x;%j`Nkdr#E{l0QL+-i51#3~ic+tStluq3_?l#en0>6K{Io}mfCqgk*K{t-_QTiPBSNos`@!~@& zWjs`wd%`HE5$fEdwEDyZ@<-(~drFYaDVZV3deZR+HJj4P_M*eXFyd>yeSUZ>E|~5( zL6JI98QN2xy)h-Ju;p9Sqm(8gLN zfdEQ!ZCxz65ir1o3jDyrZrN8}$-F(Hxar4dx%6elycg>CQ(z&BS2`OaJ?!L|qV9YS zHA1ZURlc#&X)12zY9|+#Cnzs-CS}EZS^})9aoGy4N>`XRB+As$m`Ew4{CaV{RZvo{ zVnY}WLpSAlue|5%mPCu1v7Qk!;mbP~P%WT?=i&6Dq8*e1DJ-a11tE<<4od?cBf8YE za=IKMZ$p$1y@3`}y>d`u?ut$~*}-4kO7v8Q%#k#Q`P8Zcs$q^lkptsu(EH-vpI~=t zX?t@^&K4X4eN*gM1@Ta)8@Mv=FPo_4ZEG1VY2SCRc8o~tS24t~x!AM+-cA^Dpl@t$ zf)Q|Wv=n^Q12yGlpAy>fEJyYG%;TQV8y}VZx^OVhHNq|UcG-{?Pv3YseVXLGUATFI zMGB;r-N%~z>ofxi`6HwxG;Td6)S(nN)R`RrXpwEOrleYcl&QyA+Ba{#t@8BT48~n z!4Sx@Z7Z;r`Q`u*oynvgmBAYg8W(M-DrRSSn?~y~o;0^H#s?l3StL+Oy$w@?aouAy z#dNG0J;F(B#s9l*KmHG&BN7UOAbd=e1NWii6scW=0>213VR0=j{KRdkK8Mqt zzeRHB5IQdv+0UFp3P_k7S_>%;iJA9!=ReK&h8>;_9i?b?P!{p-%d$R4+QJx+?a(0@ zj;sVVPht%fDzd=BKhR4wxL)G-Fe~mx8?j;eaX8PkPW9%o#hk6{7wwE>_j@%~b+`PA zh``<sN}I^GY@yke9FJeP5|SwiiB=I~snibrG!rc7&hlTn+nH?_Q5oonIVFb%>2q z8Doa}f#fj1196ID^BuubpFqQrT9K?7*B^M)N~8_b_u9!ucjD<$g*m6&j~EFsq9~_5MVpu zn)u_#8T9)rNDeBRke6o0zg!`HeBABf*XYG$7kgO9eI1!`XJ=&96R^F=fX|tnP65Pb zhg1A%(XrGglH@z}T%UDP7n7fZli5I+)|Zj}b{5yjo47=*M|ubRD7i&R2MMiNveeu8 z7c8ULNQzywwo055X%{bv7fv}GE>Trr$SXUQpgAb-`(+9Wx%t zel69bNcQsajs^SCwc&f=Vy*!-6?JSFnY<=G5t*+LbhV(VhBJS6-2@B$0Na8q&jbcX zx~FDDo1{K``fDI_wLZ$g>{tT$$J<91{>b}WHOlBtc5flZfz7$JcUihnVIePQh&E~{ zbp(_BoPOph1ZR`fny{-Pk2xO&3++Jm9dvA!-9(ZQUf-=31~<=E$&UZI0M8#e-SnQM zn~lhOXLx3QChYKCtn1lY;}vNB<{OY$aVO>WMu{^h4)np2d?gJpBb|*E?-EI4E=S-o z?kR-|yDrWFQWuWJu{os0d$NZ@@9o)1n%Ex5rWBE8JjX(;5B}0h+kN_j0IQXmQnLEG zuc8*_Tf$0rKyKdG+MTZf{&4EhL@JU?9u*>FttS*iiZ;G;bCkj6|X>_@jmK!sqN@O z#Jvdvdl~|-u#m?u$9#Q!#o0#Q=R5c@bFI>yC@4FytqiQ!_*l}9fs`7>lM0`LGEkAl zV1u_pfaPJ<`9!Ot*;vAfSgi}w8rF~Fj|904>$RplB;}sX)!m{G!L1Xw9C2l3D1l~# zzg72{$#2B4EORXryUF8^FR@OTw@oyZ$hjV^Y_kc{W1~=@S$3wg^J{y4lV2>}D#l0E z-CQ>;7PSvlJ&7JDlCr)Hm2!xiNk6k+?acDO{>-O={0!M{2B86eBUQ6&&Xz z*y$`jXl)8sY%j2c4;XXFb0d(p$6C}j5z^GltZzUtFG zG3%>OHNC;fZU_t%K%pB=>FjPDW^(?LDFH8m&Vsive%mQ&G}K2>-1jgLUD$WBwE=LrU2Wuzh}2bTqG3vEh(@)RabLr~9(@U? z5CnKVEmW`_42nAvzoVX5-yDh<&X^~d0z4`q6IMEpG z0L=?`qOtSfN6sE13|-Lxw`ua}!MZWlPn@phT9PMFz+w9O)O`tkQ5fMTZ*4*V{V!f9 z-w4o zk(UHz1mXdV%!H4>*9IF#UJ(_(RhflV4qs-|1QXQ$nvcmEmIifvl zyY4c%5aGk8@VVSf#cu~Yt-shtz%eo6E7o2&(r9aIirJFT(oe_O+R=&Mq^bW#aQgaT zq=4n~XH+KHrYpx|dvWKa*Quq~BiOoSIeel}Wky>w zq4x^go@;w6K(WGmy}bvCj!p+P1Psg;f1CE|T2Q>s1daLc{W$9{VDd$!Qdt(N@1Nkh z?jfF(oM+0?VkPct71L=!ZrY#R#)U)hqCDW6n+KAM3`x@NF|{nV$f**SZk4Fc#^mnI zN$mNq)(l8o^PD5OKBIZQGea>Tw4*(n;}-vr!S2m(^OA_ zTDyYCU%Pf2)Coa1VzbeWSzoBHNxvLk<&pP}EPjaAKA08z1uoA+4lmBC?7k#=Nq4;+ z_7MG2S5#_xsgg2R7wyWkJmvNxHT3Oj0wuUGePmnlE#g`--|WkbOc%Q+qPhXlw_PyHH3z4gjII@g9~(K zW%R8H%e!o|Se7tJW`s<20RRi@g>Yak_iMN*-x5u8W75r!+yyn)av?hrCGo^>eZh!4 zc8iJK6bPhnxR*%LshAcxoY)!oJ(mvdD48bqOv^h!p`X7%kNTB|hNVBGSxwMiMQ%ST z?n!CY+$3_L`XMqx&zNIodqw$Z-5GFjbHta_O#RJaP5%$+wx+T@ZcN)pJG5Dp42f;8 zFX?Fhl3ksC8JXg-A&$XY0zJ$L9LnqtwBz^}Ws4s^Pc8}gkV;kyCaJN8NBNr6Eqdo3 zpJ$s>rWc!8$=ZNT=7&i2gghurx8^aC;Dad{ap1i3JF44hfEiGwo$Z4#5?nIJaK=L``M4fg$pbrhsIXB z!9m*75KIBtbb~^MCKbeMGRT7?u1b$DZjZS=lZ-82b7xuwUN{Srt=r^bja+Tt$Axba zyE5^kKc!Y5uKk1rO=xTkv#dE9GvtM8mU~ElQ#5F%y5rS140}FEewMOwQQ1mq=SbknU0`@Ohd&nT)azz`8`+_^Bu? zl&uHT1@zo?NbF%(-_{}4`qPdz_I)zR-bdPdp8UifZGFODnf16hZq`Ft$Y6KD$sQ*M zp*5|%pXjD9s|s7(IG@c-AEF-yv=OyT%tJuJGAxY(w;4V!@Q8A9=#{IwKhIDPPGsN> zj4oZ6|A|0k&as}rr{I_|(u(IjFLMIyJxQl)WdoY(#!e>b-DgBV*3$#38^yuW+vKtz z7fHoC6a0x(vAK{h5>Mi}DRKUQ$-ZUM%ERo~ANJNpjHukHJj?5;G^wy$-b@jTT^_cd zPM6Jl%#HUqzQQ3CO1e7jMQg>yl+NLks$T}mJ0GT%(qA`&wsyEB&8>8h-P&j|(LSpZ z{buk<3#h{|xEHlAf2nR#QAN)iA2P$;mLXowL~q9HEY16 z>JGMYx)dc`TeNG`H{hZO5&^Wsg@%TJiM(%l8&uZ`eG^2Vco z-m8tx(KA}2_V}mMSZIkXqde8+_Pa1q=*iHT)K`SJxUyq0y!Tp_ih_iE=bNl#dKmMe zC#^SwFk=n5DDT)fIKW3=Z$4;KQwRn5LiXC9+u=Q2X3l{OYmM|C0#T+tDxQfg7}u~a z0)H0CwS|g>b)@)56cklVw~7)9V-qzn)3$t32$^4uQ$pAyUxc89Ij64_Cu(BQaYpAd zyLkPsNzdVzTPuKaaufbo3tx6?0`?h`%7;F(xW?>!XjM8}2GnRuy==|D(OHjB2ZE+e{59Efg>ATBJ}cXrTm8AcW%bK#&#*PK#G)1H}qKiUtTS#kCY# zT#9>ecL?tE%kw-l@4VlS_nTSYtZ&VlHS;Iyth4WJ_qoqr`|LZ{b)_czJMRc*Zym)& z8gJrRxZ6-OV2Q+Dn7WWNv8?-I7(J7DvJh=_L3fVt+IOy&IrO|=HtLr&PHco&MDxk( zR+=xcUzubaLt1Y~C!#z$0v59S{H{Mcx@wI;1`<98DryP?KZ$m+WBHAelz{d>VM&yG zQVKNh;8>mj{-W^H6iGVf?=d~CX$7j!O=_rn?V_|@n&edTa>0b`2-wQ~WG=@H>oxWk z>5q;Bm>Acew*EF^457`q5*(88M0w=cXH`E1ObpbGH?s3S#mn^gUxqmuDA&b)=fw_B z>3I9Su8u@~7-~!}g3x}VDEO98RN^)0mE#mDxx=JR1Cp4EM=egBoS$B?$@CksIYDej z(Qz4%1KE#;%<7}blD)h^GDns3E0alIj!Vct+S{v&99uF(MSR{Kmv3hc^>zVVNs(MwC z0R`D}Qon%lA7wdCxdrsU0mN$(CBLj$K8JpHp2qS9s?WHeg@na_q-hl|Ks;MPWMj+i z^u^Vc$AQ4kE?Vu<%Jp&ExU|dV(&VfiD4B;*nYj9 z2)Z&ZsjjSt#!AdEOm`W2Pj80V#ZzW)1ytlgHfA=GOX(`<)Uy*EGUh*`DPEL42lEBE zYvXs1LCTsM^t_)cz-7KvLlzdyld6H(me;`5Myl_fT`h~RVtZsOpWR)%Oejn?cHnsY z>{y>bglIvZvp62m~BS{RFr z(TPq1dUcDM^j=)8Vs;74qbix<=^8E!(t*39*8TBNg9m@+2!1yB!i2q;xb`O|f}y|{gh zMb&uhAn`#S?;>)#|EwBX7khUwUEQ*MVb^mW>n%3@ghhR58;*yEx4svprea7-C61z? ztRm~r!=w)PHWyC_6aGOXkSYiFF9a}qK6QNX z(KV#LjDh8dXpEb}CaEL}LX3TGU>Y#v$}wCz-lHgufV~vWqm#XVSV8TI1~o>$H$v*+T&Pl$#Roy&z3qWH9x zv-kk(8DH$xn+VlT{Y8^V1(Yr95yn*F0x@?q%8INrNIFY&I+8SpQJd6OFPoXYUADaq z_d<}2;$97F3o6TeKCR89vMO;a*i38+AEI&^-AX|`*(gX~^18`^faSx2-CM8Zj}|D` zR|uu-zN>d^jp67OiOOu_!NC``Z?clX z8ON7$>&92tK@02gi73T^BG^JhE3&yfWNjWQAQ?LxzHN4I#|!#hUFXS~PdIG6xn&WL zJ>!Tpp+ui7o&{=>R%2su?XQZ)7+r)-LPM*7^AU^|n!%1+MK31@_2xUh4h|DUQO-37 zhOjWJ**cf$Bs~Kke+X%!;e&4YN2X&r+~yB>EA;P^wBz=WTSz9*Ux>K(V%a7uzMy|F zvmY$1{*8$<2YLUEi5vPSOx(eI#$TAYze`TuJDH_I)@~)?fp33v5kRjwj_DuzZ7Sel zaUA!gaJM{5o5^ZqO&^`sD>OA0NUpH-qOE$6W@EH73uf{#eNa60U&sI-qSUg{Om?4Mon zIZ9z2tj;lQlhY~?f1S$5ujRWX^3ajGzG!|B91jkou?=zIGkwGqGP7U-iWssBxuS^> zE)l(aG@dMSR&ZV(V0pQ|FWd(@F2AmW_+&qawq{mqkldq(Gu%(+J zWOp|nb-N=glik;kc|uo;H+n_S#Te_l*B7EhH{|xl@iN zV%Byx?lY2&)a}&;-r~uh>keM*{V@PQrD>n~-b={`e<>?G3!MA2`#qo_13RoN07Jz~cx8MGJ zlf8r0I+jmQZF$fwso|lpp1StK_!5@hT(=K+EKnhd%^j|995dBkqs-&bp0xBt>P)zt zFV$m-qEyUg%lE)4*>WSbl}Gs%#=Dk~+G8XKb=Aw{8C1e``p5>7*I7&(+nj$d=Oel@ z%=kE_K%75R&`oywyF_%G3w&f$n{8(y7h}{re8dQIw_h_Z2FlPf=O;d-e0j1URU^wF zko81U&Frq1CGWF=T=D`b<-`Z!44CqPbVip=o~_#hBLuKEo^ANq2!S>uo>J%s++fft zU)9SMx3^}Y@c_y7E)Vxr*58fu1;2Qh$(D_o{HW5y#3A{H_L_i8Gpq{Z@Rcxq$M{}C zl6Aj?d`$8)0#}1Nk2?F@Bv^ig>`TTq)C+Aj896AItd;qwtk@Nki9w#Z?ysD%6RwV) zBKJnf<8$CeCnbgJf&#J`lFG(>q#Uys;(OzA<2~OZ<#B1PUzlExo6ArC2M(SaB7w0u z2BoUT3bkGJ>uU~uSpn@np44vJvgBp@Hizq14B)ESzsuO2Dgwm~PZeI$4K$k79MS;* zL{M(&2jhNY%a>)YRI5tk`jS=c9lCm^@A0}9gk2CD8pHLxhY_+zQ9n}L`H&8wTOr0Q z%;klFC#Z<|?pCCskI9rqxZx;Ls=k)zUvf6`>%cappR#Bo!0NUF$Fl8@ zoQNPGB&`R`F9jhaQW^gX=k;stBu6NfO-~BAe;|lJ$j#+Bc;&n@>eA~LQ32`s7K(B; zfBuZzJw?g?24EH`wiRA%c2$~e9_P1mo^Un@q`tNhuoMzpq&rM?!_YDf?~M(E&Sb0U zrVA76^MpPasX+(BC&@8Vp77u^NvwA93w&x#Q9iEX{K}PUta@eFjL(q7uXj*OGUew! zVZM#2?2_KT7EwTlSFub+JFF-z1Z5<1&4wu1oIkv5=)ZQvwHN&c^uKiWjCD7Do~0r( z9u?4?mDDHy(9P*=`$eZ8Y3)|yvdGVG~ zIg1T8&8LYp#B5n5-ShQPJ-FqX*Lq!=9kX+Q&H6p3p%u;^+HVS z%j~}6Z&Tw_?JhTmDE>sdiXje>JaV#rmSmBY!HbgKsK^8iv*J$~kjj)XX};=|Nk46W zWQ^RXgn~voaFf1OT}XjKWL4sHQI4YTiBNf#17mwJ<;^Yv{aIZ}($LrT-ud{yHYg41 z<~ypFH-X{X$1Si%ty7kS`Ok`UL*4$0@F%ovQfCc=NuRx5381$wZYIG`SoB#Ny@6k? zW{l{~lV0pSUxLp^wZh4n7e^N!d<`(2rYYH8`{J{J;`H0(+WV{vMs{zo6^Xvk4pjO6 z8vp?BNH}X1l#{ptXVrHFWMu_-Ca;zZy}9y?XAp%=nbzogb{}DV&QlI6@^NB{T+DO_ z6Oy68u__~fG!Kq=jW8TC0CRo4N{7 zUuEX=I?vdnks)hyUM@F9HhsBA~U_R$NJ`ttvy?69#b%`O5N~0CwGwt4U>%-Uk5M> z;%+UOJ-zZQUY&2kxAx%0Z-DN_goUs3b@>T9`tpm<>ZbnCG^A09g`2tde5B$8R@@)@ zc3&thpl<8{H}Cw0g?)o?vnWu3l;-pX?!D@HukRR5qpRm}9)+=b6f{st8KcjVxC`~J}-vJ?3=pwdZhc=M2b z&M!2VY4SD0v*f?L0=*bNsrn7r zRB*$??>j#hHB^TA<41F2-?G5D2+raueFbQI!?$~lIuiaKds20QH{@mcS9Z+50qY97 zf42Dl?f%(+|404u*AxDqKMxk@Zvc#M*ApLQ0ZTdxsQV`V4i7KzFW9Z6=6?iPAON40 zAC$-7!|{K?aNrtqQr2)h>uRhwfxO!G@mJUd8hA<^U8G(g;7NMD}pll)ZrzuWvA&^@gV=nwn_*a55`{x=K4f7H2Etkc4w^HbhxS+sC^ zfd{0-(8w0j8ayUsb}F~y10CYYD+L*gog%(T2Lx1O?$y7lw>*FBJ&d6b2YT?nOIX$T_zgO|{m99F zjk}_JIW21`8?_X6SBaWMO)}G-X}i4tz-f%CIQ(;l*wN3EKgiB>Sqj8?zLsxdYSDCrK-YB ze-09s_K1gM;s(4$Ih1L0r}MuS5dM8fWDOLV&3JZSEZJF5Go><6KCM%EtUDh)p}X?_cuUA*;2Xha~%!CXT|VBnjbNMPMm#xs55!UVPW z05HeEf9;0jU$#!Z+l%|b)!(>Zvf588-l^?jgE7l7*Da~Ccd_E$5$G{As5VjH#bKL)?JA% z9wjuUm%INlw*%thg$<$9+=76%L9__)>zs*@_Sy#MfOvwPjdE7Hq9B``O-Y(0I~c%TLn=#G!&ABQqF&u=P2Qskx{;u$ps%2wZhN-vOA`rzIs! z?_>6`WkI#0t=197;3U{7p#t;jG}9X2>3M`eF&r)a+4d_#gvnPBC2TijlpfpbWnKu; zDvZsj!X1Ak$o*1#q{OQ#k;_8^r)1-K&#}$iEm+J{yQghFf3!^&IYfQ`?sa}y0(IY( z9O=0bDR*+=9~0cYA5mt9AgRtlULL-Vc=$lm5CpMjBMzu{e@a3gPbb2zHD+6N<<&;Ezp0PztKHo zC{RoQ#`#*GL!Dt=CKL`3cI%y(=&)r$4{0-|Y$?X`LoaufVTTl5=5ss?=77q1NA9AS z&Q@YZXX(gQnZGj^5JO1Uo{Aq)${$A1tF-;N#RbPk10Al0H81hs7e z;ZOe`G>KsF8`OqyJ~F#d{B(4Wa4b`8mL{j1ImlY~@fNw`R2?}2d|P<%#|`XXiw$+N z#1wbc0r!|OIz3-Pe0UkA+a@T~C7qbV_)9ocPdjac1>T+$STsh-m0jytOkIb}dsrS9 z?jSK7G2p=8v1L)sc45dLa?%l4#Y^g)o4EgNzNfrUaNBM4O>SLG4+F7yIxLo(Z+{SB z^$yuQ5fR(+iybI8Z=Pz5n^MlOvJrTqXWZ4v*MCeI?UYj76*0mr$KbT@MRjuWgO=cG z^&PrAT7~G(4 zO!S#C+9WFQSY8amXayM75`lW%I52@0CXB?bW1DY;JhF7V==l{7a@4X1d67l~IE7zJ zY~pAO><{nJ?5&}bDs1y8RRzZA6C%xCwRHD$dufgG6=qJ%4zgwxIgLxdaea@Y(0ZNdF%O(sfSo)!#w7DNe>i5=EiM=9}fU&_HNZ(8vk7h;M_%XE% zch6yNMTKTPa9~ZR8e>ij{Tu|80o?NGYyjCV8;Pq~$25DKSGzL$@Xp$`S7}1NqZJx) zDiOQ&VbA%jo;Yb3C1TUfJC*eg@!FC468^7x7M7;fvei`zS&~=fgF4?a1OYn|hUw&b z=FT&|*j|XnUDc1Y3DT+MdsC?I*AMPqeZ6RM6pvJekT_ft+rVqW;C*P2732aTOu=;gKd{ zpN#i7x0Xb{5u7SfOx;xEpPy;3l4%+q*;w;ds=iNU?v|gjS7P{atBAyFlzzjZ8)8}V z1+{YTb4pmE)p04k;!CJUq>>6&I{#~4y686TmopzodVsels!g6V+-q;3*L-vI{d1-r z#tVpDfONI~azh18u|$6re@R|xw};*~Ysb-T+5CNVSgzCwlO)&0{Hc6jUF*ChmOmoS;_=*npPQJeef6_eTH_gdjUZ3 z2#hr;JfRE3>`<9VYQY_(g18c^Jdt)!Ln+PQhX4$Oh-TAKmcIcZBIw{BhW-C?Gx1+u zpeK*dm$MBU>|MuQmJFNxyyNoLMnx5v;Nh;)Je;fRemf>I9?o0NY?|bmAPxR0jwwvU zjcR8DQd9{2L>HB8b(2-t8#4LM8^rUIJQ%u4!2{&5X6|}P?PHMd>5Mz-0#j6cE{Ycn zA#6KHMo%+21rEet^WKm#xy+w5F>3VI`;X15wPjRn)mFGm& z#NyEAH{cV%WQ5|Ut`*E(83bOk>WAW<%j!L^#$YfGpx}y0J*y#h(Hyqpph?P#04~WE zg@0FfCHHH?8fj4bCKg^EFS3KA*bogAuozR&bemata3Z^WuG$ zvs<21k+_I330SA1kP)P$#KYSYW6WC}pu|(oO+pzOe5pLO8O*Wh-K)J5QPsHC{To2V z!qDp$=;4X~rc$vuO}ky7;^H@c;jOLxtjRc$LP#QCHM+an>A|O$9UR5xJz7$A>znNf z*hRX*QT=FF7fbsrzM5EmY0vbPy9?iDHRI?&`q$|)9uMd88^2~sDkMd+iB&Twvu#ZQBKy*+wnZz zp%U>M8F7_w4pEnpII%0WpgONsjf3Z5_L`%Y8>R zQ{t9D;i9ocIw`dacetKY$U_T;D+6sF|-`P!V24hN8}3rK})DN9C#+l~U?EB*25 eeqx564#ENu#6xqZd7Dh?(j literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_msa.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_msa.sh new file mode 100755 index 00000000..3939b0d1 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_msa.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# inputs +in_fasta="$1" +out_dir="$2" + +# resources +CPU="$3" +MEM="$4" + +# sequence database +DB="$5/UniRef30_2020_06/UniRef30_2020_06" + +# setup hhblits command +HHBLITS="hhblits -o /dev/null -mact 0.35 -maxfilt 100000000 -neffmax 20 -cov 25 -cpu $CPU -nodiff -realign_max 100000000 -maxseq 1000000 -maxmem $MEM -n 4 -d $DB" +echo $HHBLITS + +mkdir -p $out_dir/hhblits +tmp_dir="$out_dir/hhblits" +out_prefix="$out_dir/t000_" + +# perform iterative searches +prev_a3m="$in_fasta" +for e in 1e-30 1e-10 1e-6 1e-3 +do + echo $e + $HHBLITS -i $prev_a3m -oa3m $tmp_dir/t000_.$e.a3m -e $e -v 0 + hhfilter -id 90 -cov 75 -i $tmp_dir/t000_.$e.a3m -o $tmp_dir/t000_.$e.id90cov75.a3m + hhfilter -id 90 -cov 50 -i $tmp_dir/t000_.$e.a3m -o $tmp_dir/t000_.$e.id90cov50.a3m + prev_a3m="$tmp_dir/t000_.$e.id90cov50.a3m" + n75=`grep -c "^>" $tmp_dir/t000_.$e.id90cov75.a3m` + n50=`grep -c "^>" $tmp_dir/t000_.$e.id90cov50.a3m` + + if ((n75>2000)) + then + if [ ! -s ${out_prefix}.msa0.a3m ] + then + cp $tmp_dir/t000_.$e.id90cov75.a3m ${out_prefix}.msa0.a3m + break + fi + elif ((n50>4000)) + then + if [ ! -s ${out_prefix}.msa0.a3m ] + then + cp $tmp_dir/t000_.$e.id90cov50.a3m ${out_prefix}.msa0.a3m + break + fi + else + continue + fi + +done + +if [ ! -s ${out_prefix}.msa0.a3m ] +then + cp $tmp_dir/t000_.1e-3.id90cov50.a3m ${out_prefix}.msa0.a3m +fi diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_ss.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_ss.sh new file mode 100755 index 00000000..9a4a12ca --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/make_ss.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +DATADIR="/workspace/psipred/data/" +echo $DATADIR + +i_a3m="$1" +o_ss="$2" + +ID=$(basename $i_a3m .a3m).tmp + +/workspace/csblast-2.2.3/bin/csbuild -i $i_a3m -I a3m -D /workspace/csblast-2.2.3/data/K4000.crf -o $ID.chk -O chk + +head -n 2 $i_a3m > $ID.fasta +echo $ID.chk > $ID.pn +echo $ID.fasta > $ID.sn + +/workspace/blast-2.2.26/bin/makemat -P $ID +/workspace/psipred/bin/psipred $ID.mtx $DATADIR/weights.dat $DATADIR/weights.dat2 $DATADIR/weights.dat3 > $ID.ss +/workspace/psipred/bin/psipass2 $DATADIR/weights_p2.dat 1 1.0 1.0 $i_a3m.csb.hhblits.ss2 $ID.ss > $ID.horiz + +( +echo ">ss_pred" +grep "^Pred" $ID.horiz | awk '{print $2}' +echo ">ss_conf" +grep "^Conf" $ID.horiz | awk '{print $2}' +) | awk '{if(substr($1,1,1)==">") {print "\n"$1} else {printf "%s", $1}} END {print ""}' | sed "1d" > $o_ss + +rm ${i_a3m}.csb.hhblits.ss2 +rm $ID.* diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/prepare_templates.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/prepare_templates.sh new file mode 100755 index 00000000..2d9ed557 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/input_prep/prepare_templates.sh @@ -0,0 +1,18 @@ +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +WDIR=$1 +CPU=$2 +MEM=$3 + +DB="$4/pdb100_2021Mar03/pdb100_2021Mar03" +HH="hhsearch -b 50 -B 500 -z 50 -Z 500 -mact 0.05 -cpu $CPU -maxmem $MEM -aliw 100000 -e 100 -p 5.0 -d $DB" +cat $WDIR/t000_.ss2 $WDIR/t000_.msa0.a3m > $WDIR/t000_.msa0.ss2.a3m +$HH -i $WDIR/t000_.msa0.ss2.a3m -o $WDIR/t000_.hhr -atab $WDIR/t000_.atab -v 0 + + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/install_dependencies.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/install_dependencies.sh new file mode 100755 index 00000000..c8f5d9ad --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/install_dependencies.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# install external program not supported by conda installation +case "$(uname -s)" in + Linux*) platform=linux;; + Darwin*) platform=macosx;; + *) echo "unsupported OS type. exiting"; exit 1 +esac +echo "installing for ${platform}" + +# download lddt +echo "downloading lddt . . ." +wget https://openstructure.org/static/lddt-${platform}.zip -O lddt.zip +unzip -d lddt -j lddt.zip + +# the cs-blast platform descriptoin includes the width of memory addresses +# we expect a 64-bit operating system +if [[ ${platform} == "linux" ]]; then + platform=${platform}64 +fi + +# download cs-blast +echo "downloading cs-blast . . ." +wget http://wwwuser.gwdg.de/~compbiol/data/csblast/releases/csblast-2.2.3_${platform}.tar.gz -O csblast-2.2.3.tar.gz +mkdir -p csblast-2.2.3 +tar xf csblast-2.2.3.tar.gz -C csblast-2.2.3 --strip-components=1 + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Attention_module_w_str.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Attention_module_w_str.py new file mode 100644 index 00000000..dc8cd6ff --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Attention_module_w_str.py @@ -0,0 +1,480 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from Transformer import * +from Transformer import _get_clones +import Transformer +from resnet import ResidualNetwork +from SE3_network import SE3Transformer +from InitStrGenerator import InitStr_Network +import dgl +# Attention module based on AlphaFold2's idea written by Minkyung Baek +# - Iterative MSA feature extraction +# - 1) MSA2Pair: extract pairwise feature from MSA --> added to previous residue-pair features +# architecture design inspired by CopulaNet paper +# - 2) MSA2MSA: process MSA features using Transformer (or Performer) encoder. (Attention over L first followed by attention over N) +# - 3) Pair2MSA: Update MSA features using pair feature +# - 4) Pair2Pair: process pair features using Transformer (or Performer) encoder. + +def make_graph(xyz, pair, idx, top_k=64, kmin=9): + ''' + Input: + - xyz: current backbone cooordinates (B, L, 3, 3) + - pair: pair features from Trunk (B, L, L, E) + - idx: residue index from ground truth pdb + Output: + - G: defined graph + ''' + + B, L = xyz.shape[:2] + device = xyz.device + + # distance map from current CA coordinates + D = torch.cdist(xyz[:,:,1,:], xyz[:,:,1,:]) + torch.eye(L, device=device).unsqueeze(0)*999.9 # (B, L, L) + # seq sep + sep = idx[:,None,:] - idx[:,:,None] + sep = sep.abs() + torch.eye(L, device=device).unsqueeze(0)*999.9 + + # get top_k neighbors + D_neigh, E_idx = torch.topk(D, min(top_k, L), largest=False) # shape of E_idx: (B, L, top_k) + topk_matrix = torch.zeros((B, L, L), device=device) + topk_matrix.scatter_(2, E_idx, 1.0) + + # put an edge if any of the 3 conditions are met: + # 1) |i-j| <= kmin (connect sequentially adjacent residues) + # 2) top_k neighbors + cond = torch.logical_or(topk_matrix > 0.0, sep < kmin) + b,i,j = torch.where(cond) + + src = b*L+i + tgt = b*L+j + G = dgl.graph((src, tgt), num_nodes=B*L).to(device) + G.edata['d'] = (xyz[b,j,1,:] - xyz[b,i,1,:]).detach() # no gradient through basis function + G.edata['w'] = pair[b,i,j] + + return G + +def get_bonded_neigh(idx): + ''' + Input: + - idx: residue indices of given sequence (B,L) + Output: + - neighbor: bonded neighbor information with sign (B, L, L, 1) + ''' + neighbor = idx[:,None,:] - idx[:,:,None] + neighbor = neighbor.float() + sign = torch.sign(neighbor) # (B, L, L) + neighbor = torch.abs(neighbor) + neighbor[neighbor > 1] = 0.0 + neighbor = sign * neighbor + return neighbor.unsqueeze(-1) + +def rbf(D): + # Distance radial basis function + D_min, D_max, D_count = 0., 20., 36 + D_mu = torch.linspace(D_min, D_max, D_count).to(D.device) + D_mu = D_mu[None,:] + D_sigma = (D_max - D_min) / D_count + D_expand = torch.unsqueeze(D, -1) + RBF = torch.exp(-((D_expand - D_mu) / D_sigma)**2) + return RBF + +class CoevolExtractor(nn.Module): + def __init__(self, n_feat_proj, n_feat_out, p_drop=0.1): + super(CoevolExtractor, self).__init__() + + self.norm_2d = LayerNorm(n_feat_proj*n_feat_proj) + # project down to output dimension (pair feature dimension) + self.proj_2 = nn.Linear(n_feat_proj**2, n_feat_out) + def forward(self, x_down, x_down_w): + B, N, L = x_down.shape[:3] + + pair = torch.einsum('abij,ablm->ailjm', x_down, x_down_w) # outer-product & average pool + pair = pair.reshape(B, L, L, -1) + pair = self.norm_2d(pair) + pair = self.proj_2(pair) # (B, L, L, n_feat_out) # project down to pair dimension + return pair + +class MSA2Pair(nn.Module): + def __init__(self, n_feat=64, n_feat_out=128, n_feat_proj=32, + n_resblock=1, p_drop=0.1, n_att_head=8): + super(MSA2Pair, self).__init__() + # project down embedding dimension (n_feat --> n_feat_proj) + self.norm_1 = LayerNorm(n_feat) + self.proj_1 = nn.Linear(n_feat, n_feat_proj) + + self.encoder = SequenceWeight(n_feat_proj, 1, dropout=p_drop) + self.coevol = CoevolExtractor(n_feat_proj, n_feat_out) + + # ResNet to update pair features + self.norm_down = LayerNorm(n_feat_proj) + self.norm_orig = LayerNorm(n_feat_out) + self.norm_new = LayerNorm(n_feat_out) + self.update = ResidualNetwork(n_resblock, n_feat_out*2+n_feat_proj*4+n_att_head, n_feat_out, n_feat_out, p_drop=p_drop) + + def forward(self, msa, pair_orig, att): + # Input: MSA embeddings (B, N, L, K), original pair embeddings (B, L, L, C) + # Output: updated pair info (B, L, L, C) + B, N, L, _ = msa.shape + # project down to reduce memory + msa = self.norm_1(msa) + x_down = self.proj_1(msa) # (B, N, L, n_feat_proj) + + # get sequence weight + x_down = self.norm_down(x_down) + w_seq = self.encoder(x_down).reshape(B, L, 1, N).permute(0,3,1,2) + feat_1d = w_seq*x_down + + pair = self.coevol(x_down, feat_1d) + + # average pooling over N of given MSA info + feat_1d = feat_1d.sum(1) + + # query sequence info + query = x_down[:,0] # (B,L,K) + feat_1d = torch.cat((feat_1d, query), dim=-1) # additional 1D features + # tile 1D features + left = feat_1d.unsqueeze(2).repeat(1, 1, L, 1) + right = feat_1d.unsqueeze(1).repeat(1, L, 1, 1) + # update original pair features through convolutions after concat + pair_orig = self.norm_orig(pair_orig) + pair = self.norm_new(pair) + pair = torch.cat((pair_orig, pair, left, right, att), -1) + pair = pair.permute(0,3,1,2).contiguous() # prep for convolution layer + pair = self.update(pair) + pair = pair.permute(0,2,3,1).contiguous() # (B, L, L, C) + + return pair + +class MSA2MSA(nn.Module): + def __init__(self, n_layer=1, n_att_head=8, n_feat=256, r_ff=4, p_drop=0.1, + performer_N_opts=None, performer_L_opts=None): + super(MSA2MSA, self).__init__() + # attention along L + enc_layer_1 = EncoderLayer(d_model=n_feat, d_ff=n_feat*r_ff, + heads=n_att_head, p_drop=p_drop, + use_tied=True) + #performer_opts=performer_L_opts) + self.encoder_1 = Encoder(enc_layer_1, n_layer) + + # attention along N + enc_layer_2 = EncoderLayer(d_model=n_feat, d_ff=n_feat*r_ff, + heads=n_att_head, p_drop=p_drop, + performer_opts=performer_N_opts) + self.encoder_2 = Encoder(enc_layer_2, n_layer) + + def forward(self, x): + # Input: MSA embeddings (B, N, L, K) + # Output: updated MSA embeddings (B, N, L, K) + B, N, L, _ = x.shape + # attention along L + x, att = self.encoder_1(x, return_att=True) + # attention along N + x = x.permute(0,2,1,3).contiguous() + x = self.encoder_2(x) + x = x.permute(0,2,1,3).contiguous() + return x, att + +class Pair2MSA(nn.Module): + def __init__(self, n_layer=1, n_att_head=4, n_feat_in=128, n_feat_out=256, r_ff=4, p_drop=0.1): + super(Pair2MSA, self).__init__() + enc_layer = DirectEncoderLayer(heads=n_att_head, \ + d_in=n_feat_in, d_out=n_feat_out,\ + d_ff=n_feat_out*r_ff,\ + p_drop=p_drop) + self.encoder = CrossEncoder(enc_layer, n_layer) + + def forward(self, pair, msa): + out = self.encoder(pair, msa) # (B, N, L, K) + return out + +class Pair2Pair(nn.Module): + def __init__(self, n_layer=1, n_att_head=8, n_feat=128, r_ff=4, p_drop=0.1, + performer_L_opts=None): + super(Pair2Pair, self).__init__() + enc_layer = AxialEncoderLayer(d_model=n_feat, d_ff=n_feat*r_ff, + heads=n_att_head, p_drop=p_drop, + performer_opts=performer_L_opts) + self.encoder = Encoder(enc_layer, n_layer) + + def forward(self, x): + return self.encoder(x) + +class Str2Str(nn.Module): + def __init__(self, d_msa=64, d_pair=128, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}, p_drop=0.1): + super(Str2Str, self).__init__() + + # initial node & pair feature process + self.norm_msa = LayerNorm(d_msa) + self.norm_pair = LayerNorm(d_pair) + self.encoder_seq = SequenceWeight(d_msa, 1, dropout=p_drop) + + self.embed_x = nn.Linear(d_msa+21, SE3_param['l0_in_features']) + self.embed_e = nn.Linear(d_pair, SE3_param['num_edge_features']) + + self.norm_node = LayerNorm(SE3_param['l0_in_features']) + self.norm_edge = LayerNorm(SE3_param['num_edge_features']) + + self.se3 = SE3Transformer(**SE3_param) + + @torch.cuda.amp.autocast(enabled=False) + def forward(self, msa, pair, xyz, seq1hot, idx, top_k=64): + # process msa & pair features + B, N, L = msa.shape[:3] + msa = self.norm_msa(msa) + pair = self.norm_pair(pair) + + w_seq = self.encoder_seq(msa).reshape(B, L, 1, N).permute(0,3,1,2) + msa = w_seq*msa + msa = msa.sum(dim=1) + msa = torch.cat((msa, seq1hot), dim=-1) + msa = self.norm_node(self.embed_x(msa)) + pair = self.norm_edge(self.embed_e(pair)) + + # define graph + G = make_graph(xyz, pair, idx, top_k=top_k) + l1_feats = xyz - xyz[:,:,1,:].unsqueeze(2) # l1 features = displacement vector to CA + l1_feats = l1_feats.reshape(B*L, -1, 3) + + # apply SE(3) Transformer & update coordinates + shift = self.se3(G, msa.reshape(B*L, -1, 1), l1_feats) + + state = shift['0'].reshape(B, L, -1) # (B, L, C) + + offset = shift['1'].reshape(B, L, -1, 3) # (B, L, 3, 3) + CA_new = xyz[:,:,1] + offset[:,:,1] + N_new = CA_new + offset[:,:,0] + C_new = CA_new + offset[:,:,2] + xyz_new = torch.stack([N_new, CA_new, C_new], dim=2) + + return xyz_new, state + +class Str2MSA(nn.Module): + def __init__(self, d_msa=64, d_state=32, inner_dim=32, r_ff=4, + distbin=[8.0, 12.0, 16.0, 20.0], p_drop=0.1): + super(Str2MSA, self).__init__() + self.distbin = distbin + n_att_head = len(distbin) + + self.norm_state = LayerNorm(d_state) + self.norm1 = LayerNorm(d_msa) + self.attn = MaskedDirectMultiheadAttention(d_state, d_msa, n_att_head, d_k=inner_dim, dropout=p_drop) + self.dropout1 = nn.Dropout(p_drop,inplace=True) + + self.norm2 = LayerNorm(d_msa) + self.ff = FeedForwardLayer(d_msa, d_msa*r_ff, p_drop=p_drop) + self.dropout2 = nn.Dropout(p_drop,inplace=True) + + def forward(self, msa, xyz, state): + dist = torch.cdist(xyz[:,:,1], xyz[:,:,1]) # (B, L, L) + + mask_s = list() + for distbin in self.distbin: + mask_s.append(1.0 - torch.sigmoid(dist-distbin)) + mask_s = torch.stack(mask_s, dim=1) # (B, h, L, L) + + state = self.norm_state(state) + msa2 = self.norm1(msa) + msa2 = self.attn(state, state, msa2, mask_s) + msa = msa + self.dropout1(msa2) + + msa2 = self.norm2(msa) + msa2 = self.ff(msa2) + msa = msa + self.dropout2(msa2) + + return msa + +class IterBlock(nn.Module): + def __init__(self, n_layer=1, d_msa=64, d_pair=128, n_head_msa=4, n_head_pair=8, r_ff=4, + n_resblock=1, p_drop=0.1, performer_L_opts=None, performer_N_opts=None): + super(IterBlock, self).__init__() + + self.msa2msa = MSA2MSA(n_layer=n_layer, n_att_head=n_head_msa, n_feat=d_msa, + r_ff=r_ff, p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts) + self.msa2pair = MSA2Pair(n_feat=d_msa, n_feat_out=d_pair, n_feat_proj=32, + n_resblock=n_resblock, p_drop=p_drop, n_att_head=n_head_msa) + self.pair2pair = Pair2Pair(n_layer=n_layer, n_att_head=n_head_pair, + n_feat=d_pair, r_ff=r_ff, p_drop=p_drop, + performer_L_opts=performer_L_opts) + self.pair2msa = Pair2MSA(n_layer=n_layer, n_att_head=4, + n_feat_in=d_pair, n_feat_out=d_msa, r_ff=r_ff, p_drop=p_drop) + + def forward(self, msa, pair): + # input: + # msa: initial MSA embeddings (N, L, d_msa) + # pair: initial residue pair embeddings (L, L, d_pair) + + # 1. process MSA features + msa, att = self.msa2msa(msa) + + # 2. update pair features using given MSA + pair = self.msa2pair(msa, pair, att) + + # 3. process pair features + pair = self.pair2pair(pair) + + # 4. update MSA features using updated pair features + msa = self.pair2msa(pair, msa) + + + return msa, pair + +class IterBlock_w_Str(nn.Module): + def __init__(self, n_layer=1, d_msa=64, d_pair=128, n_head_msa=4, n_head_pair=8, r_ff=4, + n_resblock=1, p_drop=0.1, performer_L_opts=None, performer_N_opts=None, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}): + super(IterBlock_w_Str, self).__init__() + + self.msa2msa = MSA2MSA(n_layer=n_layer, n_att_head=n_head_msa, n_feat=d_msa, + r_ff=r_ff, p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts) + self.msa2pair = MSA2Pair(n_feat=d_msa, n_feat_out=d_pair, n_feat_proj=32, + n_resblock=n_resblock, p_drop=p_drop, n_att_head=n_head_msa) + self.pair2pair = Pair2Pair(n_layer=n_layer, n_att_head=n_head_pair, + n_feat=d_pair, r_ff=r_ff, p_drop=p_drop, + performer_L_opts=performer_L_opts) + self.pair2msa = Pair2MSA(n_layer=n_layer, n_att_head=4, + n_feat_in=d_pair, n_feat_out=d_msa, r_ff=r_ff, p_drop=p_drop) + self.str2str = Str2Str(d_msa=d_msa, d_pair=d_pair, SE3_param=SE3_param, p_drop=p_drop) + self.str2msa = Str2MSA(d_msa=d_msa, d_state=SE3_param['l0_out_features'], + r_ff=r_ff, p_drop=p_drop) + + def forward(self, msa, pair, xyz, seq1hot, idx, top_k=64): + # input: + # msa: initial MSA embeddings (N, L, d_msa) + # pair: initial residue pair embeddings (L, L, d_pair) + + # 1. process MSA features + msa, att = self.msa2msa(msa) + + # 2. update pair features using given MSA + pair = self.msa2pair(msa, pair, att) + + # 3. process pair features + pair = self.pair2pair(pair) + + # 4. update MSA features using updated pair features + msa = self.pair2msa(pair, msa) + + xyz, state = self.str2str(msa.float(), pair.float(), xyz.float(), seq1hot, idx, top_k=top_k) + msa = self.str2msa(msa, xyz, state) + + return msa, pair, xyz + +class FinalBlock(nn.Module): + def __init__(self, n_layer=1, d_msa=64, d_pair=128, n_head_msa=4, n_head_pair=8, r_ff=4, + n_resblock=1, p_drop=0.1, performer_L_opts=None, performer_N_opts=None, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}): + super(FinalBlock, self).__init__() + + self.msa2msa = MSA2MSA(n_layer=n_layer, n_att_head=n_head_msa, n_feat=d_msa, + r_ff=r_ff, p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts) + self.msa2pair = MSA2Pair(n_feat=d_msa, n_feat_out=d_pair, n_feat_proj=32, + n_resblock=n_resblock, p_drop=p_drop, n_att_head=n_head_msa) + self.pair2pair = Pair2Pair(n_layer=n_layer, n_att_head=n_head_pair, + n_feat=d_pair, r_ff=r_ff, p_drop=p_drop, + performer_L_opts=performer_L_opts) + self.pair2msa = Pair2MSA(n_layer=n_layer, n_att_head=4, + n_feat_in=d_pair, n_feat_out=d_msa, r_ff=r_ff, p_drop=p_drop) + self.str2str = Str2Str(d_msa=d_msa, d_pair=d_pair, SE3_param=SE3_param, p_drop=p_drop) + self.norm_state = LayerNorm(SE3_param['l0_out_features']) + self.pred_lddt = nn.Linear(SE3_param['l0_out_features'], 1) + + def forward(self, msa, pair, xyz, seq1hot, idx): + # input: + # msa: initial MSA embeddings (N, L, d_msa) + # pair: initial residue pair embeddings (L, L, d_pair) + + # 1. process MSA features + msa, att = self.msa2msa(msa) + + # 2. update pair features using given MSA + pair = self.msa2pair(msa, pair, att) + + # 3. process pair features + pair = self.pair2pair(pair) + + msa = self.pair2msa(pair, msa) + + xyz, state = self.str2str(msa.float(), pair.float(), xyz.float(), seq1hot, idx, top_k=32) + + lddt = self.pred_lddt(self.norm_state(state)) + return msa, pair, xyz, lddt.squeeze(-1) + +class IterativeFeatureExtractor(nn.Module): + def __init__(self, n_module=4, n_module_str=4, n_layer=4, d_msa=256, d_pair=128, d_hidden=64, + n_head_msa=8, n_head_pair=8, r_ff=4, + n_resblock=1, p_drop=0.1, + performer_L_opts=None, performer_N_opts=None, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}): + super(IterativeFeatureExtractor, self).__init__() + self.n_module = n_module + self.n_module_str = n_module_str + # + self.initial = Pair2Pair(n_layer=n_layer, n_att_head=n_head_pair, + n_feat=d_pair, r_ff=r_ff, p_drop=p_drop, + performer_L_opts=performer_L_opts) + + if self.n_module > 0: + self.iter_block_1 = _get_clones(IterBlock(n_layer=n_layer, + d_msa=d_msa, d_pair=d_pair, + n_head_msa=n_head_msa, + n_head_pair=n_head_pair, + r_ff=r_ff, + n_resblock=n_resblock, + p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts + ), n_module) + + self.init_str = InitStr_Network(node_dim_in=d_msa, node_dim_hidden=d_hidden, + edge_dim_in=d_pair, edge_dim_hidden=d_hidden, + nheads=4, nblocks=3, dropout=p_drop) + + if self.n_module_str > 0: + self.iter_block_2 = _get_clones(IterBlock_w_Str(n_layer=n_layer, + d_msa=d_msa, d_pair=d_pair, + n_head_msa=n_head_msa, + n_head_pair=n_head_pair, + r_ff=r_ff, + n_resblock=n_resblock, + p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts, + SE3_param=SE3_param + ), n_module_str) + + self.final = FinalBlock(n_layer=n_layer, d_msa=d_msa, d_pair=d_pair, + n_head_msa=n_head_msa, n_head_pair=n_head_pair, r_ff=r_ff, + n_resblock=n_resblock, p_drop=p_drop, + performer_L_opts=performer_L_opts, performer_N_opts=performer_N_opts, + SE3_param=SE3_param) + + def forward(self, msa, pair, seq1hot, idx): + # input: + # msa: initial MSA embeddings (N, L, d_msa) + # pair: initial residue pair embeddings (L, L, d_pair) + + pair_s = list() + pair = self.initial(pair) + if self.n_module > 0: + for i_m in range(self.n_module): + # extract features from MSA & update original pair features + msa, pair = self.iter_block_1[i_m](msa, pair) + + xyz = self.init_str(seq1hot, idx, msa, pair) + + top_ks = [128, 128, 64, 64] + if self.n_module_str > 0: + for i_m in range(self.n_module_str): + msa, pair, xyz = self.iter_block_2[i_m](msa, pair, xyz, seq1hot, idx, top_k=top_ks[i_m]) + + msa, pair, xyz, lddt = self.final(msa, pair, xyz, seq1hot, idx) + + return msa[:,0], pair, xyz, lddt diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/DistancePredictor.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/DistancePredictor.py new file mode 100644 index 00000000..7ab08e96 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/DistancePredictor.py @@ -0,0 +1,36 @@ +import torch +import torch.nn as nn +from resnet import ResidualNetwork +from Transformer import LayerNorm + +# predict distance map from pair features +# based on simple 2D ResNet + +class DistanceNetwork(nn.Module): + def __init__(self, n_feat, n_block=1, block_type='orig', p_drop=0.0): + super(DistanceNetwork, self).__init__() + self.norm = LayerNorm(n_feat) + self.proj = nn.Linear(n_feat, n_feat) + self.drop = nn.Dropout(p_drop) + # + self.resnet_dist = ResidualNetwork(n_block, n_feat, n_feat, 37, block_type=block_type, p_drop=p_drop) + self.resnet_omega = ResidualNetwork(n_block, n_feat, n_feat, 37, block_type=block_type, p_drop=p_drop) + self.resnet_theta = ResidualNetwork(n_block, n_feat, n_feat, 37, block_type=block_type, p_drop=p_drop) + self.resnet_phi = ResidualNetwork(n_block, n_feat, n_feat, 19, block_type=block_type, p_drop=p_drop) + + def forward(self, x): + # input: pair info (B, L, L, C) + x = self.norm(x) + x = self.drop(self.proj(x)) + x = x.permute(0,3,1,2).contiguous() + + # predict theta, phi (non-symmetric) + logits_theta = self.resnet_theta(x) + logits_phi = self.resnet_phi(x) + + # predict dist, omega + x = 0.5 * (x + x.permute(0,1,3,2)) + logits_dist = self.resnet_dist(x) + logits_omega = self.resnet_omega(x) + + return logits_dist, logits_omega, logits_theta, logits_phi diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Embeddings.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Embeddings.py new file mode 100644 index 00000000..318b3ffc --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Embeddings.py @@ -0,0 +1,175 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import math +from Transformer import EncoderLayer, AxialEncoderLayer, Encoder, LayerNorm + +# Initial embeddings for target sequence, msa, template info +# positional encoding +# option 1: using sin/cos --> using this for now +# option 2: learn positional embedding + +class PositionalEncoding(nn.Module): + def __init__(self, d_model, p_drop=0.1, max_len=5000): + super(PositionalEncoding, self).__init__() + self.drop = nn.Dropout(p_drop,inplace=True) + + pe = torch.zeros(max_len, d_model) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp(torch.arange(0, d_model, 2) * + -(math.log(10000.0) / d_model)) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.register_buffer('pe', pe) # (1, max_len, d_model) + def forward(self, x, idx_s): + pe = list() + for idx in idx_s: + pe.append(self.pe[:,idx,:]) + pe = torch.stack(pe) + x = x + torch.autograd.Variable(pe, requires_grad=False) + return self.drop(x) + +class PositionalEncoding2D(nn.Module): + def __init__(self, d_model, p_drop=0.1): + super(PositionalEncoding2D, self).__init__() + self.drop = nn.Dropout(p_drop,inplace=True) + # + d_model_half = d_model // 2 + div_term = torch.exp(torch.arange(0., d_model_half, 2) * + -(math.log(10000.0) / d_model_half)) + self.register_buffer('div_term', div_term) + + def forward(self, x, idx_s): + B, L, _, K = x.shape + K_half = K//2 + pe = torch.zeros_like(x) + i_batch = -1 + for idx in idx_s: + i_batch += 1 + sin_inp = idx.unsqueeze(1) * self.div_term + emb = torch.cat((sin_inp.sin(), sin_inp.cos()), dim=-1) # (L, K//2) + pe[i_batch,:,:,:K_half] = emb.unsqueeze(1) + pe[i_batch,:,:,K_half:] = emb.unsqueeze(0) + x = x + torch.autograd.Variable(pe, requires_grad=False) + return self.drop(x) + +class QueryEncoding(nn.Module): + def __init__(self, d_model): + super(QueryEncoding, self).__init__() + self.pe = nn.Embedding(2, d_model) # (0 for query, 1 for others) + + def forward(self, x): + B, N, L, K = x.shape + idx = torch.ones((B, N, L), device=x.device).long() + idx[:,0,:] = 0 # first sequence is the query + x = x + self.pe(idx) + return x + +class MSA_emb(nn.Module): + def __init__(self, d_model=64, d_msa=21, p_drop=0.1, max_len=5000): + super(MSA_emb, self).__init__() + self.emb = nn.Embedding(d_msa, d_model) + self.pos = PositionalEncoding(d_model, p_drop=p_drop, max_len=max_len) + self.pos_q = QueryEncoding(d_model) + def forward(self, msa, idx): + B, N, L = msa.shape + out = self.emb(msa) # (B, N, L, K//2) + out = self.pos(out, idx) # add positional encoding + return self.pos_q(out) # add query encoding + +# pixel-wise attention based embedding (from trRosetta-tbm) +class Templ_emb(nn.Module): + def __init__(self, d_t1d=3, d_t2d=10, d_templ=64, n_att_head=4, r_ff=4, + performer_opts=None, p_drop=0.1, max_len=5000): + super(Templ_emb, self).__init__() + self.proj = nn.Linear(d_t1d*2+d_t2d+1, d_templ) + self.pos = PositionalEncoding2D(d_templ, p_drop=p_drop) + # attention along L + enc_layer_L = AxialEncoderLayer(d_templ, d_templ*r_ff, n_att_head, p_drop=p_drop, + performer_opts=performer_opts) + self.encoder_L = Encoder(enc_layer_L, 1) + + self.norm = LayerNorm(d_templ) + self.to_attn = nn.Linear(d_templ, 1) + + def forward(self, t1d, t2d, idx): + # Input + # - t1d: 1D template info (B, T, L, 2) + # - t2d: 2D template info (B, T, L, L, 10) + B, T, L, _ = t1d.shape + left = t1d.unsqueeze(3).expand(-1,-1,-1,L,-1) + right = t1d.unsqueeze(2).expand(-1,-1,L,-1,-1) + seqsep = torch.abs(idx[:,:,None]-idx[:,None,:]) + 1 + seqsep = torch.log(seqsep.float()).view(B,L,L,1).unsqueeze(1).expand(-1,T,-1,-1,-1) + # + feat = torch.cat((t2d, left, right, seqsep), -1) + feat = self.proj(feat).reshape(B*T, L, L, -1) + tmp = self.pos(feat, idx) # add positional embedding + # + # attention along L + feat = torch.empty_like(tmp) + for i_f in range(tmp.shape[0]): + feat[i_f] = self.encoder_L(tmp[i_f].view(1,L,L,-1)) + del tmp + feat = feat.reshape(B, T, L, L, -1) + feat = feat.permute(0,2,3,1,4).contiguous().reshape(B, L*L, T, -1) + + attn = self.to_attn(self.norm(feat)) + attn = F.softmax(attn, dim=-2) # (B, L*L, T, 1) + feat = torch.matmul(attn.transpose(-2, -1), feat) + return feat.reshape(B, L, L, -1) + +class Pair_emb_w_templ(nn.Module): + def __init__(self, d_model=128, d_seq=21, d_templ=64, p_drop=0.1): + super(Pair_emb_w_templ, self).__init__() + self.d_model = d_model + self.d_emb = d_model // 2 + self.emb = nn.Embedding(d_seq, self.d_emb) + self.norm_templ = LayerNorm(d_templ) + self.projection = nn.Linear(d_model + d_templ + 1, d_model) + self.pos = PositionalEncoding2D(d_model, p_drop=p_drop) + + def forward(self, seq, idx, templ): + # input: + # seq: target sequence (B, L, 20) + B = seq.shape[0] + L = seq.shape[1] + # + # get initial sequence pair features + seq = self.emb(seq) # (B, L, d_model//2) + left = seq.unsqueeze(2).expand(-1,-1,L,-1) + right = seq.unsqueeze(1).expand(-1,L,-1,-1) + seqsep = torch.abs(idx[:,:,None]-idx[:,None,:])+1 + seqsep = torch.log(seqsep.float()).view(B,L,L,1) + # + templ = self.norm_templ(templ) + pair = torch.cat((left, right, seqsep, templ), dim=-1) + pair = self.projection(pair) # (B, L, L, d_model) + + return self.pos(pair, idx) + +class Pair_emb_wo_templ(nn.Module): + #TODO: embedding without template info + def __init__(self, d_model=128, d_seq=21, p_drop=0.1): + super(Pair_emb_wo_templ, self).__init__() + self.d_model = d_model + self.d_emb = d_model // 2 + self.emb = nn.Embedding(d_seq, self.d_emb) + self.projection = nn.Linear(d_model + 1, d_model) + self.pos = PositionalEncoding2D(d_model, p_drop=p_drop) + def forward(self, seq, idx): + # input: + # seq: target sequence (B, L, 20) + B = seq.shape[0] + L = seq.shape[1] + seq = self.emb(seq) # (B, L, d_model//2) + left = seq.unsqueeze(2).expand(-1,-1,L,-1) + right = seq.unsqueeze(1).expand(-1,L,-1,-1) + seqsep = torch.abs(idx[:,:,None]-idx[:,None,:])+1 + seqsep = torch.log(seqsep.float()).view(B,L,L,1) + # + pair = torch.cat((left, right, seqsep), dim=-1) + pair = self.projection(pair) + return self.pos(pair, idx) + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/InitStrGenerator.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/InitStrGenerator.py new file mode 100644 index 00000000..a990d44e --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/InitStrGenerator.py @@ -0,0 +1,116 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from Transformer import LayerNorm, SequenceWeight + +import torch_geometric +from torch_geometric.data import Data +from torch_geometric.nn import TransformerConv + +def get_seqsep(idx): + ''' + Input: + - idx: residue indices of given sequence (B,L) + Output: + - seqsep: sequence separation feature with sign (B, L, L, 1) + Sergey found that having sign in seqsep features helps a little + ''' + seqsep = idx[:,None,:] - idx[:,:,None] + sign = torch.sign(seqsep) + seqsep = torch.log(torch.abs(seqsep) + 1.0) + seqsep = torch.clamp(seqsep, 0.0, 5.5) + seqsep = sign * seqsep + return seqsep.unsqueeze(-1) + +def make_graph(node, idx, emb): + ''' create torch_geometric graph from Trunk outputs ''' + device = emb.device + B, L = emb.shape[:2] + + # |i-j| <= kmin (connect sequentially adjacent residues) + sep = idx[:,None,:] - idx[:,:,None] + sep = sep.abs() + b, i, j = torch.where(sep > 0) + + src = b*L+i + tgt = b*L+j + + x = node.reshape(B*L, -1) + + G = Data(x=x, + edge_index=torch.stack([src,tgt]), + edge_attr=emb[b,i,j]) + + return G + +class UniMPBlock(nn.Module): + '''https://arxiv.org/pdf/2009.03509.pdf''' + def __init__(self, + node_dim=64, + edge_dim=64, + heads=4, + dropout=0.15): + super(UniMPBlock, self).__init__() + + self.TConv = TransformerConv(node_dim, node_dim, heads, dropout=dropout, edge_dim=edge_dim) + self.LNorm = LayerNorm(node_dim*heads) + self.Linear = nn.Linear(node_dim*heads, node_dim) + self.Activ = nn.ELU(inplace=True) + + #@torch.cuda.amp.autocast(enabled=True) + def forward(self, G): + xin, e_idx, e_attr = G.x, G.edge_index, G.edge_attr + x = self.TConv(xin, e_idx, e_attr) + x = self.LNorm(x) + x = self.Linear(x) + out = self.Activ(x+xin) + return Data(x=out, edge_index=e_idx, edge_attr=e_attr) + + +class InitStr_Network(nn.Module): + def __init__(self, + node_dim_in=64, + node_dim_hidden=64, + edge_dim_in=128, + edge_dim_hidden=64, + nheads=4, + nblocks=3, + dropout=0.1): + super(InitStr_Network, self).__init__() + + # embedding layers for node and edge features + self.norm_node = LayerNorm(node_dim_in) + self.norm_edge = LayerNorm(edge_dim_in) + self.encoder_seq = SequenceWeight(node_dim_in, 1, dropout=dropout) + + self.embed_x = nn.Sequential(nn.Linear(node_dim_in+21, node_dim_hidden), nn.ELU(inplace=True)) + self.embed_e = nn.Sequential(nn.Linear(edge_dim_in+1, edge_dim_hidden), nn.ELU(inplace=True)) + + # graph transformer + blocks = [UniMPBlock(node_dim_hidden,edge_dim_hidden,nheads,dropout) for _ in range(nblocks)] + self.transformer = nn.Sequential(*blocks) + + # outputs + self.get_xyz = nn.Linear(node_dim_hidden,9) + + def forward(self, seq1hot, idx, msa, pair): + B, N, L = msa.shape[:3] + msa = self.norm_node(msa) + pair = self.norm_edge(pair) + + w_seq = self.encoder_seq(msa).reshape(B, L, 1, N).permute(0,3,1,2) + msa = w_seq*msa + msa = msa.sum(dim=1) + node = torch.cat((msa, seq1hot), dim=-1) + node = self.embed_x(node) + + seqsep = get_seqsep(idx) + pair = torch.cat((pair, seqsep), dim=-1) + pair = self.embed_e(pair) + + G = make_graph(node, idx, pair) + Gout = self.transformer(G) + + xyz = self.get_xyz(Gout.x) + + return xyz.reshape(B, L, 3, 3) #torch.cat([xyz,node_emb],dim=-1) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Refine_module.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Refine_module.py new file mode 100644 index 00000000..258017f5 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Refine_module.py @@ -0,0 +1,175 @@ +import torch +import torch.nn as nn +import torch.utils.checkpoint as checkpoint +from Transformer import LayerNorm +from InitStrGenerator import make_graph +from InitStrGenerator import get_seqsep, UniMPBlock +from Attention_module_w_str import make_graph as make_graph_topk +from Attention_module_w_str import get_bonded_neigh, rbf +from SE3_network import SE3Transformer +from Transformer import _get_clones, create_custom_forward +# Re-generate initial coordinates based on 1) final pair features 2) predicted distogram +# Then, refine it through multiple SE3 transformer block + +class Regen_Network(nn.Module): + def __init__(self, + node_dim_in=64, + node_dim_hidden=64, + edge_dim_in=128, + edge_dim_hidden=64, + state_dim=8, + nheads=4, + nblocks=3, + dropout=0.0): + super(Regen_Network, self).__init__() + + # embedding layers for node and edge features + self.norm_node = LayerNorm(node_dim_in) + self.norm_edge = LayerNorm(edge_dim_in) + + self.embed_x = nn.Sequential(nn.Linear(node_dim_in+21, node_dim_hidden), LayerNorm(node_dim_hidden)) + self.embed_e = nn.Sequential(nn.Linear(edge_dim_in+2, edge_dim_hidden), LayerNorm(edge_dim_hidden)) + + # graph transformer + blocks = [UniMPBlock(node_dim_hidden,edge_dim_hidden,nheads,dropout) for _ in range(nblocks)] + self.transformer = nn.Sequential(*blocks) + + # outputs + self.get_xyz = nn.Linear(node_dim_hidden,9) + self.norm_state = LayerNorm(node_dim_hidden) + self.get_state = nn.Linear(node_dim_hidden, state_dim) + + def forward(self, seq1hot, idx, node, edge): + B, L = node.shape[:2] + node = self.norm_node(node) + edge = self.norm_edge(edge) + + node = torch.cat((node, seq1hot), dim=-1) + node = self.embed_x(node) + + seqsep = get_seqsep(idx) + neighbor = get_bonded_neigh(idx) + edge = torch.cat((edge, seqsep, neighbor), dim=-1) + edge = self.embed_e(edge) + + G = make_graph(node, idx, edge) + Gout = self.transformer(G) + + xyz = self.get_xyz(Gout.x) + state = self.get_state(self.norm_state(Gout.x)) + return xyz.reshape(B, L, 3, 3) , state.reshape(B, L, -1) + +class Refine_Network(nn.Module): + def __init__(self, d_node=64, d_pair=128, d_state=16, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}, p_drop=0.0): + super(Refine_Network, self).__init__() + self.norm_msa = LayerNorm(d_node) + self.norm_pair = LayerNorm(d_pair) + self.norm_state = LayerNorm(d_state) + + self.embed_x = nn.Linear(d_node+21+d_state, SE3_param['l0_in_features']) + self.embed_e1 = nn.Linear(d_pair, SE3_param['num_edge_features']) + self.embed_e2 = nn.Linear(SE3_param['num_edge_features']+36+1, SE3_param['num_edge_features']) + + self.norm_node = LayerNorm(SE3_param['l0_in_features']) + self.norm_edge1 = LayerNorm(SE3_param['num_edge_features']) + self.norm_edge2 = LayerNorm(SE3_param['num_edge_features']) + + self.se3 = SE3Transformer(**SE3_param) + + @torch.cuda.amp.autocast(enabled=False) + def forward(self, msa, pair, xyz, state, seq1hot, idx, top_k=64): + # process node & pair features + B, L = msa.shape[:2] + node = self.norm_msa(msa) + pair = self.norm_pair(pair) + state = self.norm_state(state) + + node = torch.cat((node, seq1hot, state), dim=-1) + node = self.norm_node(self.embed_x(node)) + pair = self.norm_edge1(self.embed_e1(pair)) + + neighbor = get_bonded_neigh(idx) + rbf_feat = rbf(torch.cdist(xyz[:,:,1,:], xyz[:,:,1,:])) + pair = torch.cat((pair, rbf_feat, neighbor), dim=-1) + pair = self.norm_edge2(self.embed_e2(pair)) + + # define graph + G = make_graph_topk(xyz, pair, idx, top_k=top_k) + l1_feats = xyz - xyz[:,:,1,:].unsqueeze(2) # l1 features = displacement vector to CA + l1_feats = l1_feats.reshape(B*L, -1, 3) + + # apply SE(3) Transformer & update coordinates + shift = self.se3(G, node.reshape(B*L, -1, 1), l1_feats) + + state = shift['0'].reshape(B, L, -1) # (B, L, C) + + offset = shift['1'].reshape(B, L, -1, 3) # (B, L, 3, 3) + CA_new = xyz[:,:,1] + offset[:,:,1] + N_new = CA_new + offset[:,:,0] + C_new = CA_new + offset[:,:,2] + xyz_new = torch.stack([N_new, CA_new, C_new], dim=2) + + return xyz_new, state + +class Refine_module(nn.Module): + def __init__(self, n_module, d_node=64, d_node_hidden=64, d_pair=128, d_pair_hidden=64, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}, p_drop=0.0): + super(Refine_module, self).__init__() + self.n_module = n_module + self.proj_edge = nn.Linear(d_pair, d_pair_hidden*2) + + self.regen_net = Regen_Network(node_dim_in=d_node, node_dim_hidden=d_node_hidden, + edge_dim_in=d_pair_hidden*2, edge_dim_hidden=d_pair_hidden, + state_dim=SE3_param['l0_out_features'], + nheads=4, nblocks=3, dropout=p_drop) + self.refine_net = _get_clones(Refine_Network(d_node=d_node, d_pair=d_pair_hidden*2, + d_state=SE3_param['l0_out_features'], + SE3_param=SE3_param, p_drop=p_drop), self.n_module) + self.norm_state = LayerNorm(SE3_param['l0_out_features']) + self.pred_lddt = nn.Linear(SE3_param['l0_out_features'], 1) + + def forward(self, node, edge, seq1hot, idx, use_transf_checkpoint=False, eps=1e-4): + edge = self.proj_edge(edge) + + xyz, state = self.regen_net(seq1hot, idx, node, edge) + + # DOUBLE IT w/ Mirror images + xyz = torch.cat([xyz, xyz*torch.tensor([1,1,-1], dtype=xyz.dtype, device=xyz.device)]) + state = torch.cat([state, state]) + node = torch.cat([node, node]) + edge = torch.cat([edge, edge]) + idx = torch.cat([idx, idx]) + seq1hot = torch.cat([seq1hot, seq1hot]) + + best_xyz = xyz + best_lddt = torch.zeros((xyz.shape[0], xyz.shape[1], 1), device=xyz.device) + prev_lddt = 0.0 + no_impr = 0 + no_impr_best = 0 + for i_iter in range(200): + for i_m in range(self.n_module): + if use_transf_checkpoint: + xyz, state = checkpoint.checkpoint(create_custom_forward(self.refine_net[i_m], top_k=64), node.float(), edge.float(), xyz.detach().float(), state.float(), seq1hot, idx) + else: + xyz, state = self.refine_net[i_m](node.float(), edge.float(), xyz.detach().float(), state.float(), seq1hot, idx, top_k=64) + # + lddt = self.pred_lddt(self.norm_state(state)) + lddt = torch.clamp(lddt, 0.0, 1.0)[...,0] + print (f"SE(3) iteration {i_iter} {lddt.mean(-1).cpu().numpy()}") + if lddt.mean(-1).max() <= prev_lddt+eps: + no_impr += 1 + else: + no_impr = 0 + if lddt.mean(-1).max() <= best_lddt.mean(-1).max()+eps: + no_impr_best += 1 + else: + no_impr_best = 0 + if no_impr > 10 or no_impr_best > 20: + break + if lddt.mean(-1).max() > best_lddt.mean(-1).max(): + best_lddt = lddt + best_xyz = xyz + prev_lddt = lddt.mean(-1).max() + pick = best_lddt.mean(-1).argmax() + return best_xyz[pick][None], best_lddt[pick][None] diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/RoseTTAFoldModel.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/RoseTTAFoldModel.py new file mode 100644 index 00000000..600d93f1 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/RoseTTAFoldModel.py @@ -0,0 +1,131 @@ +import torch +import torch.nn as nn +from Embeddings import MSA_emb, Pair_emb_wo_templ, Pair_emb_w_templ, Templ_emb +from Attention_module_w_str import IterativeFeatureExtractor +from DistancePredictor import DistanceNetwork +from Refine_module import Refine_module + +class RoseTTAFoldModule(nn.Module): + def __init__(self, n_module=4, n_module_str=4, n_layer=4,\ + d_msa=64, d_pair=128, d_templ=64,\ + n_head_msa=4, n_head_pair=8, n_head_templ=4, + d_hidden=64, r_ff=4, n_resblock=1, p_drop=0.1, + performer_L_opts=None, performer_N_opts=None, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}, + use_templ=False): + super(RoseTTAFoldModule, self).__init__() + self.use_templ = use_templ + # + self.msa_emb = MSA_emb(d_model=d_msa, p_drop=p_drop, max_len=5000) + if use_templ: + self.templ_emb = Templ_emb(d_templ=d_templ, n_att_head=n_head_templ, r_ff=r_ff, + performer_opts=performer_L_opts, p_drop=0.0) + self.pair_emb = Pair_emb_w_templ(d_model=d_pair, d_templ=d_templ, p_drop=p_drop) + else: + self.pair_emb = Pair_emb_wo_templ(d_model=d_pair, p_drop=p_drop) + # + self.feat_extractor = IterativeFeatureExtractor(n_module=n_module,\ + n_module_str=n_module_str,\ + n_layer=n_layer,\ + d_msa=d_msa, d_pair=d_pair, d_hidden=d_hidden,\ + n_head_msa=n_head_msa, \ + n_head_pair=n_head_pair,\ + r_ff=r_ff, \ + n_resblock=n_resblock, + p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts, + SE3_param=SE3_param) + self.c6d_predictor = DistanceNetwork(d_pair, p_drop=p_drop) + + def forward(self, msa, seq, idx, t1d=None, t2d=None): + B, N, L = msa.shape + # Get embeddings + msa = self.msa_emb(msa, idx) + if self.use_templ: + tmpl = self.templ_emb(t1d, t2d, idx) + pair = self.pair_emb(seq, idx, tmpl) + else: + pair = self.pair_emb(seq, idx) + # + # Extract features + seq1hot = torch.nn.functional.one_hot(seq, num_classes=21).float() + msa, pair, xyz, lddt = self.feat_extractor(msa, pair, seq1hot, idx) + + # Predict 6D coords + logits = self.c6d_predictor(pair) + + return logits, xyz, lddt.view(B, L) + + +class RoseTTAFoldModule_e2e(nn.Module): + def __init__(self, n_module=4, n_module_str=4, n_module_ref=4, n_layer=4,\ + d_msa=64, d_pair=128, d_templ=64,\ + n_head_msa=4, n_head_pair=8, n_head_templ=4, + d_hidden=64, r_ff=4, n_resblock=1, p_drop=0.0, + performer_L_opts=None, performer_N_opts=None, + SE3_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}, + REF_param={'l0_in_features':32, 'l0_out_features':16, 'num_edge_features':32}, + use_templ=False): + super(RoseTTAFoldModule_e2e, self).__init__() + self.use_templ = use_templ + # + self.msa_emb = MSA_emb(d_model=d_msa, p_drop=p_drop, max_len=5000) + if use_templ: + self.templ_emb = Templ_emb(d_templ=d_templ, n_att_head=n_head_templ, r_ff=r_ff, + performer_opts=performer_L_opts, p_drop=0.0) + self.pair_emb = Pair_emb_w_templ(d_model=d_pair, d_templ=d_templ, p_drop=p_drop) + else: + self.pair_emb = Pair_emb_wo_templ(d_model=d_pair, p_drop=p_drop) + # + self.feat_extractor = IterativeFeatureExtractor(n_module=n_module,\ + n_module_str=n_module_str,\ + n_layer=n_layer,\ + d_msa=d_msa, d_pair=d_pair, d_hidden=d_hidden,\ + n_head_msa=n_head_msa, \ + n_head_pair=n_head_pair,\ + r_ff=r_ff, \ + n_resblock=n_resblock, + p_drop=p_drop, + performer_N_opts=performer_N_opts, + performer_L_opts=performer_L_opts, + SE3_param=SE3_param) + self.c6d_predictor = DistanceNetwork(d_pair, p_drop=p_drop) + # + self.refine = Refine_module(n_module_ref, d_node=d_msa, d_pair=130, + d_node_hidden=d_hidden, d_pair_hidden=d_hidden, + SE3_param=REF_param, p_drop=p_drop) + + def forward(self, msa, seq, idx, t1d=None, t2d=None, prob_s=None, return_raw=False, refine_only=False): + seq1hot = torch.nn.functional.one_hot(seq, num_classes=21).float() + if not refine_only: + B, N, L = msa.shape + # Get embeddings + msa = self.msa_emb(msa, idx) + if self.use_templ: + tmpl = self.templ_emb(t1d, t2d, idx) + pair = self.pair_emb(seq, idx, tmpl) + else: + pair = self.pair_emb(seq, idx) + # + # Extract features + msa, pair, xyz, lddt = self.feat_extractor(msa, pair, seq1hot, idx) + + # Predict 6D coords + logits = self.c6d_predictor(pair) + + prob_s = list() + for l in logits: + prob_s.append(nn.Softmax(dim=1)(l)) # (B, C, L, L) + prob_s = torch.cat(prob_s, dim=1).permute(0,2,3,1) + + B, L = msa.shape[:2] + if return_raw: + return logits, msa, xyz, lddt.view(B, L) + + ref_xyz, ref_lddt = self.refine(msa, prob_s, seq1hot, idx) + + if refine_only: + return ref_xyz, ref_lddt.view(B,L) + else: + return logits, msa, ref_xyz, ref_lddt.view(B,L) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/SE3_network.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/SE3_network.py new file mode 100644 index 00000000..64c82600 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/SE3_network.py @@ -0,0 +1,108 @@ +import torch +import torch.nn as nn + +from equivariant_attention.modules import get_basis_and_r, GSE3Res, GNormBias +from equivariant_attention.modules import GConvSE3, GNormSE3 +from equivariant_attention.fibers import Fiber + +class TFN(nn.Module): + """SE(3) equivariant GCN""" + def __init__(self, num_layers=2, num_channels=32, num_nonlin_layers=1, num_degrees=3, + l0_in_features=32, l0_out_features=32, + l1_in_features=3, l1_out_features=3, + num_edge_features=32, use_self=True): + super().__init__() + # Build the network + self.num_layers = num_layers + self.num_nlayers = num_nonlin_layers + self.num_channels = num_channels + self.num_degrees = num_degrees + self.edge_dim = num_edge_features + self.use_self = use_self + + if l1_out_features > 0: + fibers = {'in': Fiber(dictionary={0: l0_in_features, 1: l1_in_features}), + 'mid': Fiber(self.num_degrees, self.num_channels), + 'out': Fiber(dictionary={0: l0_out_features, 1: l1_out_features})} + else: + fibers = {'in': Fiber(dictionary={0: l0_in_features, 1: l1_in_features}), + 'mid': Fiber(self.num_degrees, self.num_channels), + 'out': Fiber(dictionary={0: l0_out_features})} + blocks = self._build_gcn(fibers) + self.block0 = blocks + + def _build_gcn(self, fibers): + + block0 = [] + fin = fibers['in'] + for i in range(self.num_layers-1): + block0.append(GConvSE3(fin, fibers['mid'], self_interaction=self.use_self, edge_dim=self.edge_dim)) + block0.append(GNormSE3(fibers['mid'], num_layers=self.num_nlayers)) + fin = fibers['mid'] + block0.append(GConvSE3(fibers['mid'], fibers['out'], self_interaction=self.use_self, edge_dim=self.edge_dim)) + return nn.ModuleList(block0) + + @torch.cuda.amp.autocast(enabled=False) + def forward(self, G, type_0_features, type_1_features): + # Compute equivariant weight basis from relative positions + basis, r = get_basis_and_r(G, self.num_degrees-1) + h = {'0': type_0_features, '1': type_1_features} + for layer in self.block0: + h = layer(h, G=G, r=r, basis=basis) + return h + +class SE3Transformer(nn.Module): + """SE(3) equivariant GCN with attention""" + def __init__(self, num_layers=2, num_channels=32, num_degrees=3, n_heads=4, div=4, + si_m='1x1', si_e='att', + l0_in_features=32, l0_out_features=32, + l1_in_features=3, l1_out_features=3, + num_edge_features=32, x_ij=None): + super().__init__() + # Build the network + self.num_layers = num_layers + self.num_channels = num_channels + self.num_degrees = num_degrees + self.edge_dim = num_edge_features + self.div = div + self.n_heads = n_heads + self.si_m, self.si_e = si_m, si_e + self.x_ij = x_ij + + if l1_out_features > 0: + fibers = {'in': Fiber(dictionary={0: l0_in_features, 1: l1_in_features}), + 'mid': Fiber(self.num_degrees, self.num_channels), + 'out': Fiber(dictionary={0: l0_out_features, 1: l1_out_features})} + else: + fibers = {'in': Fiber(dictionary={0: l0_in_features, 1: l1_in_features}), + 'mid': Fiber(self.num_degrees, self.num_channels), + 'out': Fiber(dictionary={0: l0_out_features})} + + blocks = self._build_gcn(fibers) + self.Gblock = blocks + + def _build_gcn(self, fibers): + # Equivariant layers + Gblock = [] + fin = fibers['in'] + for i in range(self.num_layers): + Gblock.append(GSE3Res(fin, fibers['mid'], edge_dim=self.edge_dim, + div=self.div, n_heads=self.n_heads, + learnable_skip=True, skip='cat', + selfint=self.si_m, x_ij=self.x_ij)) + Gblock.append(GNormBias(fibers['mid'])) + fin = fibers['mid'] + Gblock.append( + GSE3Res(fibers['mid'], fibers['out'], edge_dim=self.edge_dim, + div=1, n_heads=min(1, 2), learnable_skip=True, + skip='cat', selfint=self.si_e, x_ij=self.x_ij)) + return nn.ModuleList(Gblock) + + @torch.cuda.amp.autocast(enabled=False) + def forward(self, G, type_0_features, type_1_features): + # Compute equivariant weight basis from relative positions + basis, r = get_basis_and_r(G, self.num_degrees-1) + h = {'0': type_0_features, '1': type_1_features} + for layer in self.Gblock: + h = layer(h, G=G, r=r, basis=basis) + return h diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Transformer.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Transformer.py new file mode 100644 index 00000000..23418780 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/Transformer.py @@ -0,0 +1,480 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import copy +import math +from performer_pytorch import SelfAttention + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + +# for gradient checkpointing +def create_custom_forward(module, **kwargs): + def custom_forward(*inputs): + return module(*inputs, **kwargs) + return custom_forward + +class LayerNorm(nn.Module): + def __init__(self, d_model, eps=1e-5): + super(LayerNorm, self).__init__() + self.a_2 = nn.Parameter(torch.ones(d_model)) + self.b_2 = nn.Parameter(torch.zeros(d_model)) + self.eps = eps + + def forward(self, x): + mean = x.mean(-1, keepdim=True) + std = torch.sqrt(x.var(dim=-1, keepdim=True, unbiased=False) + self.eps) + x = self.a_2*(x-mean) + x /= std + x += self.b_2 + return x + +class FeedForwardLayer(nn.Module): + def __init__(self, d_model, d_ff, p_drop=0.1): + super(FeedForwardLayer, self).__init__() + self.linear1 = nn.Linear(d_model, d_ff) + self.dropout = nn.Dropout(p_drop, inplace=True) + self.linear2 = nn.Linear(d_ff, d_model) + + def forward(self, src): + src = self.linear2(self.dropout(F.relu_(self.linear1(src)))) + return src + +class MultiheadAttention(nn.Module): + def __init__(self, d_model, heads, k_dim=None, v_dim=None, dropout=0.1): + super(MultiheadAttention, self).__init__() + if k_dim == None: + k_dim = d_model + if v_dim == None: + v_dim = d_model + + self.heads = heads + self.d_model = d_model + self.d_k = d_model // heads + self.scaling = 1/math.sqrt(self.d_k) + + self.to_query = nn.Linear(d_model, d_model) + self.to_key = nn.Linear(k_dim, d_model) + self.to_value = nn.Linear(v_dim, d_model) + self.to_out = nn.Linear(d_model, d_model) + + self.dropout = nn.Dropout(dropout, inplace=True) + + def forward(self, query, key, value, return_att=False): + batch, L1 = query.shape[:2] + batch, L2 = key.shape[:2] + q = self.to_query(query).view(batch, L1, self.heads, self.d_k).permute(0,2,1,3) # (B, h, L, d_k) + k = self.to_key(key).view(batch, L2, self.heads, self.d_k).permute(0,2,1,3) # (B, h, L, d_k) + v = self.to_value(value).view(batch, L2, self.heads, self.d_k).permute(0,2,1,3) + # + attention = torch.matmul(q, k.transpose(-2, -1))*self.scaling + attention = F.softmax(attention, dim=-1) # (B, h, L1, L2) + attention = self.dropout(attention) + # + out = torch.matmul(attention, v) # (B, h, L, d_k) + out = out.permute(0,2,1,3).contiguous().view(batch, L1, -1) + # + out = self.to_out(out) + if return_att: + attention = 0.5*(attention + attention.permute(0,1,3,2)) + return out, attention.permute(0,2,3,1) + return out + +# Own implementation for tied multihead attention +class TiedMultiheadAttention(nn.Module): + def __init__(self, d_model, heads, k_dim=None, v_dim=None, dropout=0.1): + super(TiedMultiheadAttention, self).__init__() + if k_dim == None: + k_dim = d_model + if v_dim == None: + v_dim = d_model + + self.heads = heads + self.d_model = d_model + self.d_k = d_model // heads + self.scaling = 1/math.sqrt(self.d_k) + + self.to_query = nn.Linear(d_model, d_model) + self.to_key = nn.Linear(k_dim, d_model) + self.to_value = nn.Linear(v_dim, d_model) + self.to_out = nn.Linear(d_model, d_model) + + self.dropout = nn.Dropout(dropout, inplace=True) + + def forward(self, query, key, value, return_att=False): + B, N, L = query.shape[:3] + q = self.to_query(query).view(B, N, L, self.heads, self.d_k).permute(0,1,3,2,4).contiguous() # (B, N, h, l, k) + k = self.to_key(key).view(B, N, L, self.heads, self.d_k).permute(0,1,3,4,2).contiguous() # (B, N, h, k, l) + v = self.to_value(value).view(B, N, L, self.heads, self.d_k).permute(0,1,3,2,4).contiguous() # (B, N, h, l, k) + # + #attention = torch.matmul(q, k.transpose(-2, -1))/math.sqrt(N*self.d_k) # (B, N, h, L, L) + #attention = attention.sum(dim=1) # tied attention (B, h, L, L) + scale = self.scaling / math.sqrt(N) + q = q * scale + attention = torch.einsum('bnhik,bnhkj->bhij', q, k) + attention = F.softmax(attention, dim=-1) # (B, h, L, L) + attention = self.dropout(attention) + attention = attention.unsqueeze(1) # (B, 1, h, L, L) + # + out = torch.matmul(attention, v) # (B, N, h, L, d_k) + out = out.permute(0,1,3,2,4).contiguous().view(B, N, L, -1) + # + out = self.to_out(out) + if return_att: + attention = attention.squeeze(1) + attention = 0.5*(attention + attention.permute(0,1,3,2)) + attention = attention.permute(0,3,1,2) + return out, attention + return out + +class SequenceWeight(nn.Module): + def __init__(self, d_model, heads, dropout=0.1): + super(SequenceWeight, self).__init__() + self.heads = heads + self.d_model = d_model + self.d_k = d_model // heads + self.scale = 1.0 / math.sqrt(self.d_k) + + self.to_query = nn.Linear(d_model, d_model) + self.to_key = nn.Linear(d_model, d_model) + self.dropout = nn.Dropout(dropout, inplace=True) + + def forward(self, msa): + B, N, L = msa.shape[:3] + + msa = msa.permute(0,2,1,3) # (B, L, N, K) + tar_seq = msa[:,:,0].unsqueeze(2) # (B, L, 1, K) + + q = self.to_query(tar_seq).view(B, L, 1, self.heads, self.d_k).permute(0,1,3,2,4).contiguous() # (B, L, h, 1, k) + k = self.to_key(msa).view(B, L, N, self.heads, self.d_k).permute(0,1,3,4,2).contiguous() # (B, L, h, k, N) + + q = q * self.scale + attn = torch.matmul(q, k) # (B, L, h, 1, N) + attn = F.softmax(attn, dim=-1) + return self.dropout(attn) + +# Own implementation for multihead attention (Input shape: Batch, Len, Emb) +class SoftTiedMultiheadAttention(nn.Module): + def __init__(self, d_model, heads, k_dim=None, v_dim=None, dropout=0.1): + super(SoftTiedMultiheadAttention, self).__init__() + if k_dim == None: + k_dim = d_model + if v_dim == None: + v_dim = d_model + + self.heads = heads + self.d_model = d_model + self.d_k = d_model // heads + self.scale = 1.0 / math.sqrt(self.d_k) + + self.seq_weight = SequenceWeight(d_model, heads, dropout=dropout) + self.to_query = nn.Linear(d_model, d_model) + self.to_key = nn.Linear(k_dim, d_model) + self.to_value = nn.Linear(v_dim, d_model) + self.to_out = nn.Linear(d_model, d_model) + + self.dropout = nn.Dropout(dropout, inplace=True) + + def forward(self, query, key, value, return_att=False): + B, N, L = query.shape[:3] + # + seq_weight = self.seq_weight(query) # (B, L, h, 1, N) + seq_weight = seq_weight.permute(0,4,2,1,3) # (B, N, h, l, -1) + # + q = self.to_query(query).view(B, N, L, self.heads, self.d_k).permute(0,1,3,2,4).contiguous() # (B, N, h, l, k) + k = self.to_key(key).view(B, N, L, self.heads, self.d_k).permute(0,1,3,4,2).contiguous() # (B, N, h, k, l) + v = self.to_value(value).view(B, N, L, self.heads, self.d_k).permute(0,1,3,2,4).contiguous() # (B, N, h, l, k) + # + #attention = torch.matmul(q, k.transpose(-2, -1))/math.sqrt(N*self.d_k) # (B, N, h, L, L) + #attention = attention.sum(dim=1) # tied attention (B, h, L, L) + q = q * seq_weight # (B, N, h, l, k) + k = k * self.scale + attention = torch.einsum('bnhik,bnhkj->bhij', q, k) + attention = F.softmax(attention, dim=-1) # (B, h, L, L) + attention = self.dropout(attention) + attention = attention # (B, 1, h, L, L) + del q, k, seq_weight + # + #out = torch.matmul(attention, v) # (B, N, h, L, d_k) + out = torch.einsum('bhij,bnhjk->bnhik', attention, v) + out = out.permute(0,1,3,2,4).contiguous().view(B, N, L, -1) + # + out = self.to_out(out) + + if return_att: + attention = attention.squeeze(1) + attention = 0.5*(attention + attention.permute(0,1,3,2)) + attention = attention.permute(0,2,3,1) + return out, attention + return out + +class DirectMultiheadAttention(nn.Module): + def __init__(self, d_in, d_out, heads, dropout=0.1): + super(DirectMultiheadAttention, self).__init__() + self.heads = heads + self.proj_pair = nn.Linear(d_in, heads) + self.drop = nn.Dropout(dropout, inplace=True) + # linear projection to get values from given msa + self.proj_msa = nn.Linear(d_out, d_out) + # projection after applying attention + self.proj_out = nn.Linear(d_out, d_out) + + def forward(self, src, tgt): + B, N, L = tgt.shape[:3] + attn_map = F.softmax(self.proj_pair(src), dim=1).permute(0,3,1,2) # (B, h, L, L) + attn_map = self.drop(attn_map).unsqueeze(1) + + # apply attention + value = self.proj_msa(tgt).permute(0,3,1,2).contiguous().view(B, -1, self.heads, N, L) # (B,-1, h, N, L) + tgt = torch.matmul(value, attn_map).view(B, -1, N, L).permute(0,2,3,1) # (B,N,L,K) + tgt = self.proj_out(tgt) + return tgt + +class MaskedDirectMultiheadAttention(nn.Module): + def __init__(self, d_in, d_out, heads, d_k=32, dropout=0.1): + super(MaskedDirectMultiheadAttention, self).__init__() + self.heads = heads + self.scaling = 1/math.sqrt(d_k) + + self.to_query = nn.Linear(d_in, heads*d_k) + self.to_key = nn.Linear(d_in, heads*d_k) + self.to_value = nn.Linear(d_out, d_out) + self.to_out = nn.Linear(d_out, d_out) + self.dropout = nn.Dropout(dropout, inplace=True) + + def forward(self, query, key, value, mask): + batch, N, L = value.shape[:3] + # + # project to query, key, value + q = self.to_query(query).view(batch, L, self.heads, -1).permute(0,2,1,3) # (B, h, L, -1) + k = self.to_key(key).view(batch, L, self.heads, -1).permute(0,2,1,3) # (B, h, L, -1) + v = self.to_value(value).view(batch, N, L, self.heads, -1).permute(0,3,1,2,4) # (B, h, N, L, -1) + # + q = q*self.scaling + attention = torch.matmul(q, k.transpose(-2, -1)) # (B, h, L, L) + attention = attention.masked_fill(mask < 0.5, torch.finfo(q.dtype).min) + attention = F.softmax(attention, dim=-1) # (B, h, L1, L2) + attention = self.dropout(attention) # (B, h, 1, L, L) + # + #out = torch.matmul(attention, v) # (B, h, N, L, d_out//h) + out = torch.einsum('bhij,bhnjk->bhnik', attention, v) # (B, h, N, L, d_out//h) + out = out.permute(0,2,3,1,4).contiguous().view(batch, N, L, -1) + # + out = self.to_out(out) + return out + +# Use PreLayerNorm for more stable training +class EncoderLayer(nn.Module): + def __init__(self, d_model, d_ff, heads, p_drop=0.1, performer_opts=None, use_tied=False): + super(EncoderLayer, self).__init__() + self.use_performer = performer_opts is not None + self.use_tied = use_tied + # multihead attention + if self.use_performer: + self.attn = SelfAttention(dim=d_model, heads=heads, dropout=p_drop, + generalized_attention=True, **performer_opts) + elif use_tied: + self.attn = SoftTiedMultiheadAttention(d_model, heads, dropout=p_drop) + else: + self.attn = MultiheadAttention(d_model, heads, dropout=p_drop) + # feedforward + self.ff = FeedForwardLayer(d_model, d_ff, p_drop=p_drop) + + # normalization module + self.norm1 = LayerNorm(d_model) + self.norm2 = LayerNorm(d_model) + self.dropout1 = nn.Dropout(p_drop, inplace=True) + self.dropout2 = nn.Dropout(p_drop, inplace=True) + + def forward(self, src, return_att=False): + # Input shape for multihead attention: (BATCH, SRCLEN, EMB) + # multihead attention w/ pre-LayerNorm + B, N, L = src.shape[:3] + src2 = self.norm1(src) + if not self.use_tied: + src2 = src2.reshape(B*N, L, -1) + if return_att: + src2, att = self.attn(src2, src2, src2, return_att=return_att) + src2 = src2.reshape(B,N,L,-1) + else: + src2 = self.attn(src2, src2, src2).reshape(B,N,L,-1) + src = src + self.dropout1(src2) + + # feed-forward + src2 = self.norm2(src) # pre-normalization + src2 = self.ff(src2) + src = src + self.dropout2(src2) + if return_att: + return src, att + return src + +# AxialTransformer with tied attention for L dimension +class AxialEncoderLayer(nn.Module): + def __init__(self, d_model, d_ff, heads, p_drop=0.1, performer_opts=None, + use_tied_row=False, use_tied_col=False, use_soft_row=False): + super(AxialEncoderLayer, self).__init__() + self.use_performer = performer_opts is not None + self.use_tied_row = use_tied_row + self.use_tied_col = use_tied_col + self.use_soft_row = use_soft_row + # multihead attention + if use_tied_row: + self.attn_L = TiedMultiheadAttention(d_model, heads, dropout=p_drop) + elif use_soft_row: + self.attn_L = SoftTiedMultiheadAttention(d_model, heads, dropout=p_drop) + else: + if self.use_performer: + self.attn_L = SelfAttention(dim=d_model, heads=heads, dropout=p_drop, + generalized_attention=True, **performer_opts) + else: + self.attn_L = MultiheadAttention(d_model, heads, dropout=p_drop) + if use_tied_col: + self.attn_N = TiedMultiheadAttention(d_model, heads, dropout=p_drop) + else: + if self.use_performer: + self.attn_N = SelfAttention(dim=d_model, heads=heads, dropout=p_drop, + generalized_attention=True, **performer_opts) + else: + self.attn_N = MultiheadAttention(d_model, heads, dropout=p_drop) + + # feedforward + self.ff = FeedForwardLayer(d_model, d_ff, p_drop=p_drop) + + # normalization module + self.norm1 = LayerNorm(d_model) + self.norm2 = LayerNorm(d_model) + self.norm3 = LayerNorm(d_model) + self.dropout1 = nn.Dropout(p_drop, inplace=True) + self.dropout2 = nn.Dropout(p_drop, inplace=True) + self.dropout3 = nn.Dropout(p_drop, inplace=True) + + def forward(self, src, return_att=False): + # Input shape for multihead attention: (BATCH, NSEQ, NRES, EMB) + # Tied multihead attention w/ pre-LayerNorm + B, N, L = src.shape[:3] + src2 = self.norm1(src) + if self.use_tied_row or self.use_soft_row: + src2 = self.attn_L(src2, src2, src2) # Tied attention over L + else: + src2 = src2.reshape(B*N, L, -1) + src2 = self.attn_L(src2, src2, src2) + src2 = src2.reshape(B, N, L, -1) + src = src + self.dropout1(src2) + + # attention over N + src2 = self.norm2(src) + if self.use_tied_col: + src2 = src2.permute(0,2,1,3) + src2 = self.attn_N(src2, src2, src2) # Tied attention over N + src2 = src2.permute(0,2,1,3) + else: + src2 = src2.permute(0,2,1,3).reshape(B*L, N, -1) + src2 = self.attn_N(src2, src2, src2) # attention over N + src2 = src2.reshape(B, L, N, -1).permute(0,2,1,3) + src = src + self.dropout2(src2) + + # feed-forward + src2 = self.norm3(src) # pre-normalization + src2 = self.ff(src2) + src = src + self.dropout3(src2) + return src + +class Encoder(nn.Module): + def __init__(self, enc_layer, n_layer): + super(Encoder, self).__init__() + self.layers = _get_clones(enc_layer, n_layer) + self.n_layer = n_layer + + def forward(self, src, return_att=False): + output = src + for layer in self.layers: + output = layer(output, return_att=return_att) + return output + +class CrossEncoderLayer(nn.Module): + def __init__(self, d_model, d_ff, heads, d_k, d_v, performer_opts=None, p_drop=0.1): + super(CrossEncoderLayer, self).__init__() + self.use_performer = performer_opts is not None + + # multihead attention + if self.use_performer: + self.attn = SelfAttention(dim=d_model, k_dim=d_k, heads=heads, dropout=p_drop, + generalized_attention=True, **performer_opts) + else: + self.attn = MultiheadAttention(d_model, heads, k_dim=d_k, v_dim=d_v, dropout=p_drop) + # feedforward + self.ff = FeedForwardLayer(d_model, d_ff, p_drop=p_drop) + + # normalization module + self.norm = LayerNorm(d_k) + self.norm1 = LayerNorm(d_model) + self.norm2 = LayerNorm(d_model) + self.dropout1 = nn.Dropout(p_drop, inplace=True) + self.dropout2 = nn.Dropout(p_drop, inplace=True) + + def forward(self, src, tgt): + # Input: + # For MSA to Pair: src (N, L, K), tgt (L, L, C) + # For Pair to MSA: src (L, L, C), tgt (N, L, K) + # Input shape for multihead attention: (SRCLEN, BATCH, EMB) + # multihead attention + # pre-normalization + src = self.norm(src) + tgt2 = self.norm1(tgt) + tgt2 = self.attn(tgt2, src, src) # projection to query, key, value are done in MultiheadAttention module + tgt = tgt + self.dropout1(tgt2) + + # Feed forward + tgt2 = self.norm2(tgt) + tgt2 = self.ff(tgt2) + tgt = tgt + self.dropout2(tgt2) + + return tgt + +class DirectEncoderLayer(nn.Module): + def __init__(self, heads, d_in, d_out, d_ff, symmetrize=True, p_drop=0.1): + super(DirectEncoderLayer, self).__init__() + self.symmetrize = symmetrize + + self.attn = DirectMultiheadAttention(d_in, d_out, heads, dropout=p_drop) + self.ff = FeedForwardLayer(d_out, d_ff, p_drop=p_drop) + + # dropouts + self.drop_1 = nn.Dropout(p_drop, inplace=True) + self.drop_2 = nn.Dropout(p_drop, inplace=True) + # LayerNorm + self.norm = LayerNorm(d_in) + self.norm1 = LayerNorm(d_out) + self.norm2 = LayerNorm(d_out) + + def forward(self, src, tgt): + # Input: + # For pair to msa: src=pair (B, L, L, C), tgt=msa (B, N, L, K) + B, N, L = tgt.shape[:3] + # get attention map + if self.symmetrize: + src = 0.5*(src + src.permute(0,2,1,3)) + src = self.norm(src) + tgt2 = self.norm1(tgt) + tgt2 = self.attn(src, tgt2) + tgt = tgt + self.drop_1(tgt2) + + # feed-forward + tgt2 = self.norm2(tgt.view(B*N,L,-1)).view(B,N,L,-1) + tgt2 = self.ff(tgt2) + tgt = tgt + self.drop_2(tgt2) + + return tgt + +class CrossEncoder(nn.Module): + def __init__(self, enc_layer, n_layer): + super(CrossEncoder, self).__init__() + self.layers = _get_clones(enc_layer, n_layer) + self.n_layer = n_layer + def forward(self, src, tgt): + output = tgt + for layer in self.layers: + output = layer(src, output) + return output + + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/fibers.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/fibers.py new file mode 100644 index 00000000..4da8f06e --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/fibers.py @@ -0,0 +1,163 @@ +from utils.utils_profiling import * # load before other local modules + +import math +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import copy + +from typing import Dict, List, Tuple + + +class Fiber(object): + """A Handy Data Structure for Fibers""" + def __init__(self, num_degrees: int=None, num_channels: int=None, + structure: List[Tuple[int,int]]=None, dictionary=None): + """ + define fiber structure; use one num_degrees & num_channels OR structure + OR dictionary + + :param num_degrees: degrees will be [0, ..., num_degrees-1] + :param num_channels: number of channels, same for each degree + :param structure: e.g. [(32, 0),(16, 1),(16,2)] + :param dictionary: e.g. {0:32, 1:16, 2:16} + """ + if structure: + self.structure = structure + elif dictionary: + self.structure = [(dictionary[o], o) for o in sorted(dictionary.keys())] + else: + self.structure = [(num_channels, i) for i in range(num_degrees)] + + self.multiplicities, self.degrees = zip(*self.structure) + self.max_degree = max(self.degrees) + self.min_degree = min(self.degrees) + self.structure_dict = {k: v for v, k in self.structure} + self.dict = self.structure_dict + self.n_features = np.sum([i[0] * (2*i[1]+1) for i in self.structure]) + + self.feature_indices = {} + idx = 0 + for (num_channels, d) in self.structure: + length = num_channels * (2*d + 1) + self.feature_indices[d] = (idx, idx + length) + idx += length + + def copy_me(self, multiplicity: int=None): + s = copy.deepcopy(self.structure) + if multiplicity is not None: + # overwrite multiplicities + s = [(multiplicity, o) for m, o in s] + return Fiber(structure=s) + + @staticmethod + def combine(f1, f2): + new_dict = copy.deepcopy(f1.structure_dict) + for k, m in f2.structure_dict.items(): + if k in new_dict.keys(): + new_dict[k] += m + else: + new_dict[k] = m + structure = [(new_dict[k], k) for k in sorted(new_dict.keys())] + return Fiber(structure=structure) + + @staticmethod + def combine_max(f1, f2): + new_dict = copy.deepcopy(f1.structure_dict) + for k, m in f2.structure_dict.items(): + if k in new_dict.keys(): + new_dict[k] = max(m, new_dict[k]) + else: + new_dict[k] = m + structure = [(new_dict[k], k) for k in sorted(new_dict.keys())] + return Fiber(structure=structure) + + @staticmethod + def combine_selectively(f1, f2): + # only use orders which occur in fiber f1 + + new_dict = copy.deepcopy(f1.structure_dict) + for k in f1.degrees: + if k in f2.degrees: + new_dict[k] += f2.structure_dict[k] + structure = [(new_dict[k], k) for k in sorted(new_dict.keys())] + return Fiber(structure=structure) + + @staticmethod + def combine_fibers(val1, struc1, val2, struc2): + """ + combine two fibers + + :param val1/2: fiber tensors in dictionary form + :param struc1/2: structure of fiber + :return: fiber tensor in dictionary form + """ + struc_out = Fiber.combine(struc1, struc2) + val_out = {} + for k in struc_out.degrees: + if k in struc1.degrees: + if k in struc2.degrees: + val_out[k] = torch.cat([val1[k], val2[k]], -2) + else: + val_out[k] = val1[k] + else: + val_out[k] = val2[k] + assert val_out[k].shape[-2] == struc_out.structure_dict[k] + return val_out + + def __repr__(self): + return f"{self.structure}" + + + +def get_fiber_dict(F, struc, mask=None, return_struc=False): + if mask is None: mask = struc + index = 0 + fiber_dict = {} + first_dims = F.shape[:-1] + masked_dict = {} + for o, m in struc.structure_dict.items(): + length = m * (2*o + 1) + if o in mask.degrees: + masked_dict[o] = m + fiber_dict[o] = F[...,index:index + length].view(list(first_dims) + [m, 2*o + 1]) + index += length + assert F.shape[-1] == index + if return_struc: + return fiber_dict, Fiber(dictionary=masked_dict) + return fiber_dict + + +def get_fiber_tensor(F, struc): + some_entry = tuple(F.values())[0] + first_dims = some_entry.shape[:-2] + res = some_entry.new_empty([*first_dims, struc.n_features]) + index = 0 + for o, m in struc.structure_dict.items(): + length = m * (2*o + 1) + res[..., index: index + length] = F[o].view(*first_dims, length) + index += length + assert index == res.shape[-1] + return res + + +def fiber2tensor(F, structure, squeeze=False): + if squeeze: + fibers = [F[f'{i}'].view(*F[f'{i}'].shape[:-2], -1) for i in structure.degrees] + fibers = torch.cat(fibers, -1) + else: + fibers = [F[f'{i}'].view(*F[f'{i}'].shape[:-2], -1, 1) for i in structure.degrees] + fibers = torch.cat(fibers, -2) + return fibers + + +def fiber2head(F, h, structure, squeeze=False): + if squeeze: + fibers = [F[f'{i}'].view(*F[f'{i}'].shape[:-2], h, -1) for i in structure.degrees] + fibers = torch.cat(fibers, -1) + else: + fibers = [F[f'{i}'].view(*F[f'{i}'].shape[:-2], h, -1, 1) for i in structure.degrees] + fibers = torch.cat(fibers, -2) + return fibers + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/SO3.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/SO3.py new file mode 100644 index 00000000..5b074382 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/SO3.py @@ -0,0 +1,289 @@ +# pylint: disable=C,E1101,E1102 +''' +Some functions related to SO3 and his usual representations + +Using ZYZ Euler angles parametrisation +''' +import torch +import math +import numpy as np + + +class torch_default_dtype: + + def __init__(self, dtype): + self.saved_dtype = None + self.dtype = dtype + + def __enter__(self): + self.saved_dtype = torch.get_default_dtype() + torch.set_default_dtype(self.dtype) + + def __exit__(self, exc_type, exc_value, traceback): + torch.set_default_dtype(self.saved_dtype) + + +def rot_z(gamma): + ''' + Rotation around Z axis + ''' + if not torch.is_tensor(gamma): + gamma = torch.tensor(gamma, dtype=torch.get_default_dtype()) + return torch.tensor([ + [torch.cos(gamma), -torch.sin(gamma), 0], + [torch.sin(gamma), torch.cos(gamma), 0], + [0, 0, 1] + ], dtype=gamma.dtype) + + +def rot_y(beta): + ''' + Rotation around Y axis + ''' + if not torch.is_tensor(beta): + beta = torch.tensor(beta, dtype=torch.get_default_dtype()) + return torch.tensor([ + [torch.cos(beta), 0, torch.sin(beta)], + [0, 1, 0], + [-torch.sin(beta), 0, torch.cos(beta)] + ], dtype=beta.dtype) + + +def rot(alpha, beta, gamma): + ''' + ZYZ Eurler angles rotation + ''' + return rot_z(alpha) @ rot_y(beta) @ rot_z(gamma) + + +def x_to_alpha_beta(x): + ''' + Convert point (x, y, z) on the sphere into (alpha, beta) + ''' + if not torch.is_tensor(x): + x = torch.tensor(x, dtype=torch.get_default_dtype()) + x = x / torch.norm(x) + beta = torch.acos(x[2]) + alpha = torch.atan2(x[1], x[0]) + return (alpha, beta) + + +# These functions (x_to_alpha_beta and rot) satisfies that +# rot(*x_to_alpha_beta([x, y, z]), 0) @ np.array([[0], [0], [1]]) +# is proportional to +# [x, y, z] + + +def irr_repr(order, alpha, beta, gamma, dtype=None): + """ + irreducible representation of SO3 + - compatible with compose and spherical_harmonics + """ + # from from_lielearn_SO3.wigner_d import wigner_D_matrix + from lie_learn.representations.SO3.wigner_d import wigner_D_matrix + # if order == 1: + # # change of basis to have vector_field[x, y, z] = [vx, vy, vz] + # A = np.array([[0, 0, 1], [1, 0, 0], [0, 1, 0]]) + # return A @ wigner_D_matrix(1, alpha, beta, gamma) @ A.T + + # TODO (non-essential): try to do everything in torch + # return torch.tensor(wigner_D_matrix(torch.tensor(order), alpha, beta, gamma), dtype=torch.get_default_dtype() if dtype is None else dtype) + return torch.tensor(wigner_D_matrix(order, np.array(alpha), np.array(beta), np.array(gamma)), dtype=torch.get_default_dtype() if dtype is None else dtype) + + +# def spherical_harmonics(order, alpha, beta, dtype=None): +# """ +# spherical harmonics +# - compatible with irr_repr and compose +# """ +# # from from_lielearn_SO3.spherical_harmonics import sh +# from lie_learn.representations.SO3.spherical_harmonics import sh # real valued by default +# +# ################################################################################################################### +# # ON ANGLE CONVENTION +# # +# # sh has following convention for angles: +# # :param theta: the colatitude / polar angle, ranging from 0(North Pole, (X, Y, Z) = (0, 0, 1)) to pi(South Pole, (X, Y, Z) = (0, 0, -1)). +# # :param phi: the longitude / azimuthal angle, ranging from 0 to 2 pi. +# # +# # this function therefore (probably) has the following convention for alpha and beta: +# # beta = pi - theta; ranging from 0(South Pole, (X, Y, Z) = (0, 0, -1)) to pi(North Pole, (X, Y, Z) = (0, 0, 1)). +# # alpha = phi +# # +# ################################################################################################################### +# +# Y = torch.tensor([sh(order, m, theta=math.pi - beta, phi=alpha) for m in range(-order, order + 1)], dtype=torch.get_default_dtype() if dtype is None else dtype) +# # if order == 1: +# # # change of basis to have vector_field[x, y, z] = [vx, vy, vz] +# # A = np.array([[0, 0, 1], [1, 0, 0], [0, 1, 0]]) +# # return A @ Y +# return Y + + +def compose(a1, b1, c1, a2, b2, c2): + """ + (a, b, c) = (a1, b1, c1) composed with (a2, b2, c2) + """ + comp = rot(a1, b1, c1) @ rot(a2, b2, c2) + xyz = comp @ torch.tensor([0, 0, 1.]) + a, b = x_to_alpha_beta(xyz) + rotz = rot(0, -b, -a) @ comp + c = torch.atan2(rotz[1, 0], rotz[0, 0]) + return a, b, c + + +def kron(x, y): + assert x.ndimension() == 2 + assert y.ndimension() == 2 + return torch.einsum("ij,kl->ikjl", (x, y)).view(x.size(0) * y.size(0), x.size(1) * y.size(1)) + + +################################################################################ +# Change of basis +################################################################################ + + +def xyz_vector_basis_to_spherical_basis(): + """ + to convert a vector [x, y, z] transforming with rot(a, b, c) + into a vector transforming with irr_repr(1, a, b, c) + see assert for usage + """ + with torch_default_dtype(torch.float64): + A = torch.tensor([[0, 1, 0], [0, 0, 1], [1, 0, 0]], dtype=torch.float64) + assert all(torch.allclose(irr_repr(1, a, b, c) @ A, A @ rot(a, b, c)) for a, b, c in torch.rand(10, 3)) + return A.type(torch.get_default_dtype()) + + +def tensor3x3_repr(a, b, c): + """ + representation of 3x3 tensors + T --> R T R^t + """ + r = rot(a, b, c) + return kron(r, r) + + +def tensor3x3_repr_basis_to_spherical_basis(): + """ + to convert a 3x3 tensor transforming with tensor3x3_repr(a, b, c) + into its 1 + 3 + 5 component transforming with irr_repr(0, a, b, c), irr_repr(1, a, b, c), irr_repr(3, a, b, c) + see assert for usage + """ + with torch_default_dtype(torch.float64): + to1 = torch.tensor([ + [1, 0, 0, 0, 1, 0, 0, 0, 1], + ], dtype=torch.get_default_dtype()) + assert all(torch.allclose(irr_repr(0, a, b, c) @ to1, to1 @ tensor3x3_repr(a, b, c)) for a, b, c in torch.rand(10, 3)) + + to3 = torch.tensor([ + [0, 0, -1, 0, 0, 0, 1, 0, 0], + [0, 1, 0, -1, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 0, -1, 0], + ], dtype=torch.get_default_dtype()) + assert all(torch.allclose(irr_repr(1, a, b, c) @ to3, to3 @ tensor3x3_repr(a, b, c)) for a, b, c in torch.rand(10, 3)) + + to5 = torch.tensor([ + [0, 1, 0, 1, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 0, 1, 0], + [-3**.5/3, 0, 0, 0, -3**.5/3, 0, 0, 0, 12**.5/3], + [0, 0, 1, 0, 0, 0, 1, 0, 0], + [1, 0, 0, 0, -1, 0, 0, 0, 0] + ], dtype=torch.get_default_dtype()) + assert all(torch.allclose(irr_repr(2, a, b, c) @ to5, to5 @ tensor3x3_repr(a, b, c)) for a, b, c in torch.rand(10, 3)) + + return to1.type(torch.get_default_dtype()), to3.type(torch.get_default_dtype()), to5.type(torch.get_default_dtype()) + + +################################################################################ +# Tests +################################################################################ + + +def test_is_representation(rep): + """ + rep(Z(a1) Y(b1) Z(c1) Z(a2) Y(b2) Z(c2)) = rep(Z(a1) Y(b1) Z(c1)) rep(Z(a2) Y(b2) Z(c2)) + """ + with torch_default_dtype(torch.float64): + a1, b1, c1, a2, b2, c2 = torch.rand(6) + + r1 = rep(a1, b1, c1) + r2 = rep(a2, b2, c2) + + a, b, c = compose(a1, b1, c1, a2, b2, c2) + r = rep(a, b, c) + + r_ = r1 @ r2 + + d, r = (r - r_).abs().max(), r.abs().max() + print(d.item(), r.item()) + assert d < 1e-10 * r, d / r + + +def _test_spherical_harmonics(order): + """ + This test tests that + - irr_repr + - compose + - spherical_harmonics + are compatible + + Y(Z(alpha) Y(beta) Z(gamma) x) = D(alpha, beta, gamma) Y(x) + with x = Z(a) Y(b) eta + """ + with torch_default_dtype(torch.float64): + a, b = torch.rand(2) + alpha, beta, gamma = torch.rand(3) + + ra, rb, _ = compose(alpha, beta, gamma, a, b, 0) + Yrx = spherical_harmonics(order, ra, rb) + + Y = spherical_harmonics(order, a, b) + DrY = irr_repr(order, alpha, beta, gamma) @ Y + + d, r = (Yrx - DrY).abs().max(), Y.abs().max() + print(d.item(), r.item()) + assert d < 1e-10 * r, d / r + + +def _test_change_basis_wigner_to_rot(): + # from from_lielearn_SO3.wigner_d import wigner_D_matrix + from lie_learn.representations.SO3.wigner_d import wigner_D_matrix + + with torch_default_dtype(torch.float64): + A = torch.tensor([ + [0, 1, 0], + [0, 0, 1], + [1, 0, 0] + ], dtype=torch.float64) + + a, b, c = torch.rand(3) + + r1 = A.t() @ torch.tensor(wigner_D_matrix(1, a, b, c), dtype=torch.float64) @ A + r2 = rot(a, b, c) + + d = (r1 - r2).abs().max() + print(d.item()) + assert d < 1e-10 + + +if __name__ == "__main__": + from functools import partial + + print("Change of basis") + xyz_vector_basis_to_spherical_basis() + test_is_representation(tensor3x3_repr) + tensor3x3_repr_basis_to_spherical_basis() + + print("Change of basis Wigner <-> rot") + _test_change_basis_wigner_to_rot() + _test_change_basis_wigner_to_rot() + _test_change_basis_wigner_to_rot() + + print("Spherical harmonics are solution of Y(rx) = D(r) Y(x)") + for l in range(7): + _test_spherical_harmonics(l) + + print("Irreducible repr are indeed representations") + for l in range(7): + test_is_representation(partial(irr_repr, l)) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/0.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/0.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..3baf93e118c732b6eb3354002c557fb148ea3db9 GIT binary patch literal 346 zcmV-g0j2&QiwFp2Pu*Yw|1d6aYis~*P`gS4K@{Cr3~Y=9J3Gs@SxAB=R)PeZ#aC>m zkzpTWGBWOD9vg@dNTaAMSU7fmfWKyED_D4wP3bM}FlWv^cg|UO@2|GCs&Z3vnT|tT zXv&l+x3MHiL0N`1$rYD)+A(h?D4P{c!4Z#$DO;HFIKwxR-{EAYi2~NK2ciq-#b>`C zCqKKlpBFpf!a4nNq0faCpcX8gNUnLx8J-fUD9?or1k{8VMzv-HlRTqwE}SRi1E^;c zf)%|dO1N|31*is~8gr>dEpm_xH=1j)5qZd4YVdp{>XBCzuromQCAs>tu+o(Uf53}` zk*f;zg^jj=blB~8_S^klx83da4n!rKP72{h_L&ymCCFV1>p!j?hwCiY^?hU>%+N$u sWJmU+srqx-0Mm@58azYJjdPh1NwNV=HAJ^JG#X~YKS|WZIGO@L}^_P zf7Rs%CA4=@PvfP@zc;S2B%lnd- zx_yUDdY=>crxfh9G85D*hpa4qa0O6XEs34Y0ZrzV1!-UDY`6jXvTpQ)iOq&vx7aR)tH_%TY&U7C>N~7h?p8wjw@KJp_}@T2X)t3 z)X;q&W+u((k2t(CQ>@=7#BXvd=UlHZPdI z1a_-T`A@4BQ?W2hr8j+&zplPMU5q!Tjr>D)>iOg9e&{^=*jT$A*lJ||`mOhEZ(PtOrE0A!wXSX45Sj%Q>XL@#OLHEro^We<>$sHRhFa{164U4 zVPs%vVCv$^`7_7ur$W|>07<A;jS1g3>}}Z%MzBLY6?Faj}IA*wiv6>PbSvlHvB- z{YO>j-`Ov-i2sPb-DI1&984W4_S5%V>8+aTu2X4~Vk+S)BRth+UQNry^2?z%*P-fp z3e209DJIxFGbw!b@J{WXdn!`vmEKRbIri@Nd$Yzy8wR!=(vFuuSi{sC`6!;>oKR`A zDOHArS*UQ&&%S`Y(x$C@Os2^i+qk9J@JZ#EsVOJeT!X3);Ojn@r_#2kPt0f4_C38e zF!Nhi6_st8+-p;ISL@}TpGA9M=1*yz5;Vn|!JE~awQUNJWNe$#*~1Ntw4BuB5@4JG k(+RhKQA%o2YKlu{atSc%HG`&<0QKkr0HAm%9a;ha0Md^NcK`qY literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/11.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/11.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..b117a24d68843f0a295759919922ca492d4920f8 GIT binary patch literal 696 zcmV;p0!RHHiwFn@aou17|1mKxaBFM;YhaoBo{0erdU#6mi;^?+;!8_1bBd?*2*nqr zCY5I9q{Nq`<`w4`#g`dPnLI_KhZmx(7)T|ir%ve+h|kGSOo>k`%Fm5Ysw_z@2C8yO z0UFxC)WwzaXO7!Xg{&0;l6nnHzAu@9WNjgXMuY&+!h)jwlKkZSocOZTqTm-{>-ZkNtjWhS`dHyoyL(*E9{{538vpn> zw#faxJZaCg+{_4rtb{!e9yfehbt2P7;=ZDn>Y{0T1fll*^1ZZo|Al@V(U;++ss0me zR=!bjeH}Vw&rGoW*X{TIKb$R=s)Z5lIjW@n7o0^bC*2txqBQf&1c;ZUuYvA8o^}2(7Y%0U95+F<}@2u zqYC?rDO2~{as6Hvero2P*G&6sFU_9>wg343@0m#()$3gM@IcL*x{gV@?B66C<6j44 z8E3`VSO(w8IBDDo55Fm`Q-Y>=GkEiRGq+9gX7*-mo6_0C4NO!ysmUe4GzCn@-2O!= esYR(NE}6+Cz@(`eG^GTnM-KpWVs{_<0ssKXv0AbK literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/12.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/12.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..41f785aa461c598abe14fb411dc22deb5b496cb5 GIT binary patch literal 659 zcmV;E0&M*siwFn@aou17|1mNyaBFM;ZBtEX6G0T7{8>p&EcFn)2_nRUQA|P=wj@Xi z%35u!=pyQxWT(yQn$7Iaw$@r$u-2$KXu3(&1_@M65f7pV1s8g25qcB~fJ~MaF=Ry&G0f|#CXVj6 zW~^{2KxI0C^a!*{U816LR40WP9bdnS`-M9V9?^^2Dj*ANe+aspCf89gA zlL&-`F7jYOQ?Z&=6>$`5x~%4riwJ}OVH6o-1&S~)$>}_D7X?4L#N!1h>he{nBhMuA zh6NwF)N`tak&kx^O&4+=qfNY5@HT6x`4QB?dkurU85TO4V%jnS*_H)=pK4?jc!J*g zkxK{?(ZN_W8jtnFd*bl}v3-aZ&K3;h;rU}2c?U_ka|rxT>qI!iFN*1tD(bb@>E zqMn*Pznu8l`1E1(w+eUR?3s5GC~)S=SnA38O}j$*MRV@Ev4YDS$o;TR&0n{{*Cp}e z>DM+e|6F?2eUISI(5}XF8-TU9y`IMelM};BNjr4paAsTY`$Yi06oa7|4uHBlx~EQf z^Ud;L!wynDLAxs27JHAz1-k{wmlT1-m07=_?cA~3^yV`bU$Lt#{VLNJIEp!j*zuQW z%LmsuC!gX1i|;d>+K6Y{`{f)TPE7e##upj?h+&ywjp?5r$+h`4>s(-C zzRfe>%!kEQhiCpOtnQRkpUsE)PtO12yXy4g__gQS=H>XQIeDCQ!y2-9z;ES!(-!aJ tUDMXBQYTp=1!ghXA+oof$2AFRQ0kMjn5^RNAq$h7$X~3DjL+%=002jIOsxO_ literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/13.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/13.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..9c64e363cde77c6990afb6ef13da041563bdd871 GIT binary patch literal 765 zcmVVI&R`m)sRSx(cq5q3iBhGPt5&3|TvemStAusDn273$H_*Jt@fAc_L(H?WtIp z+wmrbj(sm_Y6xSrEw?&$;JS&zOM*9V5FdIk1YZrkh@sb$EBf*zc86CkbS}-t5Zbae zqqGa7qR+d_x}Tc}{pfEQ7%~?xz5AHjH&;lrF)+mmibY)%(??BeAHi8vkMi|Nu=fMr zp{S36PLa8v<_r1}+(hf4_nF6(H}equMnETiqB!T@cqjt@2LRguI{}A5ZyNN*&EQ(R zc>H^iS0~^w_)P)71?+yXhs>#-(#C1?E%*=q?E@`uVqG4+;2#1s%mz#R5X3Qo@5dI* z9JA`_-DZKM-+iXb($7aY8lMk~o(4L~x;TT@e(D43cJ?q vNoY0NgDGTbh`dK5JA>@0lv2gZE4;T@@MvW9>?=1k;^X=Q*$#In+5`Xqx1fI; literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/14.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/14.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..f358d8eecd9f2df846c0fc315b60156078d73d72 GIT binary patch literal 539 zcmV+$0_6Q4iwFn@aou17|1mT!aBFM;Yhan`&%^)*Jv=4(Mada@@uel1ImJ_YgyM@* zlS(slQsPTe^NRC};>(PtOrE0A!wXSX45Sj%Q>XL@#OLHEro^We<>$sHRhFa{164U4 zVPs%vVCv$^`7_7ur$W|>07<}}Z%MzBLY6?Faj}IA*wiv6>PbSvlHvB- z{YO>j-?3+!^<~pW$0>UjoQcUWEn2qc!S%bEh5IJk2oxr(ip#Covu3N%KgFl5du~G2 z?-47xs4UWI^OLi6H?Mo3&C9(;(ssvY?NQ1<9X zo`)0vR29iK+x%s`u+Z@HR2%khRW6@)#O;B|=k0;1{}BCF`NSI=lmPYU0RTvN&xu+B005I63F!a; literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/15.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/15.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..1f2495dadaa0d48c858c446c29bddffc246c7859 GIT binary patch literal 658 zcmV;D0&V>tiwFn@aou17|1mW#aBFM;ZBtK36j2y|yE7)Nf0C?&ojPup#oQGNo*5xh zJStNUbE#o>_LURU&c2y9rlK$hUCb6K_u2YqMN0@H0;xk014fq)or*v@MA9Kp=;WoK zpr`MxxYBocKfXV|@B4iZK3|>wrhr|`*HYUt`eM3AP0MBZBf5iAo@u3ZiZib5=>09M zz@qs&QRZ?;-oz~5pj&n_t>19$+j?q{VwYE)@8wPX@I=J=Q|kPE;?_!6Q_K%v+En;E zLO@h&-~+RcO>M)r^nUEPrkx=nUZB#<<649V>0lP?t~rEVqKptV zs#Woz+h;qJRH3l8vxS+VUn%-RIA=87Lo}tc2|gW*j~!QJt7(* zA+4Slok_&wt*7G6@zbq|mIM)vjjTr$gf3H}Ughns5%9m-P%?(ZUxLQrJ1@Vy(SyW~ zpJ=;Sgy^|(cCs`d6jqlyi`U<;Eo~{e7oX+>u+aL|JUSA*{oeD$Ut|Hu;ztu7JP20X ze!bfo3c%~aM>V_v!MS7i{x(m-Il*rSAW?OEaV!x2`WgTm)7{&HaB45_Z}!F~pina{ z@qQTtflT^k8;A85FQgjVKDP`1DJ)98!p})wUg9X6-J8d4kPm;WqaW2~DYfSQn!V@N@v s9J`yZ;YVa)gYp~VcUyDCNn;16J4}P}Q#{$tDCflf0i-H2%jyFF01%K*_W%F@ literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/16.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/16.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..7a7958164e4da46241545fa69e6e2ff15115153e GIT binary patch literal 779 zcmV+m1N8hKiwFn@aou17|1mZ$aBFM;ZB$E46G0fB(pHiJ22DJWc<^9i_0RxCh-Q%T zh|z3B!~+|mETvt`;@a(Qw@pDSt2{&~1Y`?>g!EEDEa@)AcpnCVNT&YB7*4WR6ZD%6fRn> zPwLJ@k=YhT;0^G4qL>pUub;c2NVm9-Zj}$CuDVY)0JOYehxlji{I6qOi%lgiE$`A7 zEBeM|;AD%@fgwdwC7&d6QCR3#c2C|x$=)hROu8m>Putpp(mRk zByj%{#xz{Z=zWRn0MwfOHD=Tgcc{8z@_O!L{ux+9@uy7Q)jrIxV+>)Og=Z;#!ZU<1 z1UJe2GV2b`x8wa$6Fq61pMW2-^=aey{7F*JnOl>A{avAX$r{1&ZAkQOQTOl1^-)Oc zYa;pyKi01q_1W{$?h#*19<4XWAMCTI57P4^{>E_38f0$kls;1L!8o2%e?t4E#USlZ zyHQ8!A$bjB0@8TFApaThubT2Vxu2)|2##T#hGf3)v*(ve=G+?qTHh3;{nd;OYW|P* z^ZJ0^tTTWqVeErCV`nUb`oly4delVTr=m9vy`u$}6+f@={szIPqNnj>v#z2&u7CXu JaJ6~`002^aiW2|; literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/17.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/17.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..a2a34d656fcb1dc1a4630ff83c26d2b3920f43be GIT binary patch literal 950 zcmV;n14;ZJiwFn@aou17|1mc%aBFM;ZB<)n6G0T6w9!giMg3Ai5K*WoA+;(BtEp1_ zSszs^L{YbGvYM4Oo86t1R{K~rsi{WkW3^F>lxnpWp%q0$ho<;If0Xt^5k(0276d;8 zQ3Rc2W@0CFKK5|$oO|cKX2)`#{W}$8R=gN$hA-ff!cb97J3d=75I55!3}^<_r-*spd@LzJs81E0UT4~e7CfLAh|X{rpIPE9NRvs(I`Cyu zD6G)JwW_>>XbW|RcLpY{Sy5KDvb4;-YV``Y+pQDFzED`t7l*b$-MJ0B+pB}gxQ-gv z5UC}YFGd|5zjvbHhyCGMgV%+GmMp#75Jf*+_ftpa^$AsfF1|f|B7suHRqK79qo{hV ze7*DhxWFK55 zdaIl8Kukyx`y`R;(eMQS0CZ7aJH-pc9weyDR*2}!8SfO0qm#-f5$`WTv6}kprx-%> zUn=oueUE9phpFFAx*w!=Dq`o3W%7GY-qYyI#|L&k{{)h{(@dsw=)!bsr_Y&af-&|b+K zonka6{_K1t`8eskm)kk@fjM!r{d25OERJm6pC;ZXP$}Hu<|8KXeP(&6RPGRXUaLUz z8=>|a5z`awVfTdhJKB`Z<0eLp%ii9*cWWWmTo+oPE_8n`{bQBwwgM#-w~eKJn1oI0&w~jj zCcK!CEShM1F)uZq1WmOJME-hAKk)eAqw7J#g1;1Rn6;{dkpQ6+JCcqY@sV^z%*4w zR)YS3H>f*Lc&g~6dAMJZLxCeoNUs>uUG<&~bTZVZ8K%BLa(XJL6$qO8Lz2tmD#Xx& zz4}JU6%Nxq>pdF_WR|jye8nSFwQJ#SRq152Rl37dLrE|1YHV!U(b%%HrRk-nW}OB0 z^@sHe>DHU3?%GSky{DrGaUI@r2tyE(VkN^R?f;4m)ayywMFig@|nRDp-ww=WlMR5Q*n z{cdp{*>NJX_^s@;7x8#$|H7DKJd`b>6<=fiR*qra3OUHm2mUF>w=!%Ub%B2Z{9u0+ z155*EfOi^j8qmf9TIX?G$Zs6)yI-Gz5`lk=`$3$OfD<@T?AO=8zd$0xH+BTBf6M)W zcNP3U0(&0pNrF@V9Ekn#J)GAf;=UHn_#t zo%Q`nFOdwZ2gI8pVqHmD9eKVp(7z0>yssZR7J)ZHguYnQSj;b0mLK;^k`|#pyq*cb zSXn-1S--jfmjPGt5AWajY4a4<7uU%GU`J892M7di`ccke%2Y-kLROe;eJ_{%I+7$wUW%wd4UKzqWxw2e9G?!P73-NWxU+y zMAQlI3$MQlcovKIjQ6n(^3IZ7LVXs>__KgVfWL&r`4c!M+^;yc*XNsG1l|hhey&XS zb7KGUS$ukZy{rg!3-47L_X+P0oR_{6%>ImsIy(gz1ssI+^Zebxo$KSTE|GJ>{Yr`b zLCZv}Gw9pm`r$erAP>CHxvZ$4QXLXy_3kUndxFikW$~DxrwHi~>cih*&_|lvk<$B^ zfbSQ}`wQsnL`gqEsINV2ci0k=YNbshwzNrdjMyim)%2sUDn65b^XdPL>bDHPVkrLh YfX}3V9kzGaCiQ9fA3cpi=b8ur08@BA+yDRo literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/2.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/2.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..c2a6cdf5d6c7eb677ade8284505f4dd9e6952cb7 GIT binary patch literal 422 zcmV;X0a^YZiwFp2Pu*Yw|1vIcYis~*QNK$AF&Mq;U6s}<;v#~B2toxZSW#Onh*Dh+ ze=j#Ep}kA>G+vrqPN+~QilTCZXxto}RIrO8P6hoB{2z33bQB!?^yp@WFMRL4yf1mF z=@+l6^0$J7pxoL>AzwJelR+2axad2?MgqGY@{0PDE(IMkP+S zQL!Sh2de2jnARdEaq@jR`_;F3yV#SeMdm&1|T(GDSF*WEMSFjdCH}zi+>aMd^ zL-&1{nKWB}#o?8iV*NcKo~M41;FS=KGu7+>QFbJg9v#jMXS33pE711bCWNh*AGJ%xu<&RZeXjPYw|4*iT8>A zy+-%FJiD4(ZG`=PqshwyxhxIE&q0I4LOVR+oa6{tya+RF}iw z%PkVFcTrE`rOD-l3Y9L3$_+*1B2F%L(Z8UeE-ng+IJh|I=AR%q_|1*~T7=R+*MG+5q&{DHp6wj~G!nIj&&sx@KrC59+S5 z4qfwon3*&>TH^4^OtF5S5HC`{MDTKm#+hn#gJ@zrpU#YB)48!+CYxuWu~qh2RFBOG zrY(Zq3RC{ms%9z{W~mH}*QYNkACu3ocbS5A+kC&dKVR-UYS!YmiqRy_zhAvA(50hz%+bIVi3bf4M*0062z$M*mL literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/4.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/4.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..ba30e849930e82095b23b4780d10a53aef259890 GIT binary patch literal 565 zcmV-50?Pd#iwFp2Pu*Yw|1>UeYis~(V3``v!~h08JSF)>$r*a_r6rj;#Z!8O;)_y~ zN;7j(;!9HVit~%&%Z#Q>o}$si3sF`Kq!QCpr}PNK=j11*#HSVI=f)>hmZTN~RXJT@ zWMF7u>f*}zGso?xLe`1^NxcRp-h$q@2`Xh$vSf zV+0>ik)^SPk*SfXg^7Wsv583`#NgzD(n4l$X}^*}mO!9!v4srS)G{XONkhVtp?KRm z(*)Z@8_`OIWT%S7Hq#s{99cJ)+qB+qbGj+kxhIL$Wjf>H);*RwJOz6dZtYL*==T@B zJzB$70GJtp+zt$SjNDlCp>SJ^CST=0Lgdi9=?u!18_l}&r5v`z_{;?3aA>CN0W z#hcliv299c4>vFpb5fH_fN=;+H{AY3DXB%NDK44GCBUSh88oE?s7DU~r{=c8Y61WN DRnie? literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/5.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/5.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..d3bd72ac713c3e3e2313989b59b473b2586bd4f4 GIT binary patch literal 699 zcmV;s0z~~EiwFp2Pu*Yw|1~afYis~(V43=!i2)3HcuMk%k~8$;OG`3yil_7l#TTU} zm1gFo#FwPz73UYlml;i&JVm327ow~fNF}DHPU#Vd&&f|riBBua&y7#2EJ-Z}s&YyJ z8rs0r#g+4Cj@wU#tQ7&0dJRmzFPVX4Z6Sk3gaFXOf};GA{N((c__EZZ;>`TKLPns1 z2wtE}PG(6-PHKE=UP@+SULjLQ1Q$@Sq_QA2zBscgwYZSEwva_5f)yxOoRMEtQpoDf z6u}JSW#*L>vU#&auz<|S&r2_4_hub(c-+sfI zid_d4X7AZltJtU;xyi;=O3Q(ned!)vsC@tDd(tI0J8e$c^jhkyHnT~e_UKy4>qRyl z(;j__v2NUB@gcvMfp^m$A*lKV%lNu$_VwCSh$X!ZY%H-^$>+R$o68~_H~aFGdd{*v z|2Hj=-*u^X&n&R}uG`P~u}){>s-QhTn-e9<1nX^5H6$i~doy`Y(CGzlEzULAOtUZc zubWq~2kL(N^SmyFkseJpqEJ3}fc61(w)r+jmF7#=brt-*t zxmiBvpZnRWJ%`#umi`tC*c0xZvax(`qm3X`|Kn=GI2nz}d# h{zWONMX4z+naL%$r*a_r6rj;#Z!8O;)_y~ zN;7j(;!9HVit~%&%Z#Q>o}$si3sF`Kq!QCpr}PNK=j11*#HSVI=f)>hmZTN~RXH7D zWMF7u>f*}zGso?xLe`1^NxcRp-h$q@2`Xh$vSf zV+0>ik)?^Tv8kc4fr+W1si{#R#NgzD(n4l$NxzapmO!9!v4srS)G{XONkYPs;r848 zM^)$F*)Ow*|A@ZbWShAhOdToq)AwBIt(xktQ)!c8D&Z?5Jk@4iP0Pgc%b_;cq3U@G z%$t@eCfGbPDSYpqvK(zd5h%xBg1J-s$C z^IKOHm2H~bYg2Yt>*b!GMSEc8PidVJG{u|2o7J1OZ3>WNY@5>A!wrnIoYdqJV4MNd j3AcYyN@`JRic4m42{7t4gQk=K_2>Zr%Oe`hS^@w7(X0y> literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/7.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/7.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..668439c60164d017d5828e7aa55cd647b9cee652 GIT binary patch literal 697 zcmV;q0!IBGiwFn@aou17|2HmhYis~(V43=!i2)3HcuMk%k~8$;OG`3yil_7l#TTU} zm1gFo#FwPz73UYlml;i&JVm327ow~fNF}DHPU#Vd&&f|riBBua&y7#2EJ-Z}s&YyJ z8rs0r#g+4Cj@wU#tQ7&0dJRmzFPVX4Z6Sk3gaFXOf};GA{N((c__EZZ;>`TKLPns1 z2wtE}PG(6-PHKE=UP@+SULjLQ1Q$@Sq_QA2zBscgwYZSEwva_5f)yxOoRMEtQpoDf z6u}JSW#*L>vU#&auz<|S&r2_4_huS&kBmY+RO1m3UXY zZO@(@ukdAl6?^nnx!N~hUu46XK7C=aY5bmv8y2`tk2`KZ-Not3yeDh+Jau``(_B8? z=2BU8_#v}NHq*fJ*Y~d!{`dRFk-|M*9tFDB?=09Odz6W@gLUDa`B43fbwuXGc=p=t z)Vb@wT7TA_Z&gi?=P(uSadeOqb9*$+W+q>zBCFm68-A$!UY7|lxs;~b@Id)|cg|<^ z=p^ldy5Igt$~P-+?GBqaiEod5I5KI^%b7-%y}GS7d#<$a47oYg#_xwslgfe^o0%t^ zF5l&xZId_AusF252=4zWty6-gcr$qOdo#C9@n-gBY@5>A!wpPSIjPAdz%&I+$K3u! fDXB%NDK44GCBUSq88oE?s7DU~IC;ZH`T_s|ve{uY literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/8.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/8.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..242a0f952a1f6ed25514838c2f5e38293832195e GIT binary patch literal 662 zcmV;H0%`ppiwFn@aou17|2QsiYis~*Q(H(BQ5ZgZOIS5UDa@CIde}e~w=FU_wGy&q zLpPI{mm2nR(ur|r&&-(x17YZ5tF7kdw65!JSXwFcP!EyB34?-OdkCbbf}VN^dMPN1 zIyz^~fzHEx{NMS$|8j=k^xpp90qZAMM^r5_5R?oeE4r0yk~9=IWF;vPl+smA%IvkK ztU#{bmgx*){m9BSN{SjwN*6SBP>K%`q%+l#K9-&Li;Jgr1Wotx&mQ(2 z#vmXzvVv(%C2B%dqzuw@SxsRV6NvTfMv)0okc3i6IhMlitmtPX8A>BbmoFh5dxo($ zAo>`o52zZ!KEW-nO(CZU-XeHK?^+Hu)sJfguVJv70kLLHY`2U-VtoXEw`#-{bjD`= z*d=adqJ#Uw;r*d-$AM6&qq7s+i4$o9dj#PG!QRs>-8l^Yr**}Gg8i1@>uk&S*MAgx z>+xK4`uv-f`DfQZ9d2Eq?LD+_;!2(__q(=7AI{UZ=aHu`mkSX51YuXBLCx9UPsYD3 zLD150Z1?yLT3Gk5u7I{zqpff6LtyTF^!`qn0x;E_Ia{K6JAVX#;+Mh87A$PUVApVa zh1%U@p^YVuH#tAg;~d8{oZE~+=FPiYe-%1-P=s|3eGyhR_s8ZeIbzH9ea!Pb1vu@P zzrgu>94D#6$L6=Ot6HB{9R4(~w;FNwTb*}?`}i@7bi(1U{+ES~ce1Woy_Nuk2EjLC w2|mF!VvXi%*%B!zLD&wly{$c|C6R`b-ExAkRov5S5#|a01)ug0y6OV}05X41s{jB1 literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/9.pkl.gz b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/9.pkl.gz new file mode 100644 index 0000000000000000000000000000000000000000..525a9a9624e585d69347969dd5ae10daf49b446f GIT binary patch literal 783 zcmV+q1MvJGiwFn@aou17|2ZyjYis~*R9#3EQ52s2v8`2Wi(ZTfAG9`vMf~5m;35*p zP?r6$57oFolTJ=MJF~NreF`f_)-tOR1^hK z58d5+R@~@5%*Q$BJLi1o%;hS^`)fujbwW9^AbFZ?Oi3g;3vTF_1X=J1Jk!cb0Z#CVI!tip;t0>lGS4!s&&#=dqQ2K@#!k82&oTk7 zgAIs=PSI#}W?^TbS&(Egi`F|0*y4P$xQjMAlN0fRuSqn~#$XWlv^q^mGLbA&p415D zS|RA>*>l8e7In_Om}D<2EiEstsI-^c?Ipz`DeUnFMFY*9mPO-PT<)?+{g112**LOG z>O$dgWRImyi+(S3-P@YcZgw53i`3jgTPu}c?&dy1bvM3!;2u9;RF*de#-s=_j#q7diyiv?jP5vsx|BWL;_Kb(=7#)35|*#e68)CLKJm4 z`%>QkDkS;U6lMOA_iP5mpgvKvpb>n52*wCHrolf?Y-NOC{xq__<3ev@%a9*I`9u$~ z8y}H8$1LPO(K=K8lKqos1mZ6t?gtztXqiAK((|Rx5a{X#-UEWlIOrOKct7|fX!FYE z`<#(g$R8!~rE!Qyf$spEC74L-9YT|X_3aDV2FX_*l6ciR>}3piy_x%>dH}=PMn?ZQ z%HS(1D$1B|5bCu7F9h}g?CB}^{U~!EquV+#r! zY9-tEB^eKE=gEB`_SZR>Bi(-(^rZhs?4No1pzkZ#e_TU5`)%)c3ixpE9CJ`d(YiA3 zYZM&?eQLMbpwbk*k2dwFw29XBsJBDe_|f25kBr|8{7z-pOI}uDy|tW2#!t`j235v6 Nw!aqocLCZ2003Roh^+ts literal 0 HcmV?d00001 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/index.pkl b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/cache/trans_Q/index.pkl new file mode 100644 index 0000000000000000000000000000000000000000..60fea06b59b64b4c9867304b11351f8e77e7ee76 GIT binary patch literal 557 zcmZY5I}XAy41nP@=|Ha0sZ&IFbA}v*0R|QpBqSEpr3;d|Zyg3d;&%LGC;8v~Z}9H_ zy}UvKI1gwmmP%Caz2Wg9Y_YUZ|ND2?sNys^D7SG+p!Kvs8Z zJCfC%+D>G_*qO{1uiqE541;UsWn$(sb+SQD6D)EYe4)N;%p3=w(W0zlkn_|tRf3qg KPj8U|J< l: + return torch.zeros_like(x) + + # Compute P_m^m + yold = ((-1)**m_abs * semifactorial(2*m_abs-1)) * torch.pow(1-x*x, m_abs/2) + + # Compute P_{m+1}^m + if m_abs != l: + y = x * (2*m_abs+1) * yold + else: + y = yold + + # Compute P_{l}^m from recursion in P_{l-1}^m and P_{l-2}^m + for i in range(m_abs+2, l+1): + tmp = y + # Inplace speedup + y = ((2*i-1) / (i-m_abs)) * x * y + y -= ((i+m_abs-1)/(i-m_abs)) * yold + yold = tmp + + if m < 0: + y *= ((-1)**m / pochhammer(l+m+1, -2*m)) + + return y + +def tesseral_harmonics(l, m, theta=0., phi=0.): + """Tesseral spherical harmonic with Condon-Shortley phase. + + The Tesseral spherical harmonics are also known as the real spherical + harmonics. + + Args: + l: int for degree + m: int for order, where -l <= m < l + theta: collatitude or polar angle + phi: longitude or azimuth + Returns: + tensor of shape theta + """ + assert abs(m) <= l, "absolute value of order m must be <= degree l" + + N = np.sqrt((2*l+1) / (4*np.pi)) + leg = lpmv(l, abs(m), torch.cos(theta)) + if m == 0: + return N*leg + elif m > 0: + Y = torch.cos(m*phi) * leg + else: + Y = torch.sin(abs(m)*phi) * leg + N *= np.sqrt(2. / pochhammer(l-abs(m)+1, 2*abs(m))) + Y *= N + return Y + +class SphericalHarmonics(object): + def __init__(self): + self.leg = {} + + def clear(self): + self.leg = {} + + def negative_lpmv(self, l, m, y): + """Compute negative order coefficients""" + if m < 0: + y *= ((-1)**m / pochhammer(l+m+1, -2*m)) + return y + + def lpmv(self, l, m, x): + """Associated Legendre function including Condon-Shortley phase. + + Args: + m: int order + l: int degree + x: float argument tensor + Returns: + tensor of x-shape + """ + # Check memoized versions + m_abs = abs(m) + if (l,m) in self.leg: + return self.leg[(l,m)] + elif m_abs > l: + return None + elif l == 0: + self.leg[(l,m)] = torch.ones_like(x) + return self.leg[(l,m)] + + # Check if on boundary else recurse solution down to boundary + if m_abs == l: + # Compute P_m^m + y = (-1)**m_abs * semifactorial(2*m_abs-1) + y *= torch.pow(1-x*x, m_abs/2) + self.leg[(l,m)] = self.negative_lpmv(l, m, y) + return self.leg[(l,m)] + else: + # Recursively precompute lower degree harmonics + self.lpmv(l-1, m, x) + + # Compute P_{l}^m from recursion in P_{l-1}^m and P_{l-2}^m + # Inplace speedup + y = ((2*l-1) / (l-m_abs)) * x * self.lpmv(l-1, m_abs, x) + if l - m_abs > 1: + y -= ((l+m_abs-1)/(l-m_abs)) * self.leg[(l-2, m_abs)] + #self.leg[(l, m_abs)] = y + + if m < 0: + y = self.negative_lpmv(l, m, y) + self.leg[(l,m)] = y + + return self.leg[(l,m)] + + def get_element(self, l, m, theta, phi): + """Tesseral spherical harmonic with Condon-Shortley phase. + + The Tesseral spherical harmonics are also known as the real spherical + harmonics. + + Args: + l: int for degree + m: int for order, where -l <= m < l + theta: collatitude or polar angle + phi: longitude or azimuth + Returns: + tensor of shape theta + """ + assert abs(m) <= l, "absolute value of order m must be <= degree l" + + N = np.sqrt((2*l+1) / (4*np.pi)) + leg = self.lpmv(l, abs(m), torch.cos(theta)) + if m == 0: + return N*leg + elif m > 0: + Y = torch.cos(m*phi) * leg + else: + Y = torch.sin(abs(m)*phi) * leg + N *= np.sqrt(2. / pochhammer(l-abs(m)+1, 2*abs(m))) + Y *= N + return Y + + def get(self, l, theta, phi, refresh=True): + """Tesseral harmonic with Condon-Shortley phase. + + The Tesseral spherical harmonics are also known as the real spherical + harmonics. + + Args: + l: int for degree + theta: collatitude or polar angle + phi: longitude or azimuth + Returns: + tensor of shape [*theta.shape, 2*l+1] + """ + results = [] + if refresh: + self.clear() + for m in range(-l, l+1): + results.append(self.get_element(l, m, theta, phi)) + return torch.stack(results, -1) + + + + +if __name__ == "__main__": + from lie_learn.representations.SO3.spherical_harmonics import sh + device = 'cuda' + dtype = torch.float64 + bs = 32 + theta = 0.1*torch.randn(bs,1024,10, dtype=dtype) + phi = 0.1*torch.randn(bs,1024,10, dtype=dtype) + cu_theta = theta.to(device) + cu_phi = phi.to(device) + s0 = s1 = s2 = 0 + max_error = -1. + + sph_har = SphericalHarmonics() + for l in range(10): + for m in range(l, -l-1, -1): + start = time.time() + #y = tesseral_harmonics(l, m, theta, phi) + y = sph_har.get_element(l, m, cu_theta, cu_phi).type(torch.float32) + #y = sph_har.lpmv(l, m, phi) + s0 += time.time() - start + start = time.time() + z = sh(l, m, theta, phi) + #z = lpmv_scipy(m, l, phi).numpy() + s1 += time.time() - start + + error = np.mean(np.abs((y.cpu().numpy() - z) / z)) + max_error = max(max_error, error) + print(f"l: {l}, m: {m} ", error) + + #start = time.time() + #sph_har.get(l, theta, phi) + #s2 += time.time() - start + + print('#################') + + print(f"Max error: {max_error}") + print(f"Time diff: {s0/s1}") + print(f"Total time: {s0}") + #print(f"Time diff: {s2/s1}") diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/utils_steerable.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/utils_steerable.py new file mode 100755 index 00000000..bfd297a1 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/from_se3cnn/utils_steerable.py @@ -0,0 +1,326 @@ +import os +import torch +import math +import numpy as np +from equivariant_attention.from_se3cnn.SO3 import irr_repr, torch_default_dtype +from equivariant_attention.from_se3cnn.cache_file import cached_dirpklgz +from equivariant_attention.from_se3cnn.representations import SphericalHarmonics + +################################################################################ +# Solving the constraint coming from the stabilizer of 0 and e +################################################################################ + +def get_matrix_kernel(A, eps=1e-10): + ''' + Compute an orthonormal basis of the kernel (x_1, x_2, ...) + A x_i = 0 + scalar_product(x_i, x_j) = delta_ij + + :param A: matrix + :return: matrix where each row is a basis vector of the kernel of A + ''' + _u, s, v = torch.svd(A) + + # A = u @ torch.diag(s) @ v.t() + kernel = v.t()[s < eps] + return kernel + + +def get_matrices_kernel(As, eps=1e-10): + ''' + Computes the commun kernel of all the As matrices + ''' + return get_matrix_kernel(torch.cat(As, dim=0), eps) + + +@cached_dirpklgz("%s/cache/trans_Q"%os.path.dirname(os.path.realpath(__file__))) +def _basis_transformation_Q_J(J, order_in, order_out, version=3): # pylint: disable=W0613 + """ + :param J: order of the spherical harmonics + :param order_in: order of the input representation + :param order_out: order of the output representation + :return: one part of the Q^-1 matrix of the article + """ + with torch_default_dtype(torch.float64): + def _R_tensor(a, b, c): return kron(irr_repr(order_out, a, b, c), irr_repr(order_in, a, b, c)) + + def _sylvester_submatrix(J, a, b, c): + ''' generate Kronecker product matrix for solving the Sylvester equation in subspace J ''' + R_tensor = _R_tensor(a, b, c) # [m_out * m_in, m_out * m_in] + R_irrep_J = irr_repr(J, a, b, c) # [m, m] + return kron(R_tensor, torch.eye(R_irrep_J.size(0))) - \ + kron(torch.eye(R_tensor.size(0)), R_irrep_J.t()) # [(m_out * m_in) * m, (m_out * m_in) * m] + + random_angles = [ + [4.41301023, 5.56684102, 4.59384642], + [4.93325116, 6.12697327, 4.14574096], + [0.53878964, 4.09050444, 5.36539036], + [2.16017393, 3.48835314, 5.55174441], + [2.52385107, 0.2908958, 3.90040975] + ] + null_space = get_matrices_kernel([_sylvester_submatrix(J, a, b, c) for a, b, c in random_angles]) + assert null_space.size(0) == 1, null_space.size() # unique subspace solution + Q_J = null_space[0] # [(m_out * m_in) * m] + Q_J = Q_J.view((2 * order_out + 1) * (2 * order_in + 1), 2 * J + 1) # [m_out * m_in, m] + assert all(torch.allclose(_R_tensor(a, b, c) @ Q_J, Q_J @ irr_repr(J, a, b, c)) for a, b, c in torch.rand(4, 3)) + + assert Q_J.dtype == torch.float64 + return Q_J # [m_out * m_in, m] + + +def get_spherical_from_cartesian_torch(cartesian, divide_radius_by=1.0): + + ################################################################################################################### + # ON ANGLE CONVENTION + # + # sh has following convention for angles: + # :param theta: the colatitude / polar angle, ranging from 0(North Pole, (X, Y, Z) = (0, 0, 1)) to pi(South Pole, (X, Y, Z) = (0, 0, -1)). + # :param phi: the longitude / azimuthal angle, ranging from 0 to 2 pi. + # + # the 3D steerable CNN code therefore (probably) has the following convention for alpha and beta: + # beta = pi - theta; ranging from 0(South Pole, (X, Y, Z) = (0, 0, -1)) to pi(North Pole, (X, Y, Z) = (0, 0, 1)). + # alpha = phi + # + ################################################################################################################### + + # initialise return array + # ptsnew = np.hstack((xyz, np.zeros(xyz.shape))) + spherical = torch.zeros_like(cartesian) + + # indices for return array + ind_radius = 0 + ind_alpha = 1 + ind_beta = 2 + + cartesian_x = 2 + cartesian_y = 0 + cartesian_z = 1 + + # get projected radius in xy plane + # xy = xyz[:,0]**2 + xyz[:,1]**2 + r_xy = cartesian[..., cartesian_x] ** 2 + cartesian[..., cartesian_y] ** 2 + + # get second angle + # version 'elevation angle defined from Z-axis down' + spherical[..., ind_beta] = torch.atan2(torch.sqrt(r_xy), cartesian[..., cartesian_z]) + # ptsnew[:,4] = np.arctan2(np.sqrt(xy), xyz[:,2]) + # version 'elevation angle defined from XY-plane up' + #ptsnew[:,4] = np.arctan2(xyz[:,2], np.sqrt(xy)) + # spherical[:, ind_beta] = np.arctan2(cartesian[:, 2], np.sqrt(r_xy)) + + # get angle in x-y plane + spherical[...,ind_alpha] = torch.atan2(cartesian[...,cartesian_y], cartesian[...,cartesian_x]) + + # get overall radius + # ptsnew[:,3] = np.sqrt(xy + xyz[:,2]**2) + if divide_radius_by == 1.0: + spherical[..., ind_radius] = torch.sqrt(r_xy + cartesian[...,cartesian_z]**2) + else: + spherical[..., ind_radius] = torch.sqrt(r_xy + cartesian[...,cartesian_z]**2)/divide_radius_by + + return spherical + + +def get_spherical_from_cartesian(cartesian): + + ################################################################################################################### + # ON ANGLE CONVENTION + # + # sh has following convention for angles: + # :param theta: the colatitude / polar angle, ranging from 0(North Pole, (X, Y, Z) = (0, 0, 1)) to pi(South Pole, (X, Y, Z) = (0, 0, -1)). + # :param phi: the longitude / azimuthal angle, ranging from 0 to 2 pi. + # + # the 3D steerable CNN code therefore (probably) has the following convention for alpha and beta: + # beta = pi - theta; ranging from 0(South Pole, (X, Y, Z) = (0, 0, -1)) to pi(North Pole, (X, Y, Z) = (0, 0, 1)). + # alpha = phi + # + ################################################################################################################### + + if torch.is_tensor(cartesian): + cartesian = np.array(cartesian.cpu()) + + # initialise return array + # ptsnew = np.hstack((xyz, np.zeros(xyz.shape))) + spherical = np.zeros(cartesian.shape) + + # indices for return array + ind_radius = 0 + ind_alpha = 1 + ind_beta = 2 + + cartesian_x = 2 + cartesian_y = 0 + cartesian_z = 1 + + # get projected radius in xy plane + # xy = xyz[:,0]**2 + xyz[:,1]**2 + r_xy = cartesian[..., cartesian_x] ** 2 + cartesian[..., cartesian_y] ** 2 + + # get overall radius + # ptsnew[:,3] = np.sqrt(xy + xyz[:,2]**2) + spherical[..., ind_radius] = np.sqrt(r_xy + cartesian[...,cartesian_z]**2) + + # get second angle + # version 'elevation angle defined from Z-axis down' + spherical[..., ind_beta] = np.arctan2(np.sqrt(r_xy), cartesian[..., cartesian_z]) + # ptsnew[:,4] = np.arctan2(np.sqrt(xy), xyz[:,2]) + # version 'elevation angle defined from XY-plane up' + #ptsnew[:,4] = np.arctan2(xyz[:,2], np.sqrt(xy)) + # spherical[:, ind_beta] = np.arctan2(cartesian[:, 2], np.sqrt(r_xy)) + + # get angle in x-y plane + spherical[...,ind_alpha] = np.arctan2(cartesian[...,cartesian_y], cartesian[...,cartesian_x]) + + return spherical + +def test_coordinate_conversion(): + p = np.array([0, 0, -1]) + expected = np.array([1, 0, 0]) + assert get_spherical_from_cartesian(p) == expected + return True + + +def spherical_harmonics(order, alpha, beta, dtype=None): + """ + spherical harmonics + - compatible with irr_repr and compose + + computation time: excecuting 1000 times with array length 1 took 0.29 seconds; + executing it once with array of length 1000 took 0.0022 seconds + """ + #Y = [tesseral_harmonics(order, m, theta=math.pi - beta, phi=alpha) for m in range(-order, order + 1)] + #Y = torch.stack(Y, -1) + # Y should have dimension 2*order + 1 + return SphericalHarmonics.get(order, theta=math.pi-beta, phi=alpha) + +def kron(a, b): + """ + A part of the pylabyk library: numpytorch.py at https://github.com/yulkang/pylabyk + + Kronecker product of matrices a and b with leading batch dimensions. + Batch dimensions are broadcast. The number of them mush + :type a: torch.Tensor + :type b: torch.Tensor + :rtype: torch.Tensor + """ + siz1 = torch.Size(torch.tensor(a.shape[-2:]) * torch.tensor(b.shape[-2:])) + res = a.unsqueeze(-1).unsqueeze(-3) * b.unsqueeze(-2).unsqueeze(-4) + siz0 = res.shape[:-4] + return res.reshape(siz0 + siz1) + + +def get_maximum_order_unary_only(per_layer_orders_and_multiplicities): + """ + determine what spherical harmonics we need to pre-compute. if we have the + unary term only, we need to compare all adjacent layers + + the spherical harmonics function depends on J (irrep order) purely, which is dedfined by + order_irreps = list(range(abs(order_in - order_out), order_in + order_out + 1)) + simplification: we only care about the maximum (in some circumstances that means we calculate a few lower + order spherical harmonics which we won't actually need) + + :param per_layer_orders_and_multiplicities: nested list of lists of 2-tuples + :return: integer indicating maximum order J + """ + + n_layers = len(per_layer_orders_and_multiplicities) + + # extract orders only + per_layer_orders = [] + for i in range(n_layers): + cur = per_layer_orders_and_multiplicities[i] + cur = [o for (m, o) in cur] + per_layer_orders.append(cur) + + track_max = 0 + # compare two (adjacent) layers at a time + for i in range(n_layers - 1): + cur = per_layer_orders[i] + nex = per_layer_orders[i + 1] + track_max = max(max(cur) + max(nex), track_max) + + return track_max + + +def get_maximum_order_with_pairwise(per_layer_orders_and_multiplicities): + """ + determine what spherical harmonics we need to pre-compute. for pairwise + interactions, this will just be twice the maximum order + + the spherical harmonics function depends on J (irrep order) purely, which is defined by + order_irreps = list(range(abs(order_in - order_out), order_in + order_out + 1)) + simplification: we only care about the maximum (in some circumstances that means we calculate a few lower + order spherical harmonics which we won't actually need) + + :param per_layer_orders_and_multiplicities: nested list of lists of 2-tuples + :return: integer indicating maximum order J + """ + + n_layers = len(per_layer_orders_and_multiplicities) + + track_max = 0 + for i in range(n_layers): + cur = per_layer_orders_and_multiplicities[i] + # extract orders only + orders = [o for (m, o) in cur] + track_max = max(track_max, max(orders)) + + return 2*track_max + + +def precompute_sh(r_ij, max_J): + """ + pre-comput spherical harmonics up to order max_J + + :param r_ij: relative positions + :param max_J: maximum order used in entire network + :return: dict where each entry has shape [B,N,K,2J+1] + """ + + i_distance = 0 + i_alpha = 1 + i_beta = 2 + + Y_Js = {} + sh = SphericalHarmonics() + + for J in range(max_J+1): + # dimension [B,N,K,2J+1] + #Y_Js[J] = spherical_harmonics(order=J, alpha=r_ij[...,i_alpha], beta=r_ij[...,i_beta]) + Y_Js[J] = sh.get(J, theta=math.pi-r_ij[...,i_beta], phi=r_ij[...,i_alpha], refresh=False) + + sh.clear() + return Y_Js + + +class ScalarActivation3rdDim(torch.nn.Module): + def __init__(self, n_dim, activation, bias=True): + ''' + Can be used only with scalar fields [B, N, s] on last dimension + + :param n_dim: number of scalar fields to apply activation to + :param bool bias: add a bias before the applying the activation + ''' + super().__init__() + + self.activation = activation + + if bias and n_dim > 0: + self.bias = torch.nn.Parameter(torch.zeros(n_dim)) + else: + self.bias = None + + def forward(self, input): + ''' + :param input: [B, N, s] + ''' + + assert len(np.array(input.shape)) == 3 + + if self.bias is not None: + x = input + self.bias.view(1, 1, -1) + else: + x = input + x = self.activation(x) + + return x diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/modules.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/modules.py new file mode 100755 index 00000000..8a1a7f74 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/equivariant_attention/modules.py @@ -0,0 +1,905 @@ +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from contextlib import nullcontext + +from typing import Dict + +from equivariant_attention.from_se3cnn import utils_steerable +from equivariant_attention.fibers import Fiber, fiber2head +from utils.utils_logging import log_gradient_norm + +import dgl +import dgl.function as fn +from dgl.nn.pytorch.softmax import edge_softmax +from dgl.nn.pytorch.glob import AvgPooling, MaxPooling + +from packaging import version + + +def get_basis(G, max_degree, compute_gradients): + """Precompute the SE(3)-equivariant weight basis, W_J^lk(x) + + This is called by get_basis_and_r(). + + Args: + G: DGL graph instance of type dgl.DGLGraph + max_degree: non-negative int for degree of highest feature type + compute_gradients: boolean, whether to compute gradients during basis construction + Returns: + dict of equivariant bases. Keys are in the form 'd_in,d_out'. Values are + tensors of shape (batch_size, 1, 2*d_out+1, 1, 2*d_in+1, number_of_bases) + where the 1's will later be broadcast to the number of output and input + channels + """ + if compute_gradients: + context = nullcontext() + else: + context = torch.no_grad() + + with context: + cloned_d = torch.clone(G.edata['d']) + + if G.edata['d'].requires_grad: + cloned_d.requires_grad_() + log_gradient_norm(cloned_d, 'Basis computation flow') + + # Relative positional encodings (vector) + r_ij = utils_steerable.get_spherical_from_cartesian_torch(cloned_d) + # Spherical harmonic basis + Y = utils_steerable.precompute_sh(r_ij, 2*max_degree) + device = Y[0].device + + basis = {} + for d_in in range(max_degree+1): + for d_out in range(max_degree+1): + K_Js = [] + for J in range(abs(d_in-d_out), d_in+d_out+1): + # Get spherical harmonic projection matrices + Q_J = utils_steerable._basis_transformation_Q_J(J, d_in, d_out) + Q_J = Q_J.float().T.to(device) + + # Create kernel from spherical harmonics + K_J = torch.matmul(Y[J], Q_J) + K_Js.append(K_J) + + # Reshape so can take linear combinations with a dot product + size = (-1, 1, 2*d_out+1, 1, 2*d_in+1, 2*min(d_in, d_out)+1) + basis[f'{d_in},{d_out}'] = torch.stack(K_Js, -1).view(*size) + return basis + + +def get_r(G): + """Compute internodal distances""" + cloned_d = torch.clone(G.edata['d']) + + if G.edata['d'].requires_grad: + cloned_d.requires_grad_() + log_gradient_norm(cloned_d, 'Neural networks flow') + + return torch.sqrt(torch.sum(cloned_d**2, -1, keepdim=True)) + + +def get_basis_and_r(G, max_degree, compute_gradients=False): + """Return equivariant weight basis (basis) and internodal distances (r). + + Call this function *once* at the start of each forward pass of the model. + It computes the equivariant weight basis, W_J^lk(x), and internodal + distances, needed to compute varphi_J^lk(x), of eqn 8 of + https://arxiv.org/pdf/2006.10503.pdf. The return values of this function + can be shared as input across all SE(3)-Transformer layers in a model. + + Args: + G: DGL graph instance of type dgl.DGLGraph() + max_degree: non-negative int for degree of highest feature-type + compute_gradients: controls whether to compute gradients during basis construction + Returns: + dict of equivariant bases, keys are in form '' + vector of relative distances, ordered according to edge ordering of G + """ + basis = get_basis(G, max_degree, compute_gradients) + r = get_r(G) + return basis, r + + +### SE(3) equivariant operations on graphs in DGL + +class GConvSE3(nn.Module): + """A tensor field network layer as a DGL module. + + GConvSE3 stands for a Graph Convolution SE(3)-equivariant layer. It is the + equivalent of a linear layer in an MLP, a conv layer in a CNN, or a graph + conv layer in a GCN. + + At each node, the activations are split into different "feature types", + indexed by the SE(3) representation type: non-negative integers 0, 1, 2, .. + """ + def __init__(self, f_in, f_out, self_interaction: bool=False, edge_dim: int=0, flavor='skip'): + """SE(3)-equivariant Graph Conv Layer + + Args: + f_in: list of tuples [(multiplicities, type),...] + f_out: list of tuples [(multiplicities, type),...] + self_interaction: include self-interaction in convolution + edge_dim: number of dimensions for edge embedding + flavor: allows ['TFN', 'skip'], where 'skip' adds a skip connection + """ + super().__init__() + self.f_in = f_in + self.f_out = f_out + self.edge_dim = edge_dim + self.self_interaction = self_interaction + self.flavor = flavor + + # Neighbor -> center weights + self.kernel_unary = nn.ModuleDict() + for (mi, di) in self.f_in.structure: + for (mo, do) in self.f_out.structure: + self.kernel_unary[f'({di},{do})'] = PairwiseConv(di, mi, do, mo, edge_dim=edge_dim) + + # Center -> center weights + self.kernel_self = nn.ParameterDict() + if self_interaction: + assert self.flavor in ['TFN', 'skip'] + if self.flavor == 'TFN': + for m_out, d_out in self.f_out.structure: + W = nn.Parameter(torch.randn(1, m_out, m_out) / np.sqrt(m_out)) + self.kernel_self[f'{d_out}'] = W + elif self.flavor == 'skip': + for m_in, d_in in self.f_in.structure: + if d_in in self.f_out.degrees: + m_out = self.f_out.structure_dict[d_in] + W = nn.Parameter(torch.randn(1, m_out, m_in) / np.sqrt(m_in)) + self.kernel_self[f'{d_in}'] = W + + + + def __repr__(self): + return f'GConvSE3(structure={self.f_out}, self_interaction={self.self_interaction})' + + + def udf_u_mul_e(self, d_out): + """Compute the convolution for a single output feature type. + + This function is set up as a User Defined Function in DGL. + + Args: + d_out: output feature type + Returns: + edge -> node function handle + """ + def fnc(edges): + # Neighbor -> center messages + msg = 0 + for m_in, d_in in self.f_in.structure: + src = edges.src[f'{d_in}'].view(-1, m_in*(2*d_in+1), 1) + edge = edges.data[f'({d_in},{d_out})'] + msg = msg + torch.matmul(edge, src) + msg = msg.view(msg.shape[0], -1, 2*d_out+1) + + # Center -> center messages + if self.self_interaction: + if f'{d_out}' in self.kernel_self.keys(): + if self.flavor == 'TFN': + W = self.kernel_self[f'{d_out}'] + msg = torch.matmul(W, msg) + if self.flavor == 'skip': + dst = edges.dst[f'{d_out}'] + W = self.kernel_self[f'{d_out}'] + msg = msg + torch.matmul(W, dst) + + return {'msg': msg.view(msg.shape[0], -1, 2*d_out+1)} + return fnc + + def forward(self, h, G=None, r=None, basis=None, **kwargs): + """Forward pass of the linear layer + + Args: + G: minibatch of (homo)graphs + h: dict of features + r: inter-atomic distances + basis: pre-computed Q * Y + Returns: + tensor with new features [B, n_points, n_features_out] + """ + with G.local_scope(): + # Add node features to local graph scope + for k, v in h.items(): + G.ndata[k] = v + + # Add edge features + if 'w' in G.edata.keys(): + w = G.edata['w'] + feat = torch.cat([w, r], -1) + else: + feat = torch.cat([r, ], -1) + + for (mi, di) in self.f_in.structure: + for (mo, do) in self.f_out.structure: + etype = f'({di},{do})' + G.edata[etype] = self.kernel_unary[etype](feat, basis) + + # Perform message-passing for each output feature type + for d in self.f_out.degrees: + G.update_all(self.udf_u_mul_e(d), fn.mean('msg', f'out{d}')) + + return {f'{d}': G.ndata[f'out{d}'] for d in self.f_out.degrees} + + +class RadialFunc(nn.Module): + """NN parameterized radial profile function.""" + def __init__(self, num_freq, in_dim, out_dim, edge_dim: int=0): + """NN parameterized radial profile function. + + Args: + num_freq: number of output frequencies + in_dim: multiplicity of input (num input channels) + out_dim: multiplicity of output (num output channels) + edge_dim: number of dimensions for edge embedding + """ + super().__init__() + self.num_freq = num_freq + self.in_dim = in_dim + self.mid_dim = 32 + self.out_dim = out_dim + self.edge_dim = edge_dim + + self.net = nn.Sequential(nn.Linear(self.edge_dim+1,self.mid_dim), + BN(self.mid_dim), + nn.ReLU(), + nn.Linear(self.mid_dim,self.mid_dim), + BN(self.mid_dim), + nn.ReLU(), + nn.Linear(self.mid_dim,self.num_freq*in_dim*out_dim)) + + nn.init.kaiming_uniform_(self.net[0].weight) + nn.init.kaiming_uniform_(self.net[3].weight) + nn.init.kaiming_uniform_(self.net[6].weight) + + def __repr__(self): + return f"RadialFunc(edge_dim={self.edge_dim}, in_dim={self.in_dim}, out_dim={self.out_dim})" + + def forward(self, x): + y = self.net(x) + return y.view(-1, self.out_dim, 1, self.in_dim, 1, self.num_freq) + + +class PairwiseConv(nn.Module): + """SE(3)-equivariant convolution between two single-type features""" + def __init__(self, degree_in: int, nc_in: int, degree_out: int, + nc_out: int, edge_dim: int=0): + """SE(3)-equivariant convolution between a pair of feature types. + + This layer performs a convolution from nc_in features of type degree_in + to nc_out features of type degree_out. + + Args: + degree_in: degree of input fiber + nc_in: number of channels on input + degree_out: degree of out order + nc_out: number of channels on output + edge_dim: number of dimensions for edge embedding + """ + super().__init__() + # Log settings + self.degree_in = degree_in + self.degree_out = degree_out + self.nc_in = nc_in + self.nc_out = nc_out + + # Functions of the degree + self.num_freq = 2*min(degree_in, degree_out) + 1 + self.d_out = 2*degree_out + 1 + self.edge_dim = edge_dim + + # Radial profile function + self.rp = RadialFunc(self.num_freq, nc_in, nc_out, self.edge_dim) + + def forward(self, feat, basis): + # Get radial weights + R = self.rp(feat) + kernel = torch.sum(R * basis[f'{self.degree_in},{self.degree_out}'], -1) + return kernel.view(kernel.shape[0], self.d_out*self.nc_out, -1) + + +class G1x1SE3(nn.Module): + """Graph Linear SE(3)-equivariant layer, equivalent to a 1x1 convolution. + + This is equivalent to a self-interaction layer in TensorField Networks. + """ + def __init__(self, f_in, f_out, learnable=True): + """SE(3)-equivariant 1x1 convolution. + + Args: + f_in: input Fiber() of feature multiplicities and types + f_out: output Fiber() of feature multiplicities and types + """ + super().__init__() + self.f_in = f_in + self.f_out = f_out + + # Linear mappings: 1 per output feature type + self.transform = nn.ParameterDict() + for m_out, d_out in self.f_out.structure: + m_in = self.f_in.structure_dict[d_out] + self.transform[str(d_out)] = nn.Parameter(torch.randn(m_out, m_in) / np.sqrt(m_in), requires_grad=learnable) + + def __repr__(self): + return f"G1x1SE3(structure={self.f_out})" + + def forward(self, features, **kwargs): + output = {} + for k, v in features.items(): + if str(k) in self.transform.keys(): + output[k] = torch.matmul(self.transform[str(k)], v) + return output + + +class GNormBias(nn.Module): + """Norm-based SE(3)-equivariant nonlinearity with only learned biases.""" + + def __init__(self, fiber, nonlin=nn.ReLU(inplace=True), + num_layers: int = 0): + """Initializer. + + Args: + fiber: Fiber() of feature multiplicities and types + nonlin: nonlinearity to use everywhere + num_layers: non-negative number of linear layers in fnc + """ + super().__init__() + self.fiber = fiber + self.nonlin = nonlin + self.num_layers = num_layers + + # Regularization for computing phase: gradients explode otherwise + self.eps = 1e-12 + + # Norm mappings: 1 per feature type + self.bias = nn.ParameterDict() + for m, d in self.fiber.structure: + self.bias[str(d)] = nn.Parameter(torch.randn(m).view(1, m)) + + def __repr__(self): + return f"GNormTFN()" + + + def forward(self, features, **kwargs): + output = {} + for k, v in features.items(): + # Compute the norms and normalized features + # v shape: [...,m , 2*k+1] + norm = v.norm(2, -1, keepdim=True).clamp_min(self.eps).expand_as(v) + phase = v / norm + + # Transform on norms + # transformed = self.transform[str(k)](norm[..., 0]).unsqueeze(-1) + transformed = self.nonlin(norm[..., 0] + self.bias[str(k)]) + + # Nonlinearity on norm + output[k] = (transformed.unsqueeze(-1) * phase).view(*v.shape) + + return output + + +class GAttentiveSelfInt(nn.Module): + + def __init__(self, f_in, f_out): + """SE(3)-equivariant 1x1 convolution. + + Args: + f_in: input Fiber() of feature multiplicities and types + f_out: output Fiber() of feature multiplicities and types + """ + super().__init__() + self.f_in = f_in + self.f_out = f_out + self.nonlin = nn.LeakyReLU() + self.num_layers = 2 + self.eps = 1e-12 # regularisation for phase: gradients explode otherwise + + # one network for attention weights per degree + self.transform = nn.ModuleDict() + for o, m_in in self.f_in.structure_dict.items(): + m_out = self.f_out.structure_dict[o] + self.transform[str(o)] = self._build_net(m_in, m_out) + + def __repr__(self): + return f"AttentiveSelfInteractionSE3(in={self.f_in}, out={self.f_out})" + + def _build_net(self, m_in: int, m_out): + n_hidden = m_in * m_out + cur_inpt = m_in * m_in + net = [] + for i in range(1, self.num_layers): + net.append(nn.LayerNorm(int(cur_inpt))) + net.append(self.nonlin) + # TODO: implement cleaner init + net.append( + nn.Linear(cur_inpt, n_hidden, bias=(i == self.num_layers - 1))) + nn.init.kaiming_uniform_(net[-1].weight) + cur_inpt = n_hidden + return nn.Sequential(*net) + + def forward(self, features, **kwargs): + output = {} + for k, v in features.items(): + # v shape: [..., m, 2*k+1] + first_dims = v.shape[:-2] + m_in = self.f_in.structure_dict[int(k)] + m_out = self.f_out.structure_dict[int(k)] + assert v.shape[-2] == m_in + assert v.shape[-1] == 2 * int(k) + 1 + + # Compute the norms and normalized features + #norm = v.norm(p=2, dim=-1, keepdim=True).clamp_min(self.eps).expand_as(v) + #phase = v / norm # [..., m, 2*k+1] + scalars = torch.einsum('...ac,...bc->...ab', [v, v]) # [..., m_in, m_in] + scalars = scalars.view(*first_dims, m_in*m_in) # [..., m_in*m_in] + sign = scalars.sign() + scalars = scalars.abs_().clamp_min(self.eps) + scalars *= sign + + # perform attention + att_weights = self.transform[str(k)](scalars) # [..., m_out*m_in] + att_weights = att_weights.view(*first_dims, m_out, m_in) # [..., m_out, m_in] + att_weights = F.softmax(input=att_weights, dim=-1) + # shape [..., m_out, 2*k+1] + # output[k] = torch.einsum('...nm,...md->...nd', [att_weights, phase]) + output[k] = torch.einsum('...nm,...md->...nd', [att_weights, v]) + + return output + + + +class GNormSE3(nn.Module): + """Graph Norm-based SE(3)-equivariant nonlinearity. + + Nonlinearities are important in SE(3) equivariant GCNs. They are also quite + expensive to compute, so it is convenient for them to share resources with + other layers, such as normalization. The general workflow is as follows: + + > for feature type in features: + > norm, phase <- feature + > output = fnc(norm) * phase + + where fnc: {R+}^m -> R^m is a learnable map from m norms to m scalars. + """ + def __init__(self, fiber, nonlin=nn.ReLU(inplace=True), num_layers: int=0): + """Initializer. + + Args: + fiber: Fiber() of feature multiplicities and types + nonlin: nonlinearity to use everywhere + num_layers: non-negative number of linear layers in fnc + """ + super().__init__() + self.fiber = fiber + self.nonlin = nonlin + self.num_layers = num_layers + + # Regularization for computing phase: gradients explode otherwise + self.eps = 1e-12 + + # Norm mappings: 1 per feature type + self.transform = nn.ModuleDict() + for m, d in self.fiber.structure: + self.transform[str(d)] = self._build_net(int(m)) + + def __repr__(self): + return f"GNormSE3(num_layers={self.num_layers}, nonlin={self.nonlin})" + + def _build_net(self, m: int): + net = [] + for i in range(self.num_layers): + net.append(BN(int(m))) + net.append(self.nonlin) + # TODO: implement cleaner init + net.append(nn.Linear(m, m, bias=(i==self.num_layers-1))) + nn.init.kaiming_uniform_(net[-1].weight) + if self.num_layers == 0: + net.append(BN(int(m))) + net.append(self.nonlin) + return nn.Sequential(*net) + + def forward(self, features, **kwargs): + output = {} + for k, v in features.items(): + # Compute the norms and normalized features + # v shape: [...,m , 2*k+1] + norm = v.norm(2, -1, keepdim=True).clamp_min(self.eps).expand_as(v) + phase = v / norm + + # Transform on norms + transformed = self.transform[str(k)](norm[...,0]).unsqueeze(-1) + + # Nonlinearity on norm + output[k] = (transformed * phase).view(*v.shape) + + return output + + +class BN(nn.Module): + """SE(3)-equvariant batch/layer normalization""" + def __init__(self, m): + """SE(3)-equvariant batch/layer normalization + + Args: + m: int for number of output channels + """ + super().__init__() + self.bn = nn.LayerNorm(m) + + def forward(self, x): + return self.bn(x) + + +class GConvSE3Partial(nn.Module): + """Graph SE(3)-equivariant node -> edge layer""" + def __init__(self, f_in, f_out, edge_dim: int=0, x_ij=None): + """SE(3)-equivariant partial convolution. + + A partial convolution computes the inner product between a kernel and + each input channel, without summing over the result from each input + channel. This unfolded structure makes it amenable to be used for + computing the value-embeddings of the attention mechanism. + + Args: + f_in: list of tuples [(multiplicities, type),...] + f_out: list of tuples [(multiplicities, type),...] + """ + super().__init__() + self.f_out = f_out + self.edge_dim = edge_dim + + # adding/concatinating relative position to feature vectors + # 'cat' concatenates relative position & existing feature vector + # 'add' adds it, but only if multiplicity > 1 + assert x_ij in [None, 'cat', 'add'] + self.x_ij = x_ij + if x_ij == 'cat': + self.f_in = Fiber.combine(f_in, Fiber(structure=[(1,1)])) + else: + self.f_in = f_in + + # Node -> edge weights + self.kernel_unary = nn.ModuleDict() + for (mi, di) in self.f_in.structure: + for (mo, do) in self.f_out.structure: + self.kernel_unary[f'({di},{do})'] = PairwiseConv(di, mi, do, mo, edge_dim=edge_dim) + + def __repr__(self): + return f'GConvSE3Partial(structure={self.f_out})' + + def udf_u_mul_e(self, d_out): + """Compute the partial convolution for a single output feature type. + + This function is set up as a User Defined Function in DGL. + + Args: + d_out: output feature type + Returns: + node -> edge function handle + """ + def fnc(edges): + # Neighbor -> center messages + msg = 0 + for m_in, d_in in self.f_in.structure: + # if type 1 and flag set, add relative position as feature + if self.x_ij == 'cat' and d_in == 1: + # relative positions + rel = (edges.dst['x'] - edges.src['x']).view(-1, 3, 1) + m_ori = m_in - 1 + if m_ori == 0: + # no type 1 input feature, just use relative position + src = rel + else: + # features of src node, shape [edges, m_in*(2l+1), 1] + src = edges.src[f'{d_in}'].view(-1, m_ori*(2*d_in+1), 1) + # add to feature vector + src = torch.cat([src, rel], dim=1) + elif self.x_ij == 'add' and d_in == 1 and m_in > 1: + src = edges.src[f'{d_in}'].view(-1, m_in*(2*d_in+1), 1) + rel = (edges.dst['x'] - edges.src['x']).view(-1, 3, 1) + src[..., :3, :1] = src[..., :3, :1] + rel + else: + src = edges.src[f'{d_in}'].view(-1, m_in*(2*d_in+1), 1) + edge = edges.data[f'({d_in},{d_out})'] + msg = msg + torch.matmul(edge, src) + msg = msg.view(msg.shape[0], -1, 2*d_out+1) + + return {f'out{d_out}': msg.view(msg.shape[0], -1, 2*d_out+1)} + return fnc + + def forward(self, h, G=None, r=None, basis=None, **kwargs): + """Forward pass of the linear layer + + Args: + h: dict of node-features + G: minibatch of (homo)graphs + r: inter-atomic distances + basis: pre-computed Q * Y + Returns: + tensor with new features [B, n_points, n_features_out] + """ + with G.local_scope(): + # Add node features to local graph scope + for k, v in h.items(): + G.ndata[k] = v + + # Add edge features + if 'w' in G.edata.keys(): + w = G.edata['w'] # shape: [#edges_in_batch, #bond_types] + feat = torch.cat([w, r], -1) + else: + feat = torch.cat([r, ], -1) + for (mi, di) in self.f_in.structure: + for (mo, do) in self.f_out.structure: + etype = f'({di},{do})' + G.edata[etype] = self.kernel_unary[etype](feat, basis) + + # Perform message-passing for each output feature type + for d in self.f_out.degrees: + G.apply_edges(self.udf_u_mul_e(d)) + + return {f'{d}': G.edata[f'out{d}'] for d in self.f_out.degrees} + + +class GMABSE3(nn.Module): + """An SE(3)-equivariant multi-headed self-attention module for DGL graphs.""" + def __init__(self, f_value: Fiber, f_key: Fiber, n_heads: int): + """SE(3)-equivariant MAB (multi-headed attention block) layer. + + Args: + f_value: Fiber() object for value-embeddings + f_key: Fiber() object for key-embeddings + n_heads: number of heads + """ + super().__init__() + self.f_value = f_value + self.f_key = f_key + self.n_heads = n_heads + self.new_dgl = version.parse(dgl.__version__) > version.parse('0.4.4') + + def __repr__(self): + return f'GMABSE3(n_heads={self.n_heads}, structure={self.f_value})' + + def udf_u_mul_e(self, d_out): + """Compute the weighted sum for a single output feature type. + + This function is set up as a User Defined Function in DGL. + + Args: + d_out: output feature type + Returns: + edge -> node function handle + """ + def fnc(edges): + # Neighbor -> center messages + attn = edges.data['a'] + value = edges.data[f'v{d_out}'] + + # Apply attention weights + msg = attn.unsqueeze(-1).unsqueeze(-1) * value + + return {'m': msg} + return fnc + + def forward(self, v, k: Dict=None, q: Dict=None, G=None, **kwargs): + """Forward pass of the linear layer + + Args: + G: minibatch of (homo)graphs + v: dict of value edge-features + k: dict of key edge-features + q: dict of query node-features + Returns: + tensor with new features [B, n_points, n_features_out] + """ + with G.local_scope(): + # Add node features to local graph scope + ## We use the stacked tensor representation for attention + for m, d in self.f_value.structure: + G.edata[f'v{d}'] = v[f'{d}'].view(-1, self.n_heads, m//self.n_heads, 2*d+1) + G.edata['k'] = fiber2head(k, self.n_heads, self.f_key, squeeze=True) # [edges, heads, channels](?) + G.ndata['q'] = fiber2head(q, self.n_heads, self.f_key, squeeze=True) # [nodes, heads, channels](?) + + # Compute attention weights + ## Inner product between (key) neighborhood and (query) center + G.apply_edges(fn.e_dot_v('k', 'q', 'e')) + + ## Apply softmax + e = G.edata.pop('e') + if self.new_dgl: + # in dgl 5.3, e has an extra dimension compared to dgl 4.3 + # the following, we get rid of this be reshaping + n_edges = G.edata['k'].shape[0] + e = e.view([n_edges, self.n_heads]) + e = e / np.sqrt(self.f_key.n_features) + G.edata['a'] = edge_softmax(G, e) + + # Perform attention-weighted message-passing + for d in self.f_value.degrees: + G.update_all(self.udf_u_mul_e(d), fn.sum('m', f'out{d}')) + + output = {} + for m, d in self.f_value.structure: + output[f'{d}'] = G.ndata[f'out{d}'].view(-1, m, 2*d+1) + + return output + + +class GSE3Res(nn.Module): + """Graph attention block with SE(3)-equivariance and skip connection""" + def __init__(self, f_in: Fiber, f_out: Fiber, edge_dim: int=0, div: float=4, + n_heads: int=1, learnable_skip=True, skip='cat', selfint='1x1', x_ij=None): + super().__init__() + self.f_in = f_in + self.f_out = f_out + self.div = div + self.n_heads = n_heads + self.skip = skip # valid: 'cat', 'sum', None + + # f_mid_out has same structure as 'f_out' but #channels divided by 'div' + # this will be used for the values + f_mid_out = {k: int(v // div) for k, v in self.f_out.structure_dict.items()} + self.f_mid_out = Fiber(dictionary=f_mid_out) + + # f_mid_in has same structure as f_mid_out, but only degrees which are in f_in + # this will be used for keys and queries + # (queries are merely projected, hence degrees have to match input) + f_mid_in = {d: m for d, m in f_mid_out.items() if d in self.f_in.degrees} + self.f_mid_in = Fiber(dictionary=f_mid_in) + + self.edge_dim = edge_dim + + self.GMAB = nn.ModuleDict() + + # Projections + self.GMAB['v'] = GConvSE3Partial(f_in, self.f_mid_out, edge_dim=edge_dim, x_ij=x_ij) + self.GMAB['k'] = GConvSE3Partial(f_in, self.f_mid_in, edge_dim=edge_dim, x_ij=x_ij) + self.GMAB['q'] = G1x1SE3(f_in, self.f_mid_in) + + # Attention + self.GMAB['attn'] = GMABSE3(self.f_mid_out, self.f_mid_in, n_heads=n_heads) + + # Skip connections + if self.skip == 'cat': + self.cat = GCat(self.f_mid_out, f_in) + if selfint == 'att': + self.project = GAttentiveSelfInt(self.cat.f_out, f_out) + elif selfint == '1x1': + self.project = G1x1SE3(self.cat.f_out, f_out, learnable=learnable_skip) + elif self.skip == 'sum': + self.project = G1x1SE3(self.f_mid_out, f_out, learnable=learnable_skip) + self.add = GSum(f_out, f_in) + # the following checks whether the skip connection would change + # the output fibre strucure; the reason can be that the input has + # more channels than the ouput (for at least one degree); this would + # then cause a (hard to debug) error in the next layer + assert self.add.f_out.structure_dict == f_out.structure_dict, \ + 'skip connection would change output structure' + + def forward(self, features, G, **kwargs): + # Embeddings + v = self.GMAB['v'](features, G=G, **kwargs) + k = self.GMAB['k'](features, G=G, **kwargs) + q = self.GMAB['q'](features, G=G) + + # Attention + z = self.GMAB['attn'](v, k=k, q=q, G=G) + + if self.skip == 'cat': + z = self.cat(z, features) + z = self.project(z) + elif self.skip == 'sum': + # Skip + residual + z = self.project(z) + z = self.add(z, features) + return z + +### Helper and wrapper functions + +class GSum(nn.Module): + """SE(3)-equvariant graph residual sum function.""" + def __init__(self, f_x: Fiber, f_y: Fiber): + """SE(3)-equvariant graph residual sum function. + + Args: + f_x: Fiber() object for fiber of summands + f_y: Fiber() object for fiber of summands + """ + super().__init__() + self.f_x = f_x + self.f_y = f_y + self.f_out = Fiber.combine_max(f_x, f_y) + + def __repr__(self): + return f"GSum(structure={self.f_out})" + + def forward(self, x, y): + out = {} + for k in self.f_out.degrees: + k = str(k) + if (k in x) and (k in y): + if x[k].shape[1] > y[k].shape[1]: + diff = x[k].shape[1] - y[k].shape[1] + zeros = torch.zeros(x[k].shape[0], diff, x[k].shape[2]).to(y[k].device) + y[k] = torch.cat([y[k], zeros], 1) + elif x[k].shape[1] < y[k].shape[1]: + diff = y[k].shape[1] - x[k].shape[1] + zeros = torch.zeros(x[k].shape[0], diff, x[k].shape[2]).to(y[k].device) + x[k] = torch.cat([x[k], zeros], 1) + + out[k] = x[k] + y[k] + elif k in x: + out[k] = x[k] + elif k in y: + out[k] = y[k] + return out + + +class GCat(nn.Module): + """Concat only degrees which are in f_x""" + def __init__(self, f_x: Fiber, f_y: Fiber): + super().__init__() + self.f_x = f_x + self.f_y = f_y + f_out = {} + for k in f_x.degrees: + f_out[k] = f_x.dict[k] + if k in f_y.degrees: + f_out[k] += f_y.dict[k] + self.f_out = Fiber(dictionary=f_out) + + def __repr__(self): + return f"GCat(structure={self.f_out})" + + def forward(self, x, y): + out = {} + for k in self.f_out.degrees: + k = str(k) + if k in y: + out[k] = torch.cat([x[k], y[k]], 1) + else: + out[k] = x[k] + return out + + +class GAvgPooling(nn.Module): + """Graph Average Pooling module.""" + def __init__(self, type='0'): + super().__init__() + self.pool = AvgPooling() + self.type = type + + def forward(self, features, G, **kwargs): + if self.type == '0': + h = features['0'][...,-1] + pooled = self.pool(G, h) + elif self.type == '1': + pooled = [] + for i in range(3): + h_i = features['1'][..., i] + pooled.append(self.pool(G, h_i).unsqueeze(-1)) + pooled = torch.cat(pooled, axis=-1) + pooled = {'1': pooled} + else: + print('GAvgPooling for type > 0 not implemented') + exit() + return pooled + + +class GMaxPooling(nn.Module): + """Graph Max Pooling module.""" + def __init__(self): + super().__init__() + self.pool = MaxPooling() + + def forward(self, features, G, **kwargs): + h = features['0'][...,-1] + return self.pool(G, h) + + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/ffindex.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/ffindex.py new file mode 100644 index 00000000..16ffc3c9 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/ffindex.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# https://raw.githubusercontent.com/ahcm/ffindex/master/python/ffindex.py + +''' +Created on Apr 30, 2014 + +@author: meiermark +''' + + +import sys +import mmap +from collections import namedtuple + +FFindexEntry = namedtuple("FFindexEntry", "name, offset, length") + + +def read_index(ffindex_filename): + entries = [] + + fh = open(ffindex_filename) + for line in fh: + tokens = line.split("\t") + entries.append(FFindexEntry(tokens[0], int(tokens[1]), int(tokens[2]))) + fh.close() + + return entries + + +def read_data(ffdata_filename): + fh = open(ffdata_filename, "rb") + data = mmap.mmap(fh.fileno(), 0, prot=mmap.PROT_READ) + fh.close() + return data + + +def get_entry_by_name(name, index): + #TODO: bsearch + for entry in index: + if(name == entry.name): + return entry + return None + + +def read_entry_lines(entry, data): + lines = data[entry.offset:entry.offset + entry.length - 1].decode("utf-8").split("\n") + return lines + + +def read_entry_data(entry, data): + return data[entry.offset:entry.offset + entry.length - 1] + + +def write_entry(entries, data_fh, entry_name, offset, data): + data_fh.write(data[:-1]) + data_fh.write(bytearray(1)) + + entry = FFindexEntry(entry_name, offset, len(data)) + entries.append(entry) + + return offset + len(data) + + +def write_entry_with_file(entries, data_fh, entry_name, offset, file_name): + with open(file_name, "rb") as fh: + data = bytearray(fh.read()) + return write_entry(entries, data_fh, entry_name, offset, data) + + +def finish_db(entries, ffindex_filename, data_fh): + data_fh.close() + write_entries_to_db(entries, ffindex_filename) + + +def write_entries_to_db(entries, ffindex_filename): + sorted(entries, key=lambda x: x.name) + index_fh = open(ffindex_filename, "w") + + for entry in entries: + index_fh.write("{name:.64}\t{offset}\t{length}\n".format(name=entry.name, offset=entry.offset, length=entry.length)) + + index_fh.close() + + +def write_entry_to_file(entry, data, file): + lines = read_lines(entry, data) + + fh = open(file, "w") + for line in lines: + fh.write(line+"\n") + fh.close() diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/kinematics.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/kinematics.py new file mode 100644 index 00000000..b7fd2bad --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/kinematics.py @@ -0,0 +1,216 @@ +import numpy as np +import torch + +PARAMS = { + "DMIN" : 2.0, + "DMAX" : 20.0, + "DBINS" : 36, + "ABINS" : 36, +} + +# ============================================================ +def get_pair_dist(a, b): + """calculate pair distances between two sets of points + + Parameters + ---------- + a,b : pytorch tensors of shape [batch,nres,3] + store Cartesian coordinates of two sets of atoms + Returns + ------- + dist : pytorch tensor of shape [batch,nres,nres] + stores paitwise distances between atoms in a and b + """ + + dist = torch.cdist(a, b, p=2) + return dist + +# ============================================================ +def get_ang(a, b, c): + """calculate planar angles for all consecutive triples (a[i],b[i],c[i]) + from Cartesian coordinates of three sets of atoms a,b,c + + Parameters + ---------- + a,b,c : pytorch tensors of shape [batch,nres,3] + store Cartesian coordinates of three sets of atoms + Returns + ------- + ang : pytorch tensor of shape [batch,nres] + stores resulting planar angles + """ + v = a - b + w = c - b + v /= torch.norm(v, dim=-1, keepdim=True) + w /= torch.norm(w, dim=-1, keepdim=True) + vw = torch.sum(v*w, dim=-1) + + return torch.acos(vw) + +# ============================================================ +def get_dih(a, b, c, d): + """calculate dihedral angles for all consecutive quadruples (a[i],b[i],c[i],d[i]) + given Cartesian coordinates of four sets of atoms a,b,c,d + + Parameters + ---------- + a,b,c,d : pytorch tensors of shape [batch,nres,3] + store Cartesian coordinates of four sets of atoms + Returns + ------- + dih : pytorch tensor of shape [batch,nres] + stores resulting dihedrals + """ + b0 = a - b + b1 = c - b + b2 = d - c + + b1 /= torch.norm(b1, dim=-1, keepdim=True) + + v = b0 - torch.sum(b0*b1, dim=-1, keepdim=True)*b1 + w = b2 - torch.sum(b2*b1, dim=-1, keepdim=True)*b1 + + x = torch.sum(v*w, dim=-1) + y = torch.sum(torch.cross(b1,v,dim=-1)*w, dim=-1) + + return torch.atan2(y, x) + + +# ============================================================ +def xyz_to_c6d(xyz, params=PARAMS): + """convert cartesian coordinates into 2d distance + and orientation maps + + Parameters + ---------- + xyz : pytorch tensor of shape [batch,nres,3,3] + stores Cartesian coordinates of backbone N,Ca,C atoms + Returns + ------- + c6d : pytorch tensor of shape [batch,nres,nres,4] + stores stacked dist,omega,theta,phi 2D maps + """ + + batch = xyz.shape[0] + nres = xyz.shape[1] + + # three anchor atoms + N = xyz[:,:,0] + Ca = xyz[:,:,1] + C = xyz[:,:,2] + + # recreate Cb given N,Ca,C + b = Ca - N + c = C - Ca + a = torch.cross(b, c, dim=-1) + Cb = -0.58273431*a + 0.56802827*b - 0.54067466*c + Ca + + # 6d coordinates order: (dist,omega,theta,phi) + c6d = torch.zeros([batch,nres,nres,4],dtype=xyz.dtype,device=xyz.device) + + dist = get_pair_dist(Cb,Cb) + dist[torch.isnan(dist)] = 999.9 + c6d[...,0] = dist + 999.9*torch.eye(nres,device=xyz.device)[None,...] + b,i,j = torch.where(c6d[...,0]=params['DMAX']] = 999.9 + + mask = torch.zeros((batch, nres,nres), dtype=xyz.dtype, device=xyz.device) + mask[b,i,j] = 1.0 + return c6d, mask + +def xyz_to_t2d(xyz_t, t0d, params=PARAMS): + """convert template cartesian coordinates into 2d distance + and orientation maps + + Parameters + ---------- + xyz_t : pytorch tensor of shape [batch,templ,nres,3,3] + stores Cartesian coordinates of template backbone N,Ca,C atoms + t0d: 0-D template features (HHprob, seqID, similarity) [batch, templ, 3] + + Returns + ------- + t2d : pytorch tensor of shape [batch,nres,nres,1+6+3] + stores stacked dist,omega,theta,phi 2D maps + """ + B, T, L = xyz_t.shape[:3] + c6d, mask = xyz_to_c6d(xyz_t.view(B*T,L,3,3), params=params) + c6d = c6d.view(B, T, L, L, 4) + mask = mask.view(B, T, L, L, 1) + # + dist = c6d[...,:1]*mask / params['DMAX'] # from 0 to 1 # (B, T, L, L, 1) + dist = torch.clamp(dist, 0.0, 1.0) + orien = torch.cat((torch.sin(c6d[...,1:]), torch.cos(c6d[...,1:])), dim=-1)*mask # (B, T, L, L, 6) + t0d = t0d.unsqueeze(2).unsqueeze(3).expand(-1, -1, L, L, -1) + # + t2d = torch.cat((dist, orien, t0d), dim=-1) + t2d[torch.isnan(t2d)] = 0.0 + return t2d + +# ============================================================ +def c6d_to_bins(c6d,params=PARAMS): + """bin 2d distance and orientation maps + """ + + dstep = (params['DMAX'] - params['DMIN']) / params['DBINS'] + astep = 2.0*np.pi / params['ABINS'] + + dbins = torch.linspace(params['DMIN']+dstep, params['DMAX'], params['DBINS'],dtype=c6d.dtype,device=c6d.device) + ab360 = torch.linspace(-np.pi+astep, np.pi, params['ABINS'],dtype=c6d.dtype,device=c6d.device) + ab180 = torch.linspace(astep, np.pi, params['ABINS']//2,dtype=c6d.dtype,device=c6d.device) + + db = torch.bucketize(c6d[...,0].contiguous(),dbins) + ob = torch.bucketize(c6d[...,1].contiguous(),ab360) + tb = torch.bucketize(c6d[...,2].contiguous(),ab360) + pb = torch.bucketize(c6d[...,3].contiguous(),ab180) + + ob[db==params['DBINS']] = params['ABINS'] + tb[db==params['DBINS']] = params['ABINS'] + pb[db==params['DBINS']] = params['ABINS']//2 + + return torch.stack([db,ob,tb,pb],axis=-1).to(torch.uint8) + + +# ============================================================ +def dist_to_bins(dist,params=PARAMS): + """bin 2d distance maps + """ + + dstep = (params['DMAX'] - params['DMIN']) / params['DBINS'] + db = torch.round((dist-params['DMIN']-dstep/2)/dstep) + + db[db<0] = 0 + db[db>params['DBINS']] = params['DBINS'] + + return db.long() + + +# ============================================================ +def c6d_to_bins2(c6d,params=PARAMS): + """bin 2d distance and orientation maps + """ + + dstep = (params['DMAX'] - params['DMIN']) / params['DBINS'] + astep = 2.0*np.pi / params['ABINS'] + + db = torch.round((c6d[...,0]-params['DMIN']-dstep/2)/dstep) + ob = torch.round((c6d[...,1]+np.pi-astep/2)/astep) + tb = torch.round((c6d[...,2]+np.pi-astep/2)/astep) + pb = torch.round((c6d[...,3]-astep/2)/astep) + + # put all dparams['DBINS']] = params['DBINS'] + ob[db==params['DBINS']] = params['ABINS'] + tb[db==params['DBINS']] = params['ABINS'] + pb[db==params['DBINS']] = params['ABINS']//2 + + return torch.stack([db,ob,tb,pb],axis=-1).long() diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/parsers.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/parsers.py new file mode 100644 index 00000000..4d117838 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/parsers.py @@ -0,0 +1,255 @@ +import numpy as np +import scipy +import scipy.spatial +import string +import os,re +import random +import util +import torch +from ffindex import * + +to1letter = { + "ALA":'A', "ARG":'R', "ASN":'N', "ASP":'D', "CYS":'C', + "GLN":'Q', "GLU":'E', "GLY":'G', "HIS":'H', "ILE":'I', + "LEU":'L', "LYS":'K', "MET":'M', "PHE":'F', "PRO":'P', + "SER":'S', "THR":'T', "TRP":'W', "TYR":'Y', "VAL":'V' } + +# read A3M and convert letters into +# integers in the 0..20 range, +def parse_a3m(filename): + + msa = [] + + table = str.maketrans(dict.fromkeys(string.ascii_lowercase)) + + # read file line by line + for line in open(filename,"r"): + + # skip labels + if line[0] == '>': + continue + + # remove right whitespaces + line = line.rstrip() + + # remove lowercase letters and append to MSA + msa.append(line.translate(table)) + + # convert letters into numbers + alphabet = np.array(list("ARNDCQEGHILKMFPSTWYV-"), dtype='|S1').view(np.uint8) + msa = np.array([list(s) for s in msa], dtype='|S1').view(np.uint8) + for i in range(alphabet.shape[0]): + msa[msa == alphabet[i]] = i + + # treat all unknown characters as gaps + msa[msa > 20] = 20 + + return msa + +# parse HHsearch output +def parse_hhr(filename, ffindex, idmax=105.0): + + # labels present in the database + label_set = set([i.name for i in ffindex]) + + out = [] + + with open(filename, "r") as hhr: + + # read .hhr into a list of lines + lines = [s.rstrip() for _,s in enumerate(hhr)] + + # read list of all hits + start = lines.index("") + 2 + stop = lines[start:].index("") + start + hits = [] + for line in lines[start:stop]: + + # ID of the hit + #label = re.sub('_','',line[4:10].strip()) + label = line[4:10].strip() + + # position in the query where the alignment starts + qstart = int(line[75:84].strip().split("-")[0])-1 + + # position in the template where the alignment starts + tstart = int(line[85:94].strip().split("-")[0])-1 + + hits.append([label, qstart, tstart, int(line[69:75])]) + + # get line numbers where each hit starts + start = [i for i,l in enumerate(lines) if l and l[0]==">"] # and l[1:].strip() in label_set] + + # process hits + for idx,i in enumerate(start): + + # skip if hit is too short + if hits[idx][3] < 10: + continue + + # skip if template is not in the database + if hits[idx][0] not in label_set: + continue + + # get hit statistics + p,e,s,_,seqid,sim,_,neff = [float(s) for s in re.sub('[=%]', ' ', lines[i+1]).split()[1::2]] + + # skip too similar hits + if seqid > idmax: + continue + + query = np.array(list(lines[i+4].split()[3]), dtype='|S1') + tmplt = np.array(list(lines[i+8].split()[3]), dtype='|S1') + + simlr = np.array(list(lines[i+6][22:]), dtype='|S1').view(np.uint8) + abc = np.array(list(" =-.+|"), dtype='|S1').view(np.uint8) + for k in range(abc.shape[0]): + simlr[simlr == abc[k]] = k + + confd = np.array(list(lines[i+11][22:]), dtype='|S1').view(np.uint8) + abc = np.array(list(" 0123456789"), dtype='|S1').view(np.uint8) + for k in range(abc.shape[0]): + confd[confd == abc[k]] = k + + qj = np.cumsum(query!=b'-') + hits[idx][1] + tj = np.cumsum(tmplt!=b'-') + hits[idx][2] + + # matched positions + matches = np.array([[q-1,t-1,s-1,c-1] for q,t,s,c in zip(qj,tj,simlr,confd) if s>0]) + + # skip short hits + ncol = matches.shape[0] + if ncol<10: + continue + + # save hit + #out.update({hits[idx][0] : [matches,p/100,seqid/100,neff/10]}) + out.append([hits[idx][0],matches,p/100,seqid/100,sim/10]) + + return out + +# read and extract xyz coords of N,Ca,C atoms +# from a PDB file +def parse_pdb(filename): + + lines = open(filename,'r').readlines() + + N = np.array([[float(l[30:38]), float(l[38:46]), float(l[46:54])] + for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="N"]) + Ca = np.array([[float(l[30:38]), float(l[38:46]), float(l[46:54])] + for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="CA"]) + C = np.array([[float(l[30:38]), float(l[38:46]), float(l[46:54])] + for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="C"]) + + xyz = np.stack([N,Ca,C], axis=0) + + # indices of residues observed in the structure + idx = np.array([int(l[22:26]) for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="CA"]) + + return xyz,idx + +def parse_pdb_lines(lines): + + # indices of residues observed in the structure + idx_s = [int(l[22:26]) for l in lines if l[:4]=="ATOM" and l[12:16].strip()=="CA"] + + # 4 BB + up to 10 SC atoms + xyz = np.full((len(idx_s), 14, 3), np.nan, dtype=np.float32) + for l in lines: + if l[:4] != "ATOM": + continue + resNo, atom, aa = int(l[22:26]), l[12:16], l[17:20] + idx = idx_s.index(resNo) + for i_atm, tgtatm in enumerate(util.aa2long[util.aa2num[aa]]): + if tgtatm == atom: + xyz[idx,i_atm,:] = [float(l[30:38]), float(l[38:46]), float(l[46:54])] + break + + # save atom mask + mask = np.logical_not(np.isnan(xyz[...,0])) + xyz[np.isnan(xyz[...,0])] = 0.0 + + return xyz,mask,np.array(idx_s) + +def parse_templates(ffdb, hhr_fn, atab_fn, n_templ=10): + + # process tabulated hhsearch output to get + # matched positions and positional scores + infile = atab_fn + hits = [] + for l in open(infile, "r").readlines(): + if l[0]=='>': + key = l[1:].split()[0] + hits.append([key,[],[]]) + elif "score" in l or "dssp" in l: + continue + else: + hi = l.split()[:5]+[0.0,0.0,0.0] + hits[-1][1].append([int(hi[0]),int(hi[1])]) + hits[-1][2].append([float(hi[2]),float(hi[3]),float(hi[4])]) + + # get per-hit statistics from an .hhr file + # (!!! assume that .hhr and .atab have the same hits !!!) + # [Probab, E-value, Score, Aligned_cols, + # Identities, Similarity, Sum_probs, Template_Neff] + lines = open(hhr_fn, "r").readlines() + pos = [i+1 for i,l in enumerate(lines) if l[0]=='>'] + for i,posi in enumerate(pos): + hits[i].append([float(s) for s in re.sub('[=%]',' ',lines[posi]).split()[1::2]]) + + # parse templates from FFDB + for hi in hits: + #if hi[0] not in ffids: + # continue + entry = get_entry_by_name(hi[0], ffdb.index) + if entry == None: + continue + data = read_entry_lines(entry, ffdb.data) + hi += list(parse_pdb_lines(data)) + + # process hits + counter = 0 + xyz,qmap,mask,f0d,f1d,ids = [],[],[],[],[],[] + for data in hits: + if len(data)<7: + continue + + qi,ti = np.array(data[1]).T + _,sel1,sel2 = np.intersect1d(ti, data[6], return_indices=True) + ncol = sel1.shape[0] + if ncol < 10: + continue + + ids.append(data[0]) + f0d.append(data[3]) + f1d.append(np.array(data[2])[sel1]) + xyz.append(data[4][sel2]) + mask.append(data[5][sel2]) + qmap.append(np.stack([qi[sel1]-1,[counter]*ncol],axis=-1)) + counter += 1 + + xyz = np.vstack(xyz).astype(np.float32) + qmap = np.vstack(qmap).astype(np.long) + f0d = np.vstack(f0d).astype(np.float32) + f1d = np.vstack(f1d).astype(np.float32) + ids = ids + + return torch.from_numpy(xyz), torch.from_numpy(qmap), \ + torch.from_numpy(f0d), torch.from_numpy(f1d), ids + +def read_templates(qlen, ffdb, hhr_fn, atab_fn, n_templ=10): + xyz_t, qmap, t0d, t1d, ids = parse_templates(ffdb, hhr_fn, atab_fn) + npick = min(n_templ, len(ids)) + sample = torch.arange(npick) + # + xyz = torch.full((npick, qlen, 3, 3), np.nan).float() + f1d = torch.zeros((npick, qlen, 3)).float() + f0d = list() + # + for i, nt in enumerate(sample): + sel = torch.where(qmap[:,1] == nt)[0] + pos = qmap[sel, 0] + xyz[i, pos] = xyz_t[sel, :3] + f1d[i, pos] = t1d[sel, :3] + f0d.append(torch.stack([t0d[nt,0]/100.0, t0d[nt, 4]/100.0, t0d[nt,5]], dim=-1)) + return xyz, f1d, torch.stack(f0d, dim=0) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/performer_pytorch.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/performer_pytorch.py new file mode 100644 index 00000000..61149b79 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/performer_pytorch.py @@ -0,0 +1,284 @@ +import math +import torch +import torch.nn.functional as F +from torch import nn + +from functools import partial + +# Original implementation from https://github.com/lucidrains/performer-pytorch + +# helpers +def exists(val): + return val is not None + +def empty(tensor): + return tensor.numel() == 0 + +def default(val, d): + return val if exists(val) else d + +def get_module_device(module): + return next(module.parameters()).device + +def find_modules(nn_module, type): + return [module for module in nn_module.modules() if isinstance(module, type)] + +# kernel functions +def softmax_kernel(data, *, projection_matrix, is_query, normalize_data=True, eps=1e-4, device = None): + b, h, *_ = data.shape + + data_normalizer = (data.shape[-1] ** -0.25) if normalize_data else 1. + + ratio = (projection_matrix.shape[0] ** -0.5) + + #projection = repeat(projection_matrix, 'j d -> b h j d', b = b, h = h) + projection = projection_matrix.unsqueeze(0).repeat(h, 1, 1) + projection = projection.unsqueeze(0).repeat(b, 1, 1, 1) # (b,h,j,d) + projection = projection.type_as(data) + + data_dash = torch.einsum('...id,...jd->...ij', (data_normalizer * data), projection) + + diag_data = data ** 2 + diag_data = torch.sum(diag_data, dim=-1) + diag_data = (diag_data / 2.0) * (data_normalizer ** 2) + diag_data = diag_data.unsqueeze(dim=-1) + + if is_query: + data_dash = ratio * ( + torch.exp(data_dash - diag_data - + torch.max(data_dash, dim=-1, keepdim=True).values) + eps) + else: + data_dash = ratio * ( + torch.exp(data_dash - diag_data - torch.max(data_dash)) + eps) + + return data_dash.type_as(data) + +def generalized_kernel(data, *, projection_matrix, kernel_fn = nn.ReLU(inplace=True), kernel_epsilon = 0.001, normalize_data = True, device = None): + b, h, *_ = data.shape + + data_normalizer = (data.shape[-1] ** -0.25) if normalize_data else 1. + + if projection_matrix is None: + return kernel_fn(data_normalizer * data) + kernel_epsilon + + data = data_normalizer*data + data = torch.matmul(data, projection_matrix.T) + data = kernel_fn(data) + kernel_epsilon + return data.type_as(data) + +def orthogonal_matrix_chunk(cols, qr_uniform_q = False, device = None): + unstructured_block = torch.randn((cols, cols), device = device) + q, r = torch.linalg.qr(unstructured_block.cpu(), 'reduced') + q, r = map(lambda t: t.to(device), (q, r)) + + # proposed by @Parskatt + # to make sure Q is uniform https://arxiv.org/pdf/math-ph/0609050.pdf + if qr_uniform_q: + d = torch.diag(r, 0) + q *= d.sign() + return q.t() + +def gaussian_orthogonal_random_matrix(nb_rows, nb_columns, scaling = 0, qr_uniform_q = False, device = None): + nb_full_blocks = int(nb_rows / nb_columns) + + block_list = [] + + for _ in range(nb_full_blocks): + q = orthogonal_matrix_chunk(nb_columns, qr_uniform_q = qr_uniform_q, device = device) + block_list.append(q) + + remaining_rows = nb_rows - nb_full_blocks * nb_columns + if remaining_rows > 0: + q = orthogonal_matrix_chunk(nb_columns, qr_uniform_q = qr_uniform_q, device = device) + block_list.append(q[:remaining_rows]) + + final_matrix = torch.cat(block_list) + + if scaling == 0: + multiplier = torch.randn((nb_rows, nb_columns), device = device).norm(dim = 1) + elif scaling == 1: + multiplier = math.sqrt((float(nb_columns))) * torch.ones((nb_rows,), device = device) + else: + raise ValueError(f'Invalid scaling {scaling}') + + return torch.diag(multiplier) @ final_matrix + +# linear attention classes with softmax kernel + +# non-causal linear attention +def linear_attention(q, k, v): + L = k.shape[-2] + D_inv = 1. / torch.einsum('...nd,...d->...n', q, k.mean(dim=-2)) + context = torch.einsum('...nd,...ne->...de', k/float(L), v) + del k, v + out = torch.einsum('...n,...nd->...nd', D_inv, q) + del D_inv, q + out = torch.einsum('...nd,...de->...ne', out, context) + return out + +class FastAttention(nn.Module): + def __init__(self, dim_heads, nb_features = None, ortho_scaling = 0, generalized_attention = False, kernel_fn = nn.ReLU(inplace=True), qr_uniform_q = False, no_projection = False): + super().__init__() + nb_features = default(nb_features, int(dim_heads * math.log(dim_heads))) + + self.dim_heads = dim_heads + self.nb_features = nb_features + self.ortho_scaling = ortho_scaling + + if not no_projection: + self.create_projection = partial(gaussian_orthogonal_random_matrix, nb_rows = self.nb_features, nb_columns = dim_heads, scaling = ortho_scaling, qr_uniform_q = qr_uniform_q) + projection_matrix = self.create_projection() + self.register_buffer('projection_matrix', projection_matrix) + + self.generalized_attention = generalized_attention + self.kernel_fn = kernel_fn + + # if this is turned on, no projection will be used + # queries and keys will be softmax-ed as in the original efficient attention paper + self.no_projection = no_projection + + + @torch.no_grad() + def redraw_projection_matrix(self, device): + projections = self.create_projection(device = device) + self.projection_matrix.copy_(projections) + del projections + + def forward(self, q, k, v): + device = q.device + + if self.no_projection: + q = q.softmax(dim = -1) + k.softmax(dim = -2) + + elif self.generalized_attention: + create_kernel = partial(generalized_kernel, kernel_fn = self.kernel_fn, projection_matrix = self.projection_matrix, device = device) + q, k = map(create_kernel, (q, k)) + + else: + create_kernel = partial(softmax_kernel, projection_matrix = self.projection_matrix, device = device) + q = create_kernel(q, is_query = True) + k = create_kernel(k, is_query = False) + + attn_fn = linear_attention + out = attn_fn(q, k, v) + return out + +# classes +class ReZero(nn.Module): + def __init__(self, fn): + super().__init__() + self.g = nn.Parameter(torch.tensor(1e-3)) + self.fn = fn + + def forward(self, x, **kwargs): + return self.fn(x, **kwargs) * self.g + +class PreScaleNorm(nn.Module): + def __init__(self, dim, fn, eps=1e-5): + super().__init__() + self.fn = fn + self.g = nn.Parameter(torch.ones(1)) + self.eps = eps + + def forward(self, x, **kwargs): + n = torch.norm(x, dim=-1, keepdim=True).clamp(min=self.eps) + x = x / n * self.g + return self.fn(x, **kwargs) + +class PreLayerNorm(nn.Module): + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + def forward(self, x, **kwargs): + return self.fn(self.norm(x), **kwargs) + +class Chunk(nn.Module): + def __init__(self, chunks, fn, along_dim = -1): + super().__init__() + self.dim = along_dim + self.chunks = chunks + self.fn = fn + + def forward(self, x, **kwargs): + if self.chunks == 1: + return self.fn(x, **kwargs) + chunks = x.chunk(self.chunks, dim = self.dim) + return torch.cat([self.fn(c, **kwargs) for c in chunks], dim = self.dim) + +class SelfAttention(nn.Module): + def __init__(self, dim, k_dim=None, heads = 8, local_heads = 0, local_window_size = 256, nb_features = None, feature_redraw_interval = 1000, generalized_attention = False, kernel_fn = nn.ReLU(inplace=True), qr_uniform_q = False, dropout = 0., no_projection = False): + super().__init__() + assert dim % heads == 0, 'dimension must be divisible by number of heads' + dim_head = dim // heads + inner_dim = dim_head * heads + + if k_dim == None: + k_dim = dim + + self.fast_attention = FastAttention(dim_head, nb_features, generalized_attention = generalized_attention, kernel_fn = kernel_fn, qr_uniform_q = qr_uniform_q, no_projection = no_projection) + + self.heads = heads + self.dim = dim + + self.to_query = nn.Linear(dim, inner_dim) + self.to_key = nn.Linear(k_dim, inner_dim) + self.to_value = nn.Linear(k_dim, inner_dim) + self.to_out = nn.Linear(inner_dim, dim) + self.dropout = nn.Dropout(dropout, inplace=True) + + self.feature_redraw_interval = feature_redraw_interval + self.register_buffer("calls_since_last_redraw", torch.tensor(0)) + + self.max_tokens = 2**16 + + def check_redraw_projections(self): + if not self.training: + return + + if exists(self.feature_redraw_interval) and self.calls_since_last_redraw >= self.feature_redraw_interval: + device = get_module_device(self) + + fast_attentions = find_modules(self, FastAttention) + for fast_attention in fast_attentions: + fast_attention.redraw_projection_matrix(device) + + self.calls_since_last_redraw.zero_() + return + + self.calls_since_last_redraw += 1 + + def _batched_forward(self, q, k, v): + b1, h, n1 = q.shape[:3] + out = torch.empty((b1, h, n1, self.dim//h), dtype=q.dtype, device=q.device) + shift = self.max_tokens // n1 + for i_b in range(0, b1, shift): + start = i_b + end = min(i_b+shift, b1) + out[start:end] = self.fast_attention(q[start:end], k[start:end], v[start:end]) + return out + + def forward(self, query, key, value, **kwargs): + self.check_redraw_projections() + + b1, n1, _, h = *query.shape, self.heads + b2, n2, _, h = *key.shape, self.heads + + q = self.to_query(query) + k = self.to_key(key) + v = self.to_value(value) + + q = q.reshape(b1, n1, h, -1).permute(0,2,1,3) # (b, h, n, d) + k = k.reshape(b2, n2, h, -1).permute(0,2,1,3) + v = v.reshape(b2, n2, h, -1).permute(0,2,1,3) + + if b1*n1 > self.max_tokens or b2*n2 > self.max_tokens: + out = self._batched_forward(q, k, v) + else: + out = self.fast_attention(q, k, v) + + out = out.permute(0,2,1,3).reshape(b1,n1,-1) + out = self.to_out(out) + return self.dropout(out) + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_complex.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_complex.py new file mode 100644 index 00000000..313919a5 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_complex.py @@ -0,0 +1,317 @@ +import sys, os +import time +import numpy as np +import torch +import torch.nn as nn +from torch.utils import data +from parsers import parse_a3m, read_templates +from RoseTTAFoldModel import RoseTTAFoldModule_e2e +import util +from collections import namedtuple +from ffindex import * +from kinematics import xyz_to_c6d, c6d_to_bins2, xyz_to_t2d +from trFold import TRFold + +script_dir = '/'.join(os.path.dirname(os.path.realpath(__file__)).split('/')[:-1]) +NBIN = [37, 37, 37, 19] + +MODEL_PARAM ={ + "n_module" : 8, + "n_module_str" : 4, + "n_module_ref" : 4, + "n_layer" : 1, + "d_msa" : 384 , + "d_pair" : 288, + "d_templ" : 64, + "n_head_msa" : 12, + "n_head_pair" : 8, + "n_head_templ" : 4, + "d_hidden" : 64, + "r_ff" : 4, + "n_resblock" : 1, + "p_drop" : 0.1, + "use_templ" : True, + "performer_N_opts": {"nb_features": 64}, + "performer_L_opts": {"nb_features": 64} + } + +SE3_param = { + "num_layers" : 2, + "num_channels" : 16, + "num_degrees" : 2, + "l0_in_features": 32, + "l0_out_features": 8, + "l1_in_features": 3, + "l1_out_features": 3, + "num_edge_features": 32, + "div": 2, + "n_heads": 4 + } + +REF_param = { + "num_layers" : 3, + "num_channels" : 32, + "num_degrees" : 3, + "l0_in_features": 32, + "l0_out_features": 8, + "l1_in_features": 3, + "l1_out_features": 3, + "num_edge_features": 32, + "div": 4, + "n_heads": 4 + } +MODEL_PARAM['SE3_param'] = SE3_param +MODEL_PARAM['REF_param'] = REF_param + +# params for the folding protocol +fold_params = { + "SG7" : np.array([[[-2,3,6,7,6,3,-2]]])/21, + "SG9" : np.array([[[-21,14,39,54,59,54,39,14,-21]]])/231, + "DCUT" : 19.5, + "ALPHA" : 1.57, + + # TODO: add Cb to the motif + "NCAC" : np.array([[-0.676, -1.294, 0. ], + [ 0. , 0. , 0. ], + [ 1.5 , -0.174, 0. ]], dtype=np.float32), + "CLASH" : 2.0, + "PCUT" : 0.5, + "DSTEP" : 0.5, + "ASTEP" : np.deg2rad(10.0), + "XYZRAD" : 7.5, + "WANG" : 0.1, + "WCST" : 0.1 +} + +fold_params["SG"] = fold_params["SG9"] + +class Predictor(): + def __init__(self, model_dir=None, use_cpu=False): + if model_dir == None: + self.model_dir = "%s/weights"%(script_dir) + else: + self.model_dir = model_dir + # + # define model name + self.model_name = "RoseTTAFold" + if torch.cuda.is_available() and (not use_cpu): + self.device = torch.device("cuda") + else: + self.device = torch.device("cpu") + self.active_fn = nn.Softmax(dim=1) + + # define model & load model + self.model = RoseTTAFoldModule_e2e(**MODEL_PARAM).to(self.device) + could_load = self.load_model(self.model_name) + if not could_load: + print ("ERROR: failed to load model") + sys.exit() + + def load_model(self, model_name, suffix='e2e'): + chk_fn = "%s/%s_%s.pt"%(self.model_dir, model_name, suffix) + if not os.path.exists(chk_fn): + return False + checkpoint = torch.load(chk_fn, map_location=self.device) + self.model.load_state_dict(checkpoint['model_state_dict'], strict=True) + return True + + def predict(self, a3m_fn, out_prefix, Ls, templ_npz=None, window=1000, shift=100): + msa = parse_a3m(a3m_fn) + N, L = msa.shape + # + if templ_npz != None: + templ = np.load(templ_npz) + xyz_t = torch.from_numpy(templ["xyz_t"]) + t1d = torch.from_numpy(templ["t1d"]) + t0d = torch.from_numpy(templ["t0d"]) + else: + xyz_t = torch.full((1, L, 3, 3), np.nan).float() + t1d = torch.zeros((1, L, 3)).float() + t0d = torch.zeros((1,3)).float() + + self.model.eval() + with torch.no_grad(): + # + msa = torch.tensor(msa).long().view(1, -1, L) + idx_pdb_orig = torch.arange(L).long().view(1, L) + idx_pdb = torch.arange(L).long().view(1, L) + L_prev = 0 + for L_i in Ls[:-1]: + idx_pdb[:,L_prev+L_i:] += 500 # it was 200 originally. + L_prev += L_i + seq = msa[:,0] + # + # template features + xyz_t = xyz_t.float().unsqueeze(0) + t1d = t1d.float().unsqueeze(0) + t0d = t0d.float().unsqueeze(0) + t2d = xyz_to_t2d(xyz_t, t0d) + # + # do cropped prediction + if L > window*2: + prob_s = [np.zeros((L,L,NBIN[i]), dtype=np.float32) for i in range(4)] + count_1d = np.zeros((L,), dtype=np.float32) + count_2d = np.zeros((L,L), dtype=np.float32) + node_s = np.zeros((L,MODEL_PARAM['d_msa']), dtype=np.float32) + # + grids = np.arange(0, L-window+shift, shift) + ngrids = grids.shape[0] + print("ngrid: ", ngrids) + print("grids: ", grids) + print("windows: ", window) + + for i in range(ngrids): + for j in range(i, ngrids): + start_1 = grids[i] + end_1 = min(grids[i]+window, L) + start_2 = grids[j] + end_2 = min(grids[j]+window, L) + sel = np.zeros((L)).astype(np.bool) + sel[start_1:end_1] = True + sel[start_2:end_2] = True + + input_msa = msa[:,:,sel] + mask = torch.sum(input_msa==20, dim=-1) < 0.5*sel.sum() # remove too gappy sequences + input_msa = input_msa[mask].unsqueeze(0) + input_msa = input_msa[:,:1000].to(self.device) + input_idx = idx_pdb[:,sel].to(self.device) + input_idx_orig = idx_pdb_orig[:,sel] + input_seq = input_msa[:,0].to(self.device) + # + # Select template + input_t1d = t1d[:,:,sel].to(self.device) # (B, T, L, 3) + input_t2d = t2d[:,:,sel][:,:,:,sel].to(self.device) + # + print ("running crop: %d-%d/%d-%d"%(start_1, end_1, start_2, end_2), input_msa.shape) + with torch.cuda.amp.autocast(): + logit_s, node, init_crds, pred_lddt = self.model(input_msa, input_seq, input_idx, t1d=input_t1d, t2d=input_t2d, return_raw=True) + # + # Not sure How can we merge init_crds..... + pred_lddt = torch.clamp(pred_lddt, 0.0, 1.0) + sub_idx = input_idx_orig[0] + sub_idx_2d = np.ix_(sub_idx, sub_idx) + count_2d[sub_idx_2d] += 1.0 + count_1d[sub_idx] += 1.0 + node_s[sub_idx] += node[0].cpu().numpy() + for i_logit, logit in enumerate(logit_s): + prob = self.active_fn(logit.float()) # calculate distogram + prob = prob.squeeze(0).permute(1,2,0).cpu().numpy() + prob_s[i_logit][sub_idx_2d] += prob + del logit_s, node + # + for i in range(4): + prob_s[i] = prob_s[i] / count_2d[:,:,None] + prob_in = np.concatenate(prob_s, axis=-1) + node_s = node_s / count_1d[:, None] + # + node_s = torch.tensor(node_s).to(self.device).unsqueeze(0) + seq = msa[:,0].to(self.device) + idx_pdb = idx_pdb.to(self.device) + prob_in = torch.tensor(prob_in).to(self.device).unsqueeze(0) + with torch.cuda.amp.autocast(): + xyz, lddt = self.model(node_s, seq, idx_pdb, prob_s=prob_in, refine_only=True) + print (lddt.mean()) + else: + msa = msa[:,:1000].to(self.device) + seq = msa[:,0] + idx_pdb = idx_pdb.to(self.device) + t1d = t1d[:,:10].to(self.device) + t2d = t2d[:,:10].to(self.device) + with torch.cuda.amp.autocast(): + logit_s, _, xyz, lddt = self.model(msa, seq, idx_pdb, t1d=t1d, t2d=t2d) + print (lddt.mean()) + prob_s = list() + for logit in logit_s: + prob = self.active_fn(logit.float()) # distogram + prob = prob.reshape(-1, L, L).permute(1,2,0).cpu().numpy() + prob_s.append(prob) + + np.savez_compressed("%s.npz"%out_prefix, dist=prob_s[0].astype(np.float16), \ + omega=prob_s[1].astype(np.float16),\ + theta=prob_s[2].astype(np.float16),\ + phi=prob_s[3].astype(np.float16)) + + # run TRFold + prob_trF = list() + for prob in prob_s: + prob = torch.tensor(prob).permute(2,0,1).to(self.device) + prob += 1e-8 + prob = prob / torch.sum(prob, dim=0)[None] + prob_trF.append(prob) + xyz = xyz[0, :, 1] + TRF = TRFold(prob_trF, fold_params) + xyz = TRF.fold(xyz, batch=15, lr=0.1, nsteps=200) + print (xyz.shape, lddt[0].shape, seq[0].shape) + self.write_pdb(seq[0], xyz, Ls, Bfacts=lddt[0], prefix=out_prefix) + + def write_pdb(self, seq, atoms, Ls, Bfacts=None, prefix=None): + chainIDs = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + L = len(seq) + filename = "%s.pdb"%prefix + ctr = 1 + with open(filename, 'wt') as f: + if Bfacts == None: + Bfacts = np.zeros(L) + else: + Bfacts = torch.clamp( Bfacts, 0, 1) + + for i,s in enumerate(seq): + if (len(atoms.shape)==2): + resNo = i+1 + chain = "A" + for i_chain in range(len(Ls)-1,0,-1): + tot_res = sum(Ls[:i_chain]) + if i+1 > tot_res: + chain = chainIDs[i_chain] + resNo = i+1 - tot_res + break + f.write ("%-6s%5s %4s %3s %s%4d %8.3f%8.3f%8.3f%6.2f%6.2f\n"%( + "ATOM", ctr, " CA ", util.num2aa[s], + chain, resNo, atoms[i,0], atoms[i,1], atoms[i,2], + 1.0, Bfacts[i] ) ) + ctr += 1 + + elif atoms.shape[1]==3: + resNo = i+1 + chain = "A" + for i_chain in range(len(Ls)-1,0,-1): + tot_res = sum(Ls[:i_chain]) + if i+1 > tot_res: + chain = chainIDs[i_chain] + resNo = i+1 - tot_res + break + for j,atm_j in enumerate((" N "," CA "," C ")): + f.write ("%-6s%5s %4s %3s %s%4d %8.3f%8.3f%8.3f%6.2f%6.2f\n"%( + "ATOM", ctr, atm_j, util.num2aa[s], + chain, resNo, atoms[i,j,0], atoms[i,j,1], atoms[i,j,2], + 1.0, Bfacts[i] ) ) + ctr += 1 + +def get_args(): + import argparse + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-m", dest="model_dir", default="%s/weights"%(script_dir), + help="Path to pre-trained network weights [%s/weights]"%script_dir) + parser.add_argument("-i", dest="a3m_fn", required=True, + help="Input multiple sequence alignments (in a3m format)") + parser.add_argument("-o", dest="out_prefix", required=True, + help="Prefix for output file. The output files will be [out_prefix].npz and [out_prefix].pdb") + parser.add_argument("-Ls", dest="Ls", required=True, nargs="+", type=int, + help="The length of the each subunit (e.g. 220 400)") + parser.add_argument("--templ_npz", default=None, + help='''npz file containing complex template information (xyz_t, t1d, t0d). If not provided, zero matrices will be given as templates + - xyz_t: N, CA, C coordinates of complex templates (T, L, 3, 3) For the unaligned region, it should be NaN + - t1d: 1-D features from HHsearch results (score, SS, probab column from atab file) (T, L, 3). For the unaligned region, it should be zeros + - t0d: 0-D features from HHsearch (Probability/100.0, Ideintities/100.0, Similarity fro hhr file) (T, 3)''') + parser.add_argument("--cpu", dest='use_cpu', default=False, action='store_true') + + args = parser.parse_args() + return args + + + +if __name__ == "__main__": + args = get_args() + if not os.path.exists("%s.npz"%args.out_prefix): + pred = Predictor(model_dir=args.model_dir, use_cpu=args.use_cpu) + pred.predict(args.a3m_fn, args.out_prefix, args.Ls, templ_npz=args.templ_npz) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_e2e.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_e2e.py new file mode 100644 index 00000000..0b1b0027 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_e2e.py @@ -0,0 +1,324 @@ +import sys, os +import time +import numpy as np +import torch +import torch.nn as nn +from torch.utils import data +from parsers import parse_a3m, read_templates +from RoseTTAFoldModel import RoseTTAFoldModule_e2e +import util +from collections import namedtuple +from ffindex import * +from kinematics import xyz_to_c6d, c6d_to_bins2, xyz_to_t2d +from trFold import TRFold + +script_dir = '/'.join(os.path.dirname(os.path.realpath(__file__)).split('/')[:-1]) + +NBIN = [37, 37, 37, 19] + +MODEL_PARAM ={ + "n_module" : 8, + "n_module_str" : 4, + "n_module_ref" : 4, + "n_layer" : 1, + "d_msa" : 384 , + "d_pair" : 288, + "d_templ" : 64, + "n_head_msa" : 12, + "n_head_pair" : 8, + "n_head_templ" : 4, + "d_hidden" : 64, + "r_ff" : 4, + "n_resblock" : 1, + "p_drop" : 0.0, + "use_templ" : True, + "performer_N_opts": {"nb_features": 64}, + "performer_L_opts": {"nb_features": 64} + } + +SE3_param = { + "num_layers" : 2, + "num_channels" : 16, + "num_degrees" : 2, + "l0_in_features": 32, + "l0_out_features": 8, + "l1_in_features": 3, + "l1_out_features": 3, + "num_edge_features": 32, + "div": 2, + "n_heads": 4 + } + +REF_param = { + "num_layers" : 3, + "num_channels" : 32, + "num_degrees" : 3, + "l0_in_features": 32, + "l0_out_features": 8, + "l1_in_features": 3, + "l1_out_features": 3, + "num_edge_features": 32, + "div": 4, + "n_heads": 4 + } +MODEL_PARAM['SE3_param'] = SE3_param +MODEL_PARAM['REF_param'] = REF_param + +# params for the folding protocol +fold_params = { + "SG7" : np.array([[[-2,3,6,7,6,3,-2]]])/21, + "SG9" : np.array([[[-21,14,39,54,59,54,39,14,-21]]])/231, + "DCUT" : 19.5, + "ALPHA" : 1.57, + + # TODO: add Cb to the motif + "NCAC" : np.array([[-0.676, -1.294, 0. ], + [ 0. , 0. , 0. ], + [ 1.5 , -0.174, 0. ]], dtype=np.float32), + "CLASH" : 2.0, + "PCUT" : 0.5, + "DSTEP" : 0.5, + "ASTEP" : np.deg2rad(10.0), + "XYZRAD" : 7.5, + "WANG" : 0.1, + "WCST" : 0.1 +} + +fold_params["SG"] = fold_params["SG9"] + +class Predictor(): + def __init__(self, model_dir=None, use_cpu=False): + if model_dir == None: + self.model_dir = "%s/models"%(os.path.dirname(os.path.realpath(__file__))) + else: + self.model_dir = model_dir + # + # define model name + self.model_name = "RoseTTAFold" + if torch.cuda.is_available() and (not use_cpu): + self.device = torch.device("cuda") + else: + self.device = torch.device("cpu") + self.active_fn = nn.Softmax(dim=1) + + # define model & load model + self.model = RoseTTAFoldModule_e2e(**MODEL_PARAM).to(self.device) + + def load_model(self, model_name, suffix='e2e'): + chk_fn = "%s/%s_%s.pt"%(self.model_dir, model_name, suffix) + if not os.path.exists(chk_fn): + return False + checkpoint = torch.load(chk_fn, map_location=self.device) + self.model.load_state_dict(checkpoint['model_state_dict'], strict=True) + return True + + def predict(self, a3m_fn, out_prefix, hhr_fn=None, atab_fn=None, window=150, shift=75): + msa = parse_a3m(a3m_fn) + N, L = msa.shape + # + if hhr_fn != None: + xyz_t, t1d, t0d = read_templates(L, ffdb, hhr_fn, atab_fn, n_templ=10) + else: + xyz_t = torch.full((1, L, 3, 3), np.nan).float() + t1d = torch.zeros((1, L, 3)).float() + t0d = torch.zeros((1,3)).float() + # + msa = torch.tensor(msa).long().view(1, -1, L) + idx_pdb = torch.arange(L).long().view(1, L) + seq = msa[:,0] + # + # template features + xyz_t = xyz_t.float().unsqueeze(0) + t1d = t1d.float().unsqueeze(0) + t0d = t0d.float().unsqueeze(0) + t2d = xyz_to_t2d(xyz_t, t0d) + + could_load = self.load_model(self.model_name, suffix="e2e") + if not could_load: + print ("ERROR: failed to load model") + sys.exit() + self.model.eval() + with torch.no_grad(): + # do cropped prediction if protein is too big + if L > window*2: + prob_s = [np.zeros((L,L,NBIN[i]), dtype=np.float32) for i in range(4)] + count_1d = np.zeros((L,), dtype=np.float32) + count_2d = np.zeros((L,L), dtype=np.float32) + node_s = np.zeros((L,MODEL_PARAM['d_msa']), dtype=np.float32) + # + grids = np.arange(0, L-window+shift, shift) + ngrids = grids.shape[0] + print("ngrid: ", ngrids) + print("grids: ", grids) + print("windows: ", window) + + for i in range(ngrids): + for j in range(i, ngrids): + start_1 = grids[i] + end_1 = min(grids[i]+window, L) + start_2 = grids[j] + end_2 = min(grids[j]+window, L) + sel = np.zeros((L)).astype(np.bool) + sel[start_1:end_1] = True + sel[start_2:end_2] = True + + input_msa = msa[:,:,sel] + mask = torch.sum(input_msa==20, dim=-1) < 0.5*sel.sum() # remove too gappy sequences + input_msa = input_msa[mask].unsqueeze(0) + input_msa = input_msa[:,:1000].to(self.device) + input_idx = idx_pdb[:,sel].to(self.device) + input_seq = input_msa[:,0].to(self.device) + # + # Select template + input_t1d = t1d[:,:,sel].to(self.device) # (B, T, L, 3) + input_t2d = t2d[:,:,sel][:,:,:,sel].to(self.device) + # + print ("running crop: %d-%d/%d-%d"%(start_1, end_1, start_2, end_2), input_msa.shape) + with torch.cuda.amp.autocast(): + logit_s, node, init_crds, pred_lddt = self.model(input_msa, input_seq, input_idx, t1d=input_t1d, t2d=input_t2d, return_raw=True) + # + # Not sure How can we merge init_crds..... + sub_idx = input_idx[0].cpu() + sub_idx_2d = np.ix_(sub_idx, sub_idx) + count_2d[sub_idx_2d] += 1.0 + count_1d[sub_idx] += 1.0 + node_s[sub_idx] += node[0].cpu().numpy() + for i_logit, logit in enumerate(logit_s): + prob = self.active_fn(logit.float()) # calculate distogram + prob = prob.squeeze(0).permute(1,2,0).cpu().numpy() + prob_s[i_logit][sub_idx_2d] += prob + del logit_s, node + # + # combine all crops + for i in range(4): + prob_s[i] = prob_s[i] / count_2d[:,:,None] + prob_in = np.concatenate(prob_s, axis=-1) + node_s = node_s / count_1d[:, None] + # + # Do iterative refinement using SE(3)-Transformers + # clear cache memory + torch.cuda.empty_cache() + # + node_s = torch.tensor(node_s).to(self.device).unsqueeze(0) + seq = msa[:,0].to(self.device) + idx_pdb = idx_pdb.to(self.device) + prob_in = torch.tensor(prob_in).to(self.device).unsqueeze(0) + with torch.cuda.amp.autocast(): + xyz, lddt = self.model(node_s, seq, idx_pdb, prob_s=prob_in, refine_only=True) + else: + msa = msa[:,:1000].to(self.device) + seq = msa[:,0] + idx_pdb = idx_pdb.to(self.device) + t1d = t1d[:,:10].to(self.device) + t2d = t2d[:,:10].to(self.device) + with torch.cuda.amp.autocast(): + logit_s, _, xyz, lddt = self.model(msa, seq, idx_pdb, t1d=t1d, t2d=t2d) + prob_s = list() + for logit in logit_s: + prob = self.active_fn(logit.float()) # distogram + prob = prob.reshape(-1, L, L).permute(1,2,0).cpu().numpy() + prob_s.append(prob) + + np.savez_compressed("%s.npz"%(out_prefix), dist=prob_s[0].astype(np.float16), \ + omega=prob_s[1].astype(np.float16),\ + theta=prob_s[2].astype(np.float16),\ + phi=prob_s[3].astype(np.float16)) + + self.write_pdb(seq[0], xyz[0], idx_pdb[0], Bfacts=lddt[0], prefix="%s_init"%(out_prefix)) + + # run TRFold + prob_trF = list() + for prob in prob_s: + prob = torch.tensor(prob).permute(2,0,1).to(self.device) + prob += 1e-8 + prob = prob / torch.sum(prob, dim=0)[None] + prob_trF.append(prob) + xyz = xyz[0, :, 1] + TRF = TRFold(prob_trF, fold_params) + xyz = TRF.fold(xyz, batch=15, lr=0.1, nsteps=200) + xyz = xyz.detach().cpu().numpy() + # add O and Cb + N = xyz[:,0,:] + CA = xyz[:,1,:] + C = xyz[:,2,:] + O = self.extend(np.roll(N, -1, axis=0), CA, C, 1.231, 2.108, -3.142) + xyz = np.concatenate((xyz, O[:,None,:]), axis=1) + self.write_pdb(seq[0], xyz, idx_pdb[0], Bfacts=lddt[0], prefix=out_prefix) + + def extend(self, a,b,c, L,A,D): + ''' + input: 3 coords (a,b,c), (L)ength, (A)ngle, and (D)ihedral + output: 4th coord + ''' + N = lambda x: x/np.sqrt(np.square(x).sum(-1,keepdims=True) + 1e-8) + bc = N(b-c) + n = N(np.cross(b-a, bc)) + m = [bc,np.cross(n,bc),n] + d = [L*np.cos(A), L*np.sin(A)*np.cos(D), -L*np.sin(A)*np.sin(D)] + return c + sum([m*d for m,d in zip(m,d)]) + + def write_pdb(self, seq, atoms, idx, Bfacts=None, prefix=None): + L = len(seq) + filename = "%s.pdb"%prefix + ctr = 1 + with open(filename, 'wt') as f: + if Bfacts == None: + Bfacts = np.zeros(L) + else: + Bfacts = torch.clamp( Bfacts, 0, 1) + + for i,s in enumerate(seq): + if (len(atoms.shape)==2): + f.write ("%-6s%5s %4s %3s %s%4d %8.3f%8.3f%8.3f%6.2f%6.2f\n"%( + "ATOM", ctr, " CA ", util.num2aa[s], + "A", idx[i]+1, atoms[i,0], atoms[i,1], atoms[i,2], + 1.0, Bfacts[i] ) ) + ctr += 1 + + elif atoms.shape[1]==3: + for j,atm_j in enumerate((" N "," CA "," C ")): + f.write ("%-6s%5s %4s %3s %s%4d %8.3f%8.3f%8.3f%6.2f%6.2f\n"%( + "ATOM", ctr, atm_j, util.num2aa[s], + "A", idx[i]+1, atoms[i,j,0], atoms[i,j,1], atoms[i,j,2], + 1.0, Bfacts[i] ) ) + ctr += 1 + + elif atoms.shape[1]==4: + for j,atm_j in enumerate((" N "," CA "," C ", " O ")): + f.write ("%-6s%5s %4s %3s %s%4d %8.3f%8.3f%8.3f%6.2f%6.2f\n"%( + "ATOM", ctr, atm_j, util.num2aa[s], + "A", idx[i]+1, atoms[i,j,0], atoms[i,j,1], atoms[i,j,2], + 1.0, Bfacts[i] ) ) + ctr += 1 + + +def get_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-m", dest="model_dir", default="%s/weights"%(script_dir), + help="Path to pre-trained network weights [%s/weights]"%script_dir) + parser.add_argument("-i", dest="a3m_fn", required=True, + help="Input multiple sequence alignments (in a3m format)") + parser.add_argument("-o", dest="out_prefix", required=True, + help="Prefix for output file. The output files will be [out_prefix].npz and [out_prefix].pdb") + parser.add_argument("--hhr", default=None, + help="HHsearch output file (hhr file). If not provided, zero matrices will be given as templates") + parser.add_argument("--atab", default=None, + help="HHsearch output file (atab file)") + parser.add_argument("--db", default="%s/pdb100_2021Mar03/pdb100_2021Mar03"%script_dir, + help="Path to template database [%s/pdb100_2021Mar03]"%script_dir) + parser.add_argument("--cpu", dest='use_cpu', default=False, action='store_true') + + args = parser.parse_args() + return args + +if __name__ == "__main__": + args = get_args() + FFDB=args.db + FFindexDB = namedtuple("FFindexDB", "index, data") + ffdb = FFindexDB(read_index(FFDB+'_pdb.ffindex'), + read_data(FFDB+'_pdb.ffdata')) + + if not os.path.exists("%s.npz"%args.out_prefix): + pred = Predictor(model_dir=args.model_dir, use_cpu=args.use_cpu) + pred.predict(args.a3m_fn, args.out_prefix, args.hhr, args.atab) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_pyRosetta.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_pyRosetta.py new file mode 100644 index 00000000..68da6f7b --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/predict_pyRosetta.py @@ -0,0 +1,200 @@ +import sys, os +import time +import numpy as np +import torch +import torch.nn as nn +from torch.utils import data +from parsers import parse_a3m, read_templates +from RoseTTAFoldModel import RoseTTAFoldModule +import util +from collections import namedtuple +from ffindex import * +from kinematics import xyz_to_c6d, c6d_to_bins2, xyz_to_t2d + +script_dir = '/'.join(os.path.dirname(os.path.realpath(__file__)).split('/')[:-1]) + +NBIN = [37, 37, 37, 19] + +MODEL_PARAM ={ + "n_module" : 8, + "n_module_str" : 4, + "n_layer" : 1, + "d_msa" : 384 , + "d_pair" : 288, + "d_templ" : 64, + "n_head_msa" : 12, + "n_head_pair" : 8, + "n_head_templ" : 4, + "d_hidden" : 64, + "r_ff" : 4, + "n_resblock" : 1, + "p_drop" : 0.1, + "use_templ" : True, + "performer_N_opts": {"nb_features": 64}, + "performer_L_opts": {"nb_features": 64} + } + +SE3_param = { + "num_layers" : 2, + "num_channels" : 16, + "num_degrees" : 2, + "l0_in_features": 32, + "l0_out_features": 8, + "l1_in_features": 3, + "l1_out_features": 3, + "num_edge_features": 32, + "div": 2, + "n_heads": 4 + } +MODEL_PARAM['SE3_param'] = SE3_param + +class Predictor(): + def __init__(self, model_dir=None, use_cpu=False): + if model_dir == None: + self.model_dir = "%s/models"%(os.path.dirname(os.path.realpath(__file__))) + else: + self.model_dir = model_dir + # + # define model name + self.model_name = "RoseTTAFold" + if torch.cuda.is_available() and (not use_cpu): + self.device = torch.device("cuda") + else: + self.device = torch.device("cpu") + self.active_fn = nn.Softmax(dim=1) + + # define model & load model + self.model = RoseTTAFoldModule(**MODEL_PARAM).to(self.device) + could_load = self.load_model(self.model_name) + if not could_load: + print ("ERROR: failed to load model") + sys.exit() + + def load_model(self, model_name, suffix='pyrosetta'): + chk_fn = "%s/%s_%s.pt"%(self.model_dir, model_name, suffix) + if not os.path.exists(chk_fn): + return False + checkpoint = torch.load(chk_fn, map_location=self.device) + self.model.load_state_dict(checkpoint['model_state_dict'], strict=False) + return True + + def predict(self, a3m_fn, out_prefix, hhr_fn=None, atab_fn=None, window=150, shift=50): + msa = parse_a3m(a3m_fn) + N, L = msa.shape + # + if hhr_fn != None: + xyz_t, t1d, t0d = read_templates(L, ffdb, hhr_fn, atab_fn, n_templ=25) + else: + xyz_t = torch.full((1, L, 3, 3), np.nan).float() + t1d = torch.zeros((1, L, 3)).float() + t0d = torch.zeros((1,3)).float() + + self.model.eval() + with torch.no_grad(): + # + msa = torch.tensor(msa).long().view(1, -1, L) + idx_pdb = torch.arange(L).long().view(1, L) + seq = msa[:,0] + # + # template features + xyz_t = xyz_t.float().unsqueeze(0) + t1d = t1d.float().unsqueeze(0) + t0d = t0d.float().unsqueeze(0) + t2d = xyz_to_t2d(xyz_t, t0d) + # + # do cropped prediction + if L > window*2: + prob_s = [np.zeros((L,L,NBIN[i]), dtype=np.float32) for i in range(4)] + count = np.zeros((L,L), dtype=np.float32) + # + grids = np.arange(0, L-window+shift, shift) + ngrids = grids.shape[0] + print("ngrid: ", ngrids) + print("grids: ", grids) + print("windows: ", window) + + for i in range(ngrids): + for j in range(i, ngrids): + start_1 = grids[i] + end_1 = min(grids[i]+window, L) + start_2 = grids[j] + end_2 = min(grids[j]+window, L) + sel = np.zeros((L)).astype(np.bool) + sel[start_1:end_1] = True + sel[start_2:end_2] = True + + input_msa = msa[:,:,sel] + mask = torch.sum(input_msa==20, dim=-1) < 0.5*sel.sum() # remove too gappy sequences + input_msa = input_msa[mask].unsqueeze(0) + input_msa = input_msa[:,:1000].to(self.device) + input_idx = idx_pdb[:,sel].to(self.device) + input_seq = input_msa[:,0].to(self.device) + # + input_t1d = t1d[:,:,sel].to(self.device) + input_t2d = t2d[:,:,sel][:,:,:,sel].to(self.device) + # + print ("running crop: %d-%d/%d-%d"%(start_1, end_1, start_2, end_2), input_msa.shape) + with torch.cuda.amp.autocast(): + logit_s, init_crds, pred_lddt = self.model(input_msa, input_seq, input_idx, t1d=input_t1d, t2d=input_t2d) + # + pred_lddt = torch.clamp(pred_lddt, 0.0, 1.0) + weight = pred_lddt[0][:,None] + pred_lddt[0][None,:] + weight = weight.cpu().numpy() + 1e-8 + sub_idx = input_idx[0].cpu() + sub_idx_2d = np.ix_(sub_idx, sub_idx) + count[sub_idx_2d] += weight + for i_logit, logit in enumerate(logit_s): + prob = self.active_fn(logit.float()) # calculate distogram + prob = prob.squeeze(0).permute(1,2,0).cpu().numpy() + prob_s[i_logit][sub_idx_2d] += weight[:,:,None]*prob + for i in range(4): + prob_s[i] = prob_s[i] / count[:,:,None] + else: + msa = msa[:,:1000].to(self.device) + seq = msa[:,0] + idx_pdb = idx_pdb.to(self.device) + t1d = t1d.to(self.device) + t2d = t2d.to(self.device) + logit_s, init_crds, pred_lddt = self.model(msa, seq, idx_pdb, t1d=t1d, t2d=t2d) + prob_s = list() + for logit in logit_s: + prob = self.active_fn(logit.float()) # distogram + prob = prob.reshape(-1, L, L).permute(1,2,0).cpu().numpy() + prob_s.append(prob) + + np.savez_compressed("%s.npz"%out_prefix, dist=prob_s[0].astype(np.float16), \ + omega=prob_s[1].astype(np.float16),\ + theta=prob_s[2].astype(np.float16),\ + phi=prob_s[3].astype(np.float16)) + + +def get_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("-m", dest="model_dir", default="%s/weights"%(script_dir), + help="Path to pre-trained network weights [%s/weights]"%script_dir) + parser.add_argument("-i", dest="a3m_fn", required=True, + help="Input multiple sequence alignments (in a3m format)") + parser.add_argument("-o", dest="out_prefix", required=True, + help="Prefix for output file. The output file will be [out_prefix].npz") + parser.add_argument("--hhr", default=None, + help="HHsearch output file (hhr file). If not provided, zero matrices will be given as templates") + parser.add_argument("--atab", default=None, + help="HHsearch output file (atab file)") + parser.add_argument("--db", default="%s/pdb100_2021Mar03/pdb100_2021Mar03"%script_dir, + help="Path to template database [%s/pdb100_2021Mar03]"%script_dir) + parser.add_argument("--cpu", dest='use_cpu', default=False, action='store_true') + + args = parser.parse_args() + return args + +if __name__ == "__main__": + args = get_args() + FFDB=args.db + FFindexDB = namedtuple("FFindexDB", "index, data") + ffdb = FFindexDB(read_index(FFDB+'_pdb.ffindex'), + read_data(FFDB+'_pdb.ffdata')) + + if not os.path.exists("%s.npz"%args.out_prefix): + pred = Predictor(model_dir=args.model_dir, use_cpu=args.use_cpu) + pred.predict(args.a3m_fn, args.out_prefix, args.hhr, args.atab) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/resnet.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/resnet.py new file mode 100644 index 00000000..6171b0a9 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/resnet.py @@ -0,0 +1,95 @@ +import torch +import torch.nn as nn + +# original resblock +class ResBlock2D(nn.Module): + def __init__(self, n_c, kernel=3, dilation=1, p_drop=0.15): + super(ResBlock2D, self).__init__() + padding = self._get_same_padding(kernel, dilation) + + layer_s = list() + layer_s.append(nn.Conv2d(n_c, n_c, kernel, padding=padding, dilation=dilation, bias=False)) + layer_s.append(nn.InstanceNorm2d(n_c, affine=True, eps=1e-6)) + layer_s.append(nn.ELU(inplace=True)) + # dropout + layer_s.append(nn.Dropout(p_drop)) + # convolution + layer_s.append(nn.Conv2d(n_c, n_c, kernel, dilation=dilation, padding=padding, bias=False)) + layer_s.append(nn.InstanceNorm2d(n_c, affine=True, eps=1e-6)) + self.layer = nn.Sequential(*layer_s) + self.final_activation = nn.ELU(inplace=True) + + def _get_same_padding(self, kernel, dilation): + return (kernel + (kernel - 1) * (dilation - 1) - 1) // 2 + + def forward(self, x): + out = self.layer(x) + return self.final_activation(x + out) + +# pre-activation bottleneck resblock +class ResBlock2D_bottleneck(nn.Module): + def __init__(self, n_c, kernel=3, dilation=1, p_drop=0.15): + super(ResBlock2D_bottleneck, self).__init__() + padding = self._get_same_padding(kernel, dilation) + + n_b = n_c // 2 # bottleneck channel + + layer_s = list() + # pre-activation + layer_s.append(nn.InstanceNorm2d(n_c, affine=True, eps=1e-6)) + layer_s.append(nn.ELU(inplace=True)) + # project down to n_b + layer_s.append(nn.Conv2d(n_c, n_b, 1, bias=False)) + layer_s.append(nn.InstanceNorm2d(n_b, affine=True, eps=1e-6)) + layer_s.append(nn.ELU(inplace=True)) + # convolution + layer_s.append(nn.Conv2d(n_b, n_b, kernel, dilation=dilation, padding=padding, bias=False)) + layer_s.append(nn.InstanceNorm2d(n_b, affine=True, eps=1e-6)) + layer_s.append(nn.ELU(inplace=True)) + # dropout + layer_s.append(nn.Dropout(p_drop)) + # project up + layer_s.append(nn.Conv2d(n_b, n_c, 1, bias=False)) + + self.layer = nn.Sequential(*layer_s) + + def _get_same_padding(self, kernel, dilation): + return (kernel + (kernel - 1) * (dilation - 1) - 1) // 2 + + def forward(self, x): + out = self.layer(x) + return x + out + +class ResidualNetwork(nn.Module): + def __init__(self, n_block, n_feat_in, n_feat_block, n_feat_out, + dilation=[1,2,4,8], block_type='orig', p_drop=0.15): + super(ResidualNetwork, self).__init__() + + + layer_s = list() + # project to n_feat_block + if n_feat_in != n_feat_block: + layer_s.append(nn.Conv2d(n_feat_in, n_feat_block, 1, bias=False)) + if block_type =='orig': # should acitivate input + layer_s.append(nn.InstanceNorm2d(n_feat_block, affine=True, eps=1e-6)) + layer_s.append(nn.ELU(inplace=True)) + + # add resblocks + for i_block in range(n_block): + d = dilation[i_block%len(dilation)] + if block_type == 'orig': + res_block = ResBlock2D(n_feat_block, kernel=3, dilation=d, p_drop=p_drop) + else: + res_block = ResBlock2D_bottleneck(n_feat_block, kernel=3, dilation=d, p_drop=p_drop) + layer_s.append(res_block) + + if n_feat_out != n_feat_block: + # project to n_feat_out + layer_s.append(nn.Conv2d(n_feat_block, n_feat_out, 1)) + + self.layer = nn.Sequential(*layer_s) + + def forward(self, x): + output = self.layer(x) + return output + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/trFold.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/trFold.py new file mode 100644 index 00000000..1b1d6b16 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/trFold.py @@ -0,0 +1,252 @@ +import numpy as np +import torch +import torch.nn as nn + +def perturb_init(xyz, batch, noise=0.5): + L = xyz.shape[0] + pert = torch.tensor(np.random.uniform(noise, size=(batch, L, 3)), device=xyz.device) + + xyz = xyz.unsqueeze(0) + pert.detach() + return xyz + +def Q2R(Q): + '''convert quaternions to rotation matrices''' + b,l,_ = Q.shape + w,x,y,z = Q[...,0],Q[...,1],Q[...,2],Q[...,3] + xx,xy,xz,xw = x*x, x*y, x*z, x*w + yy,yz,yw = y*y, y*z, y*w + zz,zw = z*z, z*w + R = torch.stack([1-2*yy-2*zz, 2*xy-2*zw, 2*xz+2*yw, + 2*xy+2*zw, 1-2*xx-2*zz, 2*yz-2*xw, + 2*xz-2*yw, 2*yz+2*xw, 1-2*xx-2*yy],dim=-1).view(b,l,3,3) + return R + +def get_cb(N,Ca,C): + """recreate Cb given N,Ca,C""" + b = Ca - N + c = C - Ca + a = torch.cross(b, c, dim=-1) + Cb = -0.58273431*a + 0.56802827*b - 0.54067466*c + Ca + return Cb + +# ============================================================ +def get_ang(a, b, c): + """calculate planar angles for all consecutive triples (a[i],b[i],c[i]) + from Cartesian coordinates of three sets of atoms a,b,c + + Parameters + ---------- + a,b,c : pytorch tensors of shape [batch,nres,3] + store Cartesian coordinates of three sets of atoms + Returns + ------- + ang : pytorch tensor of shape [batch,nres] + stores resulting planar angles + """ + v = a - b + w = c - b + v = v / torch.norm(v, dim=-1, keepdim=True) + w = w / torch.norm(w, dim=-1, keepdim=True) + vw = torch.sum(v*w, dim=-1) + + return torch.acos(vw) + +# ============================================================ +def get_dih(a, b, c, d): + """calculate dihedral angles for all consecutive quadruples (a[i],b[i],c[i],d[i]) + given Cartesian coordinates of four sets of atoms a,b,c,d + + Parameters + ---------- + a,b,c,d : pytorch tensors of shape [batch,nres,3] + store Cartesian coordinates of four sets of atoms + Returns + ------- + dih : pytorch tensor of shape [batch,nres] + stores resulting dihedrals + """ + b0 = a - b + b1 = c - b + b2 = d - c + + b1 = b1 / torch.norm(b1, dim=-1, keepdim=True) + + v = b0 - torch.sum(b0*b1, dim=-1, keepdim=True)*b1 + w = b2 - torch.sum(b2*b1, dim=-1, keepdim=True)*b1 + + x = torch.sum(v*w, dim=-1) + y = torch.sum(torch.cross(b1,v,dim=-1)*w, dim=-1) + + return torch.atan2(y, x) + + +class TRFold(): + + def __init__(self, pred, params): + + self.pred = pred + self.params = params + self.device = self.pred[0].device + + # dfire background correction for distograms + self.bkgd = (torch.linspace(4.25,19.75,32,device=self.device)/ + self.params['DCUT'])**self.params['ALPHA'] + + # background correction for phi + ang = torch.linspace(0,np.pi,19,device=self.device)[:-1] + self.bkgp = 0.5*(torch.cos(ang)-torch.cos(ang+np.deg2rad(10.0))) + + # Sav-Gol filter + self.sg = torch.from_numpy(self.params['SG']).float().to(self.device) + + # paddings for distograms: + # left - linear clash; right - zeroes + padRsize = self.sg.shape[-1]//2+3 + padLsize = padRsize + 8 + padR = torch.zeros(padRsize,device=self.device) + padL = torch.arange(1,padLsize+1,device=self.device).flip(0)*self.params['CLASH'] + self.padR = padR[:,None] + self.padL = padL[:,None] + + # backbone motif + self.ncac = torch.from_numpy(self.params['NCAC']).to(self.device) + + def akima(self, y,h): + ''' Akima spline coefficients (boundaries trimmed to [2:-2]) + https://doi.org/10.1145/321607.321609 ''' + m = (y[:,1:]-y[:,:-1])/h + #m += 1e-3*torch.randn(m.shape, device=m.device) + m4m3 = torch.abs(m[:,3:]-m[:,2:-1]) + m2m1 = torch.abs(m[:,1:-2]-m[:,:-3]) + t = (m4m3*m[:,1:-2] + m2m1*m[:,2:-1])/(m4m3+m2m1) + t[torch.isnan(t)] = 0.0 + dy = y[:,3:-2]-y[:,2:-3] + coef = torch.stack([y[:,2:-3], + t[:,:-1], + (3*dy/h - 2*t[:,:-1] - t[:,1:])/h, + (t[:,:-1]+t[:,1:] - 2*dy/h)/h**2], dim=-1) + return coef + + def fold(self, xyz, batch=32, lr=0.8, nsteps=100): + + pd,po,pt,pp = self.pred + L = pd.shape[-1] + + p20 = (6.0-pd[-1]-po[-1]-pt[-1]-pp[-1]-(pt[-1]+pp[-1]).T)/6 + i,j = torch.triu_indices(L,L,1,device=self.device) + sel = torch.where(p20[i,j]>self.params['PCUT'])[0] + + # indices for dist and omega (symmetric) + i_s,j_s = i[sel], j[sel] + + # indices for theta and phi (asymmetric) + i_a,j_a = torch.hstack([i_s,j_s]), torch.hstack([j_s,i_s]) + + # background-corrected initial restraints + cstd = -torch.log(pd[4:36,i_s,j_s]/self.bkgd[:,None]) + csto = -torch.log(po[0:36,i_s,j_s]/(1./36)) # omega and theta + cstt = -torch.log(pt[0:36,i_a,j_a]/(1./36)) # are almost uniform + cstp = -torch.log(pp[0:18,i_a,j_a]/self.bkgp[:,None]) + + # padded restraints + pad = self.sg.shape[-1]//2+3 + cstd = torch.cat([self.padL+cstd[0],cstd,self.padR+cstd[-1]],dim=0) + csto = torch.cat([csto[-pad:],csto,csto[:pad]],dim=0) + cstt = torch.cat([cstt[-pad:],cstt,cstt[:pad]],dim=0) + cstp = torch.cat([cstp[:pad].flip(0),cstp,cstp[-pad:].flip(0)],dim=0) + + # smoothed restraints + cstd,csto,cstt,cstp = [nn.functional.conv1d(cst.T.unsqueeze(1),self.sg)[:,0] + for cst in [cstd,csto,cstt,cstp]] + + # force distance restraints vanish at long distances + cstd = cstd-cstd[:,-1][:,None] + + # akima spline coefficients + coefd = self.akima(cstd, self.params['DSTEP']).detach() + coefo = self.akima(csto, self.params['ASTEP']).detach() + coeft = self.akima(cstt, self.params['ASTEP']).detach() + coefp = self.akima(cstp, self.params['ASTEP']).detach() + + astep = self.params['ASTEP'] + + ko = torch.arange(i_s.shape[0],device=self.device).repeat(batch) + kt = torch.arange(i_a.shape[0],device=self.device).repeat(batch) + + # initial Ca placement using EDM+minimization + xyz = perturb_init(xyz, batch) # (batch, L, 3) + + # optimization variables: T - shift vectors, Q - rotation quaternions + T = torch.zeros_like(xyz,device=self.device,requires_grad=True) + Q = torch.randn([batch,L,4],device=self.device,requires_grad=True) + bb0 = self.ncac[None,:,None,:].repeat(batch,1,L,1) + + opt = torch.optim.Adam([T,Q], lr=lr) + for step in range(nsteps): + + + R = Q2R(Q/torch.norm(Q,dim=-1,keepdim=True)) + bb = torch.einsum("blij,bklj->bkli",R,bb0)+(xyz+T)[:,None] + + # TODO: include Cb in the motif + N,Ca,C = bb[:,0],bb[:,1],bb[:,2] + Cb = get_cb(N,Ca,C) + + o = get_dih(Ca[:,i_s],Cb[:,i_s],Cb[:,j_s],Ca[:,j_s]) + np.pi + t = get_dih(N[:,i_a],Ca[:,i_a],Cb[:,i_a],Cb[:,j_a]) + np.pi + p = get_ang(Ca[:,i_a],Cb[:,i_a],Cb[:,j_a]) + + dij = torch.norm(Cb[:,i_s]-Cb[:,j_s],dim=-1) + b,k = torch.where(dij<20.0) + dk = dij[b,k] + + #coord = [coord/step-0.5 for coord,step in zip([dij,o,t,p],[dstep,astep,astep,astep])] + #bins = [torch.ceil(c).long() for c in coord] + #delta = [torch.frac(c) for c in coord] + + kbin = torch.ceil((dk-0.25)/0.5).long() + dx = (dk-0.25)%0.5 + c = coefd[k,kbin] + lossd = c[:,0]+c[:,1]*dx+c[:,2]*dx**2+c[:,3]*dx**3 + + # omega + obin = torch.ceil((o.view(-1)-astep/2)/astep).long() + do = (o.view(-1)-astep/2)%astep + co = coefo[ko,obin] + losso = (co[:,0]+co[:,1]*do+co[:,2]*do**2+co[:,3]*do**3).view(batch,-1) #.mean(1) + + # theta + tbin = torch.ceil((t.view(-1)-astep/2)/astep).long() + dt = (t.view(-1)-astep/2)%astep + ct = coeft[kt,tbin] + losst = (ct[:,0]+ct[:,1]*dt+ct[:,2]*dt**2+ct[:,3]*dt**3).view(batch,-1) #.mean(1) + + # phi + pbin = torch.ceil((p.view(-1)-astep/2)/astep).long() + dp = (p.view(-1)-astep/2)%astep + cp = coefp[kt,pbin] + lossp = (cp[:,0]+cp[:,1]*dp+cp[:,2]*dp**2+cp[:,3]*dp**3).view(batch,-1) #.mean(1) + + # restrain geometry of peptide bonds + loss_nc = (torch.norm(C[:,:-1]-N[:,1:],dim=-1)-1.32868)**2 + loss_cacn = (get_ang(Ca[:,:-1], C[:,:-1], N[:,1:]) - 2.02807)**2 + loss_canc = (get_ang(Ca[:,1:], N[:,1:], C[:,:-1]) - 2.12407)**2 + + loss_geom = loss_nc.mean(1) + loss_cacn.mean(1) + loss_canc.mean(1) + loss_ang = losst.mean(1) + losso.mean(1) + lossp.mean(1) + + # coefficient for ramping up geometric restraints during minimization + coef = (1.0+step)/nsteps + + loss = lossd.mean() + self.params['WANG']*loss_ang.mean() + coef*self.params['WCST']*loss_geom.mean() + + opt.zero_grad() + loss.backward() + opt.step() + + lossd = torch.stack([lossd[b==i].mean() for i in range(batch)]) + loss = lossd + self.params['WANG']*loss_ang + self.params['WCST']*loss_geom + minidx = torch.argmin(loss) + + return bb[minidx].permute(1,0,2) + diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/util.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/util.py new file mode 100644 index 00000000..0b51e533 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/util.py @@ -0,0 +1,253 @@ +import sys + +import numpy as np +import torch + +num2aa=[ + 'ALA','ARG','ASN','ASP','CYS', + 'GLN','GLU','GLY','HIS','ILE', + 'LEU','LYS','MET','PHE','PRO', + 'SER','THR','TRP','TYR','VAL', + ] + +aa2num= {x:i for i,x in enumerate(num2aa)} + +# minimal sc atom representation (Nx8) +aa2short=[ + (" N "," CA "," C "," CB ", None, None, None, None), # ala + (" N "," CA "," C "," CB "," CG "," CD "," NE "," CZ "), # arg + (" N "," CA "," C "," CB "," CG "," OD1", None, None), # asn + (" N "," CA "," C "," CB "," CG "," OD1", None, None), # asp + (" N "," CA "," C "," CB "," SG ", None, None, None), # cys + (" N "," CA "," C "," CB "," CG "," CD "," OE1", None), # gln + (" N "," CA "," C "," CB "," CG "," CD "," OE1", None), # glu + (" N "," CA "," C ", None, None, None, None, None), # gly + (" N "," CA "," C "," CB "," CG "," ND1", None, None), # his + (" N "," CA "," C "," CB "," CG1"," CD1", None, None), # ile + (" N "," CA "," C "," CB "," CG "," CD1", None, None), # leu + (" N "," CA "," C "," CB "," CG "," CD "," CE "," NZ "), # lys + (" N "," CA "," C "," CB "," CG "," SD "," CE ", None), # met + (" N "," CA "," C "," CB "," CG "," CD1", None, None), # phe + (" N "," CA "," C "," CB "," CG "," CD ", None, None), # pro + (" N "," CA "," C "," CB "," OG ", None, None, None), # ser + (" N "," CA "," C "," CB "," OG1", None, None, None), # thr + (" N "," CA "," C "," CB "," CG "," CD1", None, None), # trp + (" N "," CA "," C "," CB "," CG "," CD1", None, None), # tyr + (" N "," CA "," C "," CB "," CG1", None, None, None), # val +] + +# full sc atom representation (Nx14) +aa2long=[ + (" N "," CA "," C "," O "," CB ", None, None, None, None, None, None, None, None, None), # ala + (" N "," CA "," C "," O "," CB "," CG "," CD "," NE "," CZ "," NH1"," NH2", None, None, None), # arg + (" N "," CA "," C "," O "," CB "," CG "," OD1"," ND2", None, None, None, None, None, None), # asn + (" N "," CA "," C "," O "," CB "," CG "," OD1"," OD2", None, None, None, None, None, None), # asp + (" N "," CA "," C "," O "," CB "," SG ", None, None, None, None, None, None, None, None), # cys + (" N "," CA "," C "," O "," CB "," CG "," CD "," OE1"," NE2", None, None, None, None, None), # gln + (" N "," CA "," C "," O "," CB "," CG "," CD "," OE1"," OE2", None, None, None, None, None), # glu + (" N "," CA "," C "," O ", None, None, None, None, None, None, None, None, None, None), # gly + (" N "," CA "," C "," O "," CB "," CG "," ND1"," CD2"," CE1"," NE2", None, None, None, None), # his + (" N "," CA "," C "," O "," CB "," CG1"," CG2"," CD1", None, None, None, None, None, None), # ile + (" N "," CA "," C "," O "," CB "," CG "," CD1"," CD2", None, None, None, None, None, None), # leu + (" N "," CA "," C "," O "," CB "," CG "," CD "," CE "," NZ ", None, None, None, None, None), # lys + (" N "," CA "," C "," O "," CB "," CG "," SD "," CE ", None, None, None, None, None, None), # met + (" N "," CA "," C "," O "," CB "," CG "," CD1"," CD2"," CE1"," CE2"," CZ ", None, None, None), # phe + (" N "," CA "," C "," O "," CB "," CG "," CD ", None, None, None, None, None, None, None), # pro + (" N "," CA "," C "," O "," CB "," OG ", None, None, None, None, None, None, None, None), # ser + (" N "," CA "," C "," O "," CB "," OG1"," CG2", None, None, None, None, None, None, None), # thr + (" N "," CA "," C "," O "," CB "," CG "," CD1"," CD2"," CE2"," CE3"," NE1"," CZ2"," CZ3"," CH2"), # trp + (" N "," CA "," C "," O "," CB "," CG "," CD1"," CD2"," CE1"," CE2"," CZ "," OH ", None, None), # tyr + (" N "," CA "," C "," O "," CB "," CG1"," CG2", None, None, None, None, None, None, None), # val +] + +# build the "alternate" sc mapping +aa2longalt=[ + (" N "," CA "," C "," O "," CB ", None, None, None, None, None, None, None, None, None), # ala + (" N "," CA "," C "," O "," CB "," CG "," CD "," NE "," CZ "," NH2"," NH1", None, None, None), # arg + (" N "," CA "," C "," O "," CB "," CG "," OD1"," ND2", None, None, None, None, None, None), # asn + (" N "," CA "," C "," O "," CB "," CG "," OD2"," OD1", None, None, None, None, None, None), # asp + (" N "," CA "," C "," O "," CB "," SG ", None, None, None, None, None, None, None, None), # cys + (" N "," CA "," C "," O "," CB "," CG "," CD "," OE1"," NE2", None, None, None, None, None), # gln + (" N "," CA "," C "," O "," CB "," CG "," CD "," OE2"," OE1", None, None, None, None, None), # glu + (" N "," CA "," C "," O ", None, None, None, None, None, None, None, None, None, None), # gly + (" N "," CA "," C "," O "," CB "," CG "," ND1"," CD2"," CE1"," NE2", None, None, None, None), # his + (" N "," CA "," C "," O "," CB "," CG1"," CG2"," CD1", None, None, None, None, None, None), # ile + (" N "," CA "," C "," O "," CB "," CG "," CD2"," CD1", None, None, None, None, None, None), # leu + (" N "," CA "," C "," O "," CB "," CG "," CD "," CE "," NZ ", None, None, None, None, None), # lys + (" N "," CA "," C "," O "," CB "," CG "," SD "," CE ", None, None, None, None, None, None), # met + (" N "," CA "," C "," O "," CB "," CG "," CD2"," CD1"," CE2"," CE1"," CZ ", None, None, None), # phe + (" N "," CA "," C "," O "," CB "," CG "," CD ", None, None, None, None, None, None, None), # pro + (" N "," CA "," C "," O "," CB "," OG ", None, None, None, None, None, None, None, None), # ser + (" N "," CA "," C "," O "," CB "," OG1"," CG2", None, None, None, None, None, None, None), # thr + (" N "," CA "," C "," O "," CB "," CG "," CD1"," CD2"," CE2"," CE3"," NE1"," CZ2"," CZ3"," CH2"), # trp + (" N "," CA "," C "," O "," CB "," CG "," CD2"," CD1"," CE2"," CE1"," CZ "," OH ", None, None), # tyr + (" N "," CA "," C "," O "," CB "," CG2"," CG1", None, None, None, None, None, None, None), # val +] + + +# build "deterministic" atoms +# see notebook (se3_experiments.ipynb for derivation) +aa2frames=[ + [], # ala + [ # arg + [' NH1', ' CZ ', ' NE ', ' CD ', [-0.7218378782272339, 1.0856682062149048, -0.006118079647421837]], + [' NH2', ' CZ ', ' NE ', ' CD ', [-0.6158039569854736, -1.1400136947631836, 0.006467342376708984]]], + [ # asn + [' ND2', ' CG ', ' CB ', ' OD1', [-0.6304131746292114, -1.1431225538253784, 0.02364802360534668]]], + [ # asp + [' OD2', ' CG ', ' CB ', ' OD1', [-0.5972501039505005, -1.0955055952072144, 0.04530305415391922]]], + [], # cys + [ # gln + [' NE2', ' CD ', ' CG ', ' OE1', [-0.6558755040168762, -1.1324536800384521, 0.026521772146224976]]], + [ # glu + [' OE2', ' CD ', ' CG ', ' OE1', [-0.5578438639640808, -1.1161314249038696, -0.015464287251234055]]], + [], # gly + [ # his + [' CD2', ' CG ', ' CB ', ' ND1', [-0.7502505779266357, -1.1680538654327393, 0.0005368441343307495]], + [' CE1', ' CG ', ' CB ', ' ND1', [-2.0262467861175537, 0.539483368396759, -0.004495501518249512]], + [' NE2', ' CG ', ' CB ', ' ND1', [-2.0761325359344482, -0.8199722766876221, -0.0018703639507293701]]], + [ # ile + [' CG2', ' CB ', ' CA ', ' CG1', [-0.6059935688972473, -0.8108057379722595, 1.1861376762390137]]], + [ # leu + [' CD2', ' CG ', ' CB ', ' CD1', [-0.5942193269729614, -0.7693282961845398, -1.1914138793945312]]], + [], # lys + [], # met + [ # phe + [' CD2', ' CG ', ' CB ', ' CD1', [-0.7164441347122192, -1.197853446006775, 0.06416648626327515]], + [' CE1', ' CG ', ' CB ', ' CD1', [-2.0785865783691406, 1.2366485595703125, 0.08100450038909912]], + [' CE2', ' CG ', ' CB ', ' CD1', [-2.107091188430786, -1.178497076034546, 0.13524535298347473]], + [' CZ ', ' CG ', ' CB ', ' CD1', [-2.786630630493164, 0.03873880207538605, 0.14633776247501373]]], + [], # pro + [], # ser + [ # thr + [' CG2', ' CB ', ' CA ', ' OG1', [-0.6842088103294373, -0.6709619164466858, 1.2105456590652466]]], + [ # trp + [' CD2', ' CG ', ' CB ', ' CD1', [-0.8550368547439575, -1.0790592432022095, 0.09017711877822876]], + [' NE1', ' CG ', ' CB ', ' CD1', [-2.1863200664520264, 0.8064242601394653, 0.08350661396980286]], + [' CE2', ' CG ', ' CB ', ' CD1', [-2.1801204681396484, -0.5795643329620361, 0.14015203714370728]], + [' CE3', ' CG ', ' CB ', ' CD1', [-0.605582594871521, -2.4733362197875977, 0.16200461983680725]], + [' CE2', ' CG ', ' CB ', ' CD1', [-2.1801204681396484, -0.5795643329620361, 0.14015203714370728]], + [' CZ2', ' CG ', ' CB ', ' CD1', [-3.2672977447509766, -1.473116159439087, 0.250858873128891]], + [' CZ3', ' CG ', ' CB ', ' CD1', [-1.6969941854476929, -3.3360071182250977, 0.264143705368042]], + [' CH2', ' CG ', ' CB ', ' CD1', [-3.009331703186035, -2.8451972007751465, 0.3059283494949341]]], + [ # tyr + [' CD2', ' CG ', ' CB ', ' CD1', [-0.69439297914505, -1.2123756408691406, -0.009198814630508423]], + [' CE1', ' CG ', ' CB ', ' CD1', [-2.104464054107666, 1.1910505294799805, -0.014679580926895142]], + [' CE2', ' CG ', ' CB ', ' CD1', [-2.0857787132263184, -1.2231677770614624, -0.024517983198165894]], + [' CZ ', ' CG ', ' CB ', ' CD1', [-2.7897322177886963, -0.021470561623573303, -0.026979409158229828]], + [' OH ', ' CG ', ' CB ', ' CD1', [-4.1559271812438965, -0.029129385948181152, -0.044720835983753204]]], + [ # val + [' CG2', ' CB ', ' CA ', ' CG1', [-0.6258467435836792, -0.7654698491096497, -1.1894742250442505]]], +] + +# O from frame (C,N-1,CA) +bb2oframe=[-0.5992066264152527, -1.0820008516311646, 0.0001476481556892395] + +# build the mapping from indices in reduced representation to +# indices in the full representation +# N x 14 x 6 = +# base-idx < 0 ==> no atom +# xyz = 0 ==> no mapping +short2long = np.zeros((20,14,6)) +for i in range(20): + i_s, i_l = aa2short[i],aa2long[i] + for j,a in enumerate(i_l): + # case 1: if no atom defined, blank + if (a is None): + short2long[i,j,0] = -1 + # case 2: atom is a base atom + elif (a in i_s): + short2long[i,j,0] = i_s.index(a) + if (short2long[i,j,0] == 0): + short2long[i,j,1] = 1 + short2long[i,j,2] = 2 + else: + short2long[i,j,1] = 0 + if (short2long[i,j,0] == 1): + short2long[i,j,2] = 2 + else: + short2long[i,j,2] = 1 + # case 3: atom is ' O ' + elif (a == " O "): + short2long[i,j,0] = 2 + short2long[i,j,1] = 0 #Nprev (will pre-roll N as nothing else needs it) + short2long[i,j,2] = 1 + short2long[i,j,3:] = np.array(bb2oframe) + # case 4: build this atom + else: + i_f = aa2frames[i] + names = [f[0] for f in i_f] + idx = names.index(a) + short2long[i,j,0] = i_s.index(i_f[idx][1]) + short2long[i,j,1] = i_s.index(i_f[idx][2]) + short2long[i,j,2] = i_s.index(i_f[idx][3]) + short2long[i,j,3:] = np.array(i_f[idx][4]) + +# build the mapping from atoms in the full rep (Nx14) to the "alternate" rep +long2alt = np.zeros((20,14)) +for i in range(20): + i_l, i_lalt = aa2long[i], aa2longalt[i] + for j,a in enumerate(i_l): + if (a is None): + long2alt[i,j] = j + else: + long2alt[i,j] = i_lalt.index(a) + +def atoms_from_frames(base,parent,gparent,points): + xs = parent-base + + # handle parent==base + mask = (torch.sum(torch.square(xs),dim=-1)==0) + xs[mask,0] = 1.0 + xs = xs / torch.norm(xs, dim=-1)[:,None] + + ys = gparent-base + ys = ys - torch.sum(xs*ys,dim=-1)[:,None]*xs + + # handle gparent==base + mask = (torch.sum(torch.square(ys),dim=-1)==0) + ys[mask,1] = 1.0 + + ys = ys / torch.norm(ys, dim=-1)[:,None] + zs = torch.cross(xs,ys) + q = torch.stack((xs,ys,zs),dim=2) + + retval = base + torch.einsum('nij,nj->ni',q,points) + + return retval +#def atoms_from_frames(base,parent,gparent,points): +# xs = parent-base +# # handle parent=base +# mask = (torch.sum(torch.square(xs), dim=-1) == 0) +# xs[mask,0] = 1.0 +# xs = xs / torch.norm(xs, dim=-1)[:,None] +# +# ys = gparent-base +# # handle gparent=base +# mask = (torch.sum(torch.square(ys),dim=-1)==0) +# ys[mask,1] = 1.0 +# +# ys = ys - torch.sum(xs*ys,dim=-1)[:,None]*xs +# ys = ys / torch.norm(ys, dim=-1)[:,None] +# zs = torch.cross(xs,ys) +# q = torch.stack((xs,ys,zs),dim=2) +# #return base + q@points +# return base + torch.einsum('nij,nj->ni',q,points) + +# writepdb +def writepdb(filename, atoms, bfacts, seq): + f = open(filename,"w") + + ctr = 1 + scpu = seq.cpu() + atomscpu = atoms.cpu() + Bfacts = torch.clamp( bfacts.cpu(), 0, 1) + for i,s in enumerate(scpu): + atms = aa2long[s] + for j,atm_j in enumerate(atms): + if (atm_j is not None): + f.write ("%-6s%5s %4s %3s %s%4d %8.3f%8.3f%8.3f%6.2f%6.2f\n"%( + "ATOM", ctr, atm_j, num2aa[s], + "A", i+1, atomscpu[i,j,0], atomscpu[i,j,1], atomscpu[i,j,2], + 1.0, Bfacts[i] ) ) + ctr += 1 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_data.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_data.py new file mode 100644 index 00000000..a80fca72 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_data.py @@ -0,0 +1,64 @@ +import warnings + +import dgl +import torch + + +def to_np(x): + return x.cpu().detach().numpy() + + +class PickleGraph: + """Lightweight graph object for easy pickling. Does not support batched graphs.""" + + def __init__(self, G=None, desired_keys=None): + self.ndata = dict() + self.edata = dict() + + if G is None: + self.src = [] + self.dst = [] + else: + if G.batch_size > 1: + warnings.warn("Copying a batched graph to a PickleGraph is not supported. " + "All node and edge data will be copied, but batching information will be lost.") + + self.src, self.dst = (to_np(idx) for idx in G.all_edges()) + + for k in G.ndata: + if desired_keys is None or k in desired_keys: + self.ndata[k] = to_np(G.ndata[k]) + + for k in G.edata: + if desired_keys is None or k in desired_keys: + self.edata[k] = to_np(G.edata[k]) + + def all_edges(self): + return self.src, self.dst + + +def copy_dgl_graph(G): + if G.batch_size == 1: + src, dst = G.all_edges() + G2 = dgl.DGLGraph((src, dst)) + for edge_key in list(G.edata.keys()): + G2.edata[edge_key] = torch.clone(G.edata[edge_key]) + for node_key in list(G.ndata.keys()): + G2.ndata[node_key] = torch.clone(G.ndata[node_key]) + return G2 + else: + list_of_graphs = dgl.unbatch(G) + list_of_copies = [] + + for batch_G in list_of_graphs: + list_of_copies.append(copy_dgl_graph(batch_G)) + + return dgl.batch(list_of_copies) + + +def update_relative_positions(G, *, relative_position_key='d', absolute_position_key='x'): + """For each directed edge in the graph, calculate the relative position of the destination node with respect + to the source node. Write the relative positions to the graph as edge data.""" + src, dst = G.all_edges() + absolute_positions = G.ndata[absolute_position_key] + G.edata[relative_position_key] = absolute_positions[dst] - absolute_positions[src] diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_logging.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_logging.py new file mode 100644 index 00000000..95f54714 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_logging.py @@ -0,0 +1,123 @@ +import os +import sys +import time +import datetime +import subprocess +import numpy as np +import torch + +from utils.utils_data import to_np + + +_global_log = {} + + +def try_mkdir(path): + if not os.path.exists(path): + os.makedirs(path) + + +# @profile +def make_logdir(checkpoint_dir, run_name=None): + if run_name is None: + now = datetime.datetime.now().strftime("%Y_%m_%d_%H.%M.%S") + else: + assert type(run_name) == str + now = run_name + + log_dir = os.path.join(checkpoint_dir, now) + try_mkdir(log_dir) + return log_dir + + +def count_parameters(model): + """ + count number of trainable parameters in module + :param model: nn.Module instance + :return: integer + """ + model_parameters = filter(lambda p: p.requires_grad, model.parameters()) + n_params = sum([np.prod(p.size()) for p in model_parameters]) + return n_params + + +def write_info_file(model, FLAGS, UNPARSED_ARGV, wandb_log_dir=None): + time_str = time.strftime("%m%d_%H%M%S") + filename_log = "info_" + time_str + ".txt" + filename_git_diff = "git_diff_" + time_str + ".txt" + + checkpoint_name = 'model' + + if wandb_log_dir: + log_dir = wandb_log_dir + os.mkdir(os.path.join(log_dir, 'checkpoints')) + checkpoint_path = os.path.join(log_dir, 'checkpoints', checkpoint_name) + elif FLAGS.restore: + # set restore path + assert FLAGS.run_name is not None + log_dir = os.path.join(FLAGS.checkpoint_dir, FLAGS.run_name) + checkpoint_path = os.path.join(log_dir, 'checkpoints', checkpoint_name) + else: + # makes logdir with time stamp + log_dir = make_logdir(FLAGS.checkpoint_dir, FLAGS.run_name) + os.mkdir(os.path.join(log_dir, 'checkpoints')) + os.mkdir(os.path.join(log_dir, 'point_clouds')) + # os.mkdir(os.path.join(log_dir, 'train_log')) + # os.mkdir(os.path.join(log_dir, 'test_log')) + checkpoint_path = os.path.join(log_dir, 'checkpoints', checkpoint_name) + + # writing arguments and git hash to info file + file = open(os.path.join(log_dir, filename_log), "w") + label = subprocess.check_output(["git", "describe", "--always"]).strip() + file.write('latest git commit on this branch: ' + str(label) + '\n') + file.write('\nFLAGS: \n') + for key in sorted(vars(FLAGS)): + file.write(key + ': ' + str(vars(FLAGS)[key]) + '\n') + + # count number of parameters + if hasattr(model, 'parameters'): + file.write('\nNumber of Model Parameters: ' + str(count_parameters(model)) + '\n') + if hasattr(model, 'enc'): + file.write('\nNumber of Encoder Parameters: ' + str( + count_parameters(model.enc)) + '\n') + if hasattr(model, 'dec'): + file.write('\nNumber of Decoder Parameters: ' + str( + count_parameters(model.dec)) + '\n') + + file.write('\nUNPARSED_ARGV:\n' + str(UNPARSED_ARGV)) + file.write('\n\nBASH COMMAND: \n') + bash_command = 'python' + for argument in sys.argv: + bash_command += (' ' + argument) + file.write(bash_command) + file.close() + + # write 'git diff' output into extra file + subprocess.call(["git diff > " + os.path.join(log_dir, filename_git_diff)], shell=True) + + return log_dir, checkpoint_path + + +def log_gradient_norm(tensor, variable_name): + if variable_name not in _global_log: + _global_log[variable_name] = [] + + def log_gradient_norm_inner(gradient): + gradient_norm = torch.norm(gradient, dim=-1) + _global_log[variable_name].append(to_np(gradient_norm)) + + tensor.register_hook(log_gradient_norm_inner) + + +def get_average(variable_name): + if variable_name not in _global_log: + return float('nan') + elif _global_log[variable_name]: + overall_tensor = np.concatenate(_global_log[variable_name]) + return np.mean(overall_tensor) + else: + return 0 + + +def clear_data(variable_name): + _global_log[variable_name] = [] diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_profiling.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_profiling.py new file mode 100644 index 00000000..976ec9eb --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/network/utils/utils_profiling.py @@ -0,0 +1,5 @@ +try: + profile +except NameError: + def profile(func): + return func diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/notebooks/run_inference.ipynb b/DGLPyTorch/DrugDiscovery/RoseTTAFold/notebooks/run_inference.ipynb new file mode 100644 index 00000000..e2cbcb4c --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/notebooks/run_inference.ipynb @@ -0,0 +1,267 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# RoseTTAFold Inference Pipeline for Protein Folding." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook acts as a guide on how to use RoseTTAFold for protein folding using NVIDIA GPUs." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "RoseTTAFold takes an aminoacid sequence and returns a [PDB](https://en.wikipedia.org/wiki/Protein_Data_Bank) file with atoms and their 3D coordinates. Having just an input sequence is often insufficient to provide an accurate structure prediction - to improve the accuracy, RoseTTAFold can take additional inputs in form of [MSA](https://en.wikipedia.org/wiki/Multiple_sequence_alignment), [Secondary structures](https://proteopedia.org/wiki/index.php/Structural_templates#Secondary_structure_elements), and [Templates](https://proteopedia.org/wiki/index.php/Structural_Templates). Those additional inputs are derived from the input sequence, and prepared automatically through the database lookup. Please make sure you downloaded the required databases in the Quick Start Guide.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's import a pipeline launcher that will allow us to execute a prediction pipeline:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pipeline_utils import execute_pipeline, cleanup, display_pdb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can provide an input in two ways:\n", + "1. As a path to a FASTA file containing a sequence\n", + "2. As a string sequence itself.\n", + "\n", + "We have provided an example FASTA file in `example/input.fa`. Running the pipeline is as simple as putting that path as a parameter to the `execute_pipeline` function. The example sequence is:\n", + "`MAAPTPADKSMMAAVPEWTITNLKRVCNAGNTSCTWTFGVDTHLATATSCTYVVKANANASQASGGPVTCGPYTITSSWSGQFGPNNGFTTFAVTDFSKKLIVWPAYTDVQVQAGKVVSPNQSYAPANLPLEHHHHHH`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Running the pipeline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The pipeline has the following steps:\n", + "1. Run HHblits - the script is looking for matching MSAs in the `UniRef30` database.\n", + "2. Run PSIPRED - the script is looking for matching Secondary Structures \n", + "3. Run hhsearch - the script is looking for matching Templates\n", + "4. Run iterative structure refinement with SE(3)-Transformer." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running inference on 1078 Tsp1, Trichoderma virens, 138 residues|\n", + "\n", + "Running HHblits - looking for the MSAs\n", + "Running PSIPRED - looking for the Secondary Structures\n", + "Running hhsearch - looking for the Templates\n", + "Running end-to-end prediction\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using backend: pytorch\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Done.\n", + "Output saved as /results/output.e2e.pdb\n" + ] + } + ], + "source": [ + "execute_pipeline(\"example/input.fa\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the script has finished successfully, there should be an output PDB file present in the `/results` directory." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Result visualization" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install visualization packages:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Building jupyterlab assets (build:prod:minimize)\n" + ] + } + ], + "source": [ + "!jupyter labextension install jupyterlab_3dmol" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Load the output file and display it using `py3Dmol`. Please note that the image is interactive - it can be manipulated using the mouse." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/3dmoljs_load.v0": "
\n

You appear to be running in JupyterLab (or JavaScript failed to load for some other reason). You need to install the 3dmol extension:
\n jupyter labextension install jupyterlab_3dmol

\n
\n", + "text/html": [ + "
\n", + "

You appear to be running in JupyterLab (or JavaScript failed to load for some other reason). You need to install the 3dmol extension:
\n", + " jupyter labextension install jupyterlab_3dmol

\n", + "
\n", + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "display_pdb(\"/results/output.e2e.pdb\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Make sure you remove temporary files before you run another pipeline:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "cleanup()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/pipeline_utils.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/pipeline_utils.py new file mode 100644 index 00000000..3be9746b --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/pipeline_utils.py @@ -0,0 +1,42 @@ +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import py3Dmol + + +def execute_pipeline(sequence): + if os.path.isfile(sequence): + with open(sequence, "r") as f: + title = f.readlines()[0][2:] + print(f"Running inference on {title}") + os.system(f"bash run_inference_pipeline.sh {sequence}") + else: + try: + with open("temp_input.fa", "w") as f: + f.write(f"> {sequence[:8]}...\n") + f.write(sequence.upper()) + print(f"Running inference on {sequence[:8]}...") + os.system(f"bash run_inference_pipeline.sh temp_input.fa") + except: + print("Unable to run the pipeline.") + raise + + +def display_pdb(path_to_pdb): + with open(path_to_pdb) as ifile: + protein = "".join([x for x in ifile]) + view = py3Dmol.view(width=400, height=300) + view.addModelsAsFrames(protein) + view.setStyle({'model': -1}, {"cartoon": {'color': 'spectrum'}}) + view.zoomTo() + view.show() + + +def cleanup(): + os.system("rm t000*") diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/requirements.txt b/DGLPyTorch/DrugDiscovery/RoseTTAFold/requirements.txt new file mode 100644 index 00000000..1290950c --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/requirements.txt @@ -0,0 +1,4 @@ +pandas==1.1.4 +scikit-learn==0.24 +packaging==21.0 +py3Dmol==1.7.0 diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_e2e_ver.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_e2e_ver.sh new file mode 100644 index 00000000..572b3437 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_e2e_ver.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# make the script stop when error (non-true exit code) is occured +set -e + +############################################################ +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)" +eval "$__conda_setup" +unset __conda_setup +# <<< conda initialize <<< +############################################################ + +SCRIPT=`realpath -s $0` +export PIPEDIR=`dirname $SCRIPT` + +CPU="8" # number of CPUs to use +MEM="64" # max memory (in GB) + +# Inputs: +IN="$1" # input.fasta +WDIR=`realpath -s $2` # working folder + + +LEN=`tail -n1 $IN | wc -m` + +mkdir -p $WDIR/log + +conda activate RoseTTAFold +############################################################ +# 1. generate MSAs +############################################################ +if [ ! -s $WDIR/t000_.msa0.a3m ] +then + echo "Running HHblits" + $PIPEDIR/input_prep/make_msa.sh $IN $WDIR $CPU $MEM > $WDIR/log/make_msa.stdout 2> $WDIR/log/make_msa.stderr +fi + + +############################################################ +# 2. predict secondary structure for HHsearch run +############################################################ +if [ ! -s $WDIR/t000_.ss2 ] +then + echo "Running PSIPRED" + $PIPEDIR/input_prep/make_ss.sh $WDIR/t000_.msa0.a3m $WDIR/t000_.ss2 > $WDIR/log/make_ss.stdout 2> $WDIR/log/make_ss.stderr +fi + + +############################################################ +# 3. search for templates +############################################################ +DB="$PIPEDIR/pdb100_2021Mar03/pdb100_2021Mar03" +if [ ! -s $WDIR/t000_.hhr ] +then + echo "Running hhsearch" + HH="hhsearch -b 50 -B 500 -z 50 -Z 500 -mact 0.05 -cpu $CPU -maxmem $MEM -aliw 100000 -e 100 -p 5.0 -d $DB" + cat $WDIR/t000_.ss2 $WDIR/t000_.msa0.a3m > $WDIR/t000_.msa0.ss2.a3m + $HH -i $WDIR/t000_.msa0.ss2.a3m -o $WDIR/t000_.hhr -atab $WDIR/t000_.atab -v 0 > $WDIR/log/hhsearch.stdout 2> $WDIR/log/hhsearch.stderr +fi + + +############################################################ +# 4. end-to-end prediction +############################################################ +if [ ! -s $WDIR/t000_.3track.npz ] +then + echo "Running end-to-end prediction" + python $PIPEDIR/network/predict_e2e.py \ + -m $PIPEDIR/weights \ + -i $WDIR/t000_.msa0.a3m \ + -o $WDIR/t000_.e2e \ + --hhr $WDIR/t000_.hhr \ + --atab $WDIR/t000_.atab \ + --db $DB 1> $WDIR/log/network.stdout 2> $WDIR/log/network.stderr +fi +echo "Done" diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.py b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.py new file mode 100644 index 00000000..482e4ee3 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.py @@ -0,0 +1,17 @@ +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +from pipeline_utils import execute_pipeline + +PARSER = argparse.ArgumentParser() +PARSER.add_argument('sequence', type=str, help='A sequence or a path to a FASTA file') + +if __name__ == '__main__': + args = PARSER.parse_args() + execute_pipeline(args.sequence) diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.sh new file mode 100755 index 00000000..dab62c3f --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_inference_pipeline.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# make the script stop when error (non-true exit code) is occurred +set -e +CPU="32" # number of CPUs to use +MEM="64" # max memory (in GB) + +# Inputs: +IN="$1" # input.fasta +#WDIR=`realpath -s $2` # working folder +DATABASES_DIR="${2:-/databases}" +WEIGHTS_DIR="${3:-/weights}" +WDIR="${4:-.}" + + + +LEN=`tail -n1 $IN | wc -m` + +mkdir -p $WDIR/logs + +########################################################### +# 1. generate MSAs +############################################################ +if [ ! -s $WDIR/t000_.msa0.a3m ] +then + echo "Running HHblits - looking for the MSAs" + /workspace/rf/input_prep/make_msa.sh $IN $WDIR $CPU $MEM $DATABASES_DIR > $WDIR/logs/make_msa.stdout 2> $WDIR/logs/make_msa.stderr +fi + + +############################################################ +# 2. predict secondary structure for HHsearch run +############################################################ +if [ ! -s $WDIR/t000_.ss2 ] +then + echo "Running PSIPRED - looking for the Secondary Structures" + /workspace/rf/input_prep/make_ss.sh $WDIR/t000_.msa0.a3m $WDIR/t000_.ss2 > $WDIR/logs/make_ss.stdout 2> $WDIR/logs/make_ss.stderr +fi + + +############################################################ +# 3. search for templates +############################################################ +if [ ! -s $WDIR/t000_.hhr ] +then + echo "Running hhsearch - looking for the Templates" + /workspace/rf/input_prep/prepare_templates.sh $WDIR $CPU $MEM $DATABASES_DIR > $WDIR/logs/prepare_templates.stdout 2> $WDIR/logs/prepare_templates.stderr +fi + + +############################################################ +# 4. end-to-end prediction +############################################################ +DB="$DATABASES_DIR/pdb100_2021Mar03/pdb100_2021Mar03" +if [ ! -s $WDIR/t000_.3track.npz ] +then + echo "Running end-to-end prediction" + python /workspace/rf/network/predict_e2e.py \ + -m $WEIGHTS_DIR \ + -i $WDIR/t000_.msa0.a3m \ + -o /results/output.e2e \ + --hhr $WDIR/t000_.hhr \ + --atab $WDIR/t000_.atab \ + --db $DB +fi +echo "Done." +echo "Output saved as /results/output.e2e.pdb" + +# 1> $WDIR/log/network.stdout 2> $WDIR/log/network.stderr \ No newline at end of file diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_pyrosetta_ver.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_pyrosetta_ver.sh new file mode 100755 index 00000000..53a67871 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/run_pyrosetta_ver.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# make the script stop when error (non-true exit code) is occured +set -e + +############################################################ +# >>> conda initialize >>> +# !! Contents within this block are managed by 'conda init' !! +__conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)" +eval "$__conda_setup" +unset __conda_setup +# <<< conda initialize <<< +############################################################ + +SCRIPT=`realpath -s $0` +export PIPEDIR=`dirname $SCRIPT` + +CPU="8" # number of CPUs to use +MEM="64" # max memory (in GB) + +# Inputs: +IN="$1" # input.fasta +WDIR=`realpath -s $2` # working folder + + +LEN=`tail -n1 $IN | wc -m` + +mkdir -p $WDIR/log + +conda activate RoseTTAFold +############################################################ +# 1. generate MSAs +############################################################ +if [ ! -s $WDIR/t000_.msa0.a3m ] +then + echo "Running HHblits" + $PIPEDIR/input_prep/make_msa.sh $IN $WDIR $CPU $MEM > $WDIR/log/make_msa.stdout 2> $WDIR/log/make_msa.stderr +fi + + +############################################################ +# 2. predict secondary structure for HHsearch run +############################################################ +if [ ! -s $WDIR/t000_.ss2 ] +then + echo "Running PSIPRED" + $PIPEDIR/input_prep/make_ss.sh $WDIR/t000_.msa0.a3m $WDIR/t000_.ss2 > $WDIR/log/make_ss.stdout 2> $WDIR/log/make_ss.stderr +fi + + +############################################################ +# 3. search for templates +############################################################ +DB="$PIPEDIR/pdb100_2021Mar03/pdb100_2021Mar03" +if [ ! -s $WDIR/t000_.hhr ] +then + echo "Running hhsearch" + HH="hhsearch -b 50 -B 500 -z 50 -Z 500 -mact 0.05 -cpu $CPU -maxmem $MEM -aliw 100000 -e 100 -p 5.0 -d $DB" + cat $WDIR/t000_.ss2 $WDIR/t000_.msa0.a3m > $WDIR/t000_.msa0.ss2.a3m + $HH -i $WDIR/t000_.msa0.ss2.a3m -o $WDIR/t000_.hhr -atab $WDIR/t000_.atab -v 0 > $WDIR/log/hhsearch.stdout 2> $WDIR/log/hhsearch.stderr +fi + + +############################################################ +# 4. predict distances and orientations +############################################################ +if [ ! -s $WDIR/t000_.3track.npz ] +then + echo "Predicting distance and orientations" + python $PIPEDIR/network/predict_pyRosetta.py \ + -m $PIPEDIR/weights \ + -i $WDIR/t000_.msa0.a3m \ + -o $WDIR/t000_.3track \ + --hhr $WDIR/t000_.hhr \ + --atab $WDIR/t000_.atab \ + --db $DB 1> $WDIR/log/network.stdout 2> $WDIR/log/network.stderr +fi + +############################################################ +# 5. perform modeling +############################################################ +mkdir -p $WDIR/pdb-3track + +conda deactivate +conda activate folding + +for m in 0 1 2 +do + for p in 0.05 0.15 0.25 0.35 0.45 + do + for ((i=0;i<1;i++)) + do + if [ ! -f $WDIR/pdb-3track/model${i}_${m}_${p}.pdb ]; then + echo "python -u $PIPEDIR/folding/RosettaTR.py --roll -r 3 -pd $p -m $m -sg 7,3 $WDIR/t000_.3track.npz $IN $WDIR/pdb-3track/model${i}_${m}_${p}.pdb" + fi + done + done +done > $WDIR/parallel.fold.list + +N=`cat $WDIR/parallel.fold.list | wc -l` +if [ "$N" -gt "0" ]; then + echo "Running parallel RosettaTR.py" + parallel -j $CPU < $WDIR/parallel.fold.list > $WDIR/log/folding.stdout 2> $WDIR/log/folding.stderr +fi + +############################################################ +# 6. Pick final models +############################################################ +count=$(find $WDIR/pdb-3track -maxdepth 1 -name '*.npz' | grep -v 'features' | wc -l) +if [ "$count" -lt "15" ]; then + # run DeepAccNet-msa + echo "Running DeepAccNet-msa" + python $PIPEDIR/DAN-msa/ErrorPredictorMSA.py --roll -p $CPU $WDIR/t000_.3track.npz $WDIR/pdb-3track $WDIR/pdb-3track 1> $WDIR/log/DAN_msa.stdout 2> $WDIR/log/DAN_msa.stderr +fi + +if [ ! -s $WDIR/model/model_5.crderr.pdb ] +then + echo "Picking final models" + python -u -W ignore $PIPEDIR/DAN-msa/pick_final_models.div.py \ + $WDIR/pdb-3track $WDIR/model $CPU > $WDIR/log/pick.stdout 2> $WDIR/log/pick.stderr + echo "Final models saved in: $2/model" +fi +echo "Done" diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/scripts/download_databases.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/scripts/download_databases.sh new file mode 100644 index 00000000..57e34a1f --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/scripts/download_databases.sh @@ -0,0 +1,33 @@ +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +WEIGHTS_DIR="${1:-.}" +DATABASES_DIR="${2:-./databases}" + +mkdir -p $DATABASES_DIR + +echo "Downloading pre-trained model weights [1G]" +wget https://files.ipd.uw.edu/pub/RoseTTAFold/weights.tar.gz +tar xfz weights.tar.gz -C $WEIGHTS_DIR + + +# uniref30 [46G] +echo "Downloading UniRef30_2020_06 [46G]" +wget http://wwwuser.gwdg.de/~compbiol/uniclust/2020_06/UniRef30_2020_06_hhsuite.tar.gz -P $DATABASES_DIR +mkdir -p $DATABASES_DIR/UniRef30_2020_06 +tar xfz $DATABASES_DIR/UniRef30_2020_06_hhsuite.tar.gz -C $DATABASES_DIR/UniRef30_2020_06 + +# BFD [272G] +#wget https://bfd.mmseqs.com/bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt.tar.gz +#mkdir -p bfd +#tar xfz bfd_metaclust_clu_complete_id30_c90_final_seq.sorted_opt.tar.gz -C ./bfd + +# structure templates (including *_a3m.ffdata, *_a3m.ffindex) [over 100G] +echo "Downloading pdb100_2021Mar03 [over 100G]" +wget https://files.ipd.uw.edu/pub/RoseTTAFold/pdb100_2021Mar03.tar.gz -P $DATABASES_DIR +tar xfz $DATABASES_DIR/pdb100_2021Mar03.tar.gz -C $DATABASES_DIR/ diff --git a/DGLPyTorch/DrugDiscovery/RoseTTAFold/start_jupyter.sh b/DGLPyTorch/DrugDiscovery/RoseTTAFold/start_jupyter.sh new file mode 100644 index 00000000..978945f2 --- /dev/null +++ b/DGLPyTorch/DrugDiscovery/RoseTTAFold/start_jupyter.sh @@ -0,0 +1,11 @@ +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +PORT="${1:-6006}" + +jupyter lab --ip=0.0.0.0 --allow-root --no-browser --NotebookApp.token='' --notebook-dir=/workspace/rf --NotebookApp.allow_origin='*' --port $PORT \ No newline at end of file