From bad62d83c6d0cd67167eeaba4496021d02a38270 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 13:28:34 +0200 Subject: [PATCH 01/94] take static methods out of tcrdist --- src/scirpy/ir_dist/metrics.py | 95 +++++++++++++++++------------------ 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 19a059637..dd6708992 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -413,6 +413,51 @@ def _make_numba_matrix(distance_matrix: dict, alphabet: str = "ARNDCQEGHILKMFPST dm[alphabet.index(aa2), alphabet.index(aa1)] = d return dm +def _seqs2mat( + seqs: Sequence[str], alphabet: str = "ARNDCQEGHILKMFPSTWYVBZX", max_len: Union[None, int] = None + ) -> tuple[np.ndarray, np.ndarray]: + """Convert a collection of gene sequences into a + numpy matrix of integers for fast comparison. + + Parameters + ---------- + seqs: + Sequence of strings + + Returns + ------- + mat: + matrix with gene sequences encoded as integers + L: + vector with length values of the gene sequences in the matrix + + Examples + -------- + >>> seqs2mat(["CAT", "HAT"]) + array([[ 4, 0, 16], + [ 8, 0, 16]], dtype=int8) + + Notes + ----- + Requires all seqs to have the same length, therefore shorter sequences + are filled up with -1 entries at the end. + """ + if max_len is None: + max_len = np.max([len(s) for s in seqs]) + mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) + L = np.zeros(len(seqs), dtype=np.int8) + for si, s in enumerate(seqs): + L[si] = len(s) + for aai in range(max_len): + if aai >= len(s): + break + try: + mat[si, aai] = alphabet.index(s[aai]) + except ValueError: + # Unknown symbols given value for last column/row of matrix + mat[si, aai] = len(alphabet) + return mat, L + class TCRdistDistanceCalculator: """Computes pairwise distances between TCR CDR3 sequences based on the "tcrdist" distance metric. @@ -468,52 +513,6 @@ def __init__( self.cutoff = cutoff self.n_jobs = n_jobs - @staticmethod - def _seqs2mat( - seqs: Sequence[str], alphabet: str = parasail_aa_alphabet, max_len: Union[None, int] = None - ) -> tuple[np.ndarray, np.ndarray]: - """Convert a collection of gene sequences into a - numpy matrix of integers for fast comparison. - - Parameters - ---------- - seqs: - Sequence of strings - - Returns - ------- - mat: - matrix with gene sequences encoded as integers - L: - vector with length values of the gene sequences in the matrix - - Examples - -------- - >>> seqs2mat(["CAT", "HAT"]) - array([[ 4, 0, 16], - [ 8, 0, 16]], dtype=int8) - - Notes - ----- - Requires all seqs to have the same length, therefore shorter sequences - are filled up with -1 entries at the end. - """ - if max_len is None: - max_len = np.max([len(s) for s in seqs]) - mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) - L = np.zeros(len(seqs), dtype=np.int8) - for si, s in enumerate(seqs): - L[si] = len(s) - for aai in range(max_len): - if aai >= len(s): - break - try: - mat[si, aai] = alphabet.index(s[aai]) - except ValueError: - # Unknown symbols given value for last column/row of matrix - mat[si, aai] = len(alphabet) - return mat, L - @staticmethod def _tcrdist_mat( *, @@ -676,8 +675,8 @@ def _calc_dist_mat_block( if len(seqs) == 0 or len(seqs2) == 0: return csr_matrix((len(seqs), len(seqs2))) - seqs_mat1, seqs_L1 = self._seqs2mat(seqs) - seqs_mat2, seqs_L2 = self._seqs2mat(seqs2) + seqs_mat1, seqs_L1 = _seqs2mat(seqs) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2) data_rows, indices_rows, row_element_counts = self._tcrdist_mat( seqs_mat1=seqs_mat1, From 72565bf48ce19a35961f1b615c9cfec3b578396c Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 13:52:42 +0200 Subject: [PATCH 02/94] made _tcrdist_mat a normal class method --- src/scirpy/ir_dist/metrics.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index dd6708992..51f5fc4a9 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -513,8 +513,8 @@ def __init__( self.cutoff = cutoff self.n_jobs = n_jobs - @staticmethod def _tcrdist_mat( + self, *, seqs_mat1: np.ndarray, seqs_mat2: np.ndarray, @@ -523,12 +523,6 @@ def _tcrdist_mat( is_symmetric: bool = False, start_column: int = 0, distance_matrix: np.ndarray = tcr_nb_distance_matrix, - dist_weight: int = 3, - gap_penalty: int = 4, - ntrim: int = 3, - ctrim: int = 2, - fixed_gappos: bool = True, - cutoff: int = 20, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: """Computes the pairwise TCRdist distances for sequences in seqs_mat1 and seqs_mat2. @@ -583,6 +577,13 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ + cutoff=self.cutoff + dist_weight=self.dist_weight + gap_penalty=self.gap_penalty + ntrim=self.ntrim + ctrim=self.ctrim + fixed_gappos=self.fixed_gappos + dist_mat_weighted = distance_matrix * dist_weight start_column *= is_symmetric @@ -685,12 +686,6 @@ def _calc_dist_mat_block( seqs_L2=seqs_L2, is_symmetric=is_symmetric, start_column=start_column, - dist_weight=self.dist_weight, - gap_penalty=self.gap_penalty, - ntrim=self.ntrim, - ctrim=self.ctrim, - fixed_gappos=self.fixed_gappos, - cutoff=self.cutoff, ) indptr = np.zeros(row_element_counts.shape[0] + 1) From add8e7f4246397faf4ae95c9bfc6ddad7409bd26 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 14:33:35 +0200 Subject: [PATCH 03/94] parent method NumbaDistanceCalculator extracted --- src/scirpy/ir_dist/metrics.py | 152 +++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 67 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 51f5fc4a9..343e0c332 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -2,7 +2,7 @@ import itertools import warnings from collections.abc import Sequence -from typing import Optional, Union +from typing import Optional, Union, Callable import joblib import numba as nb @@ -459,7 +459,87 @@ def _seqs2mat( return mat, L -class TCRdistDistanceCalculator: +class NumbaDistanceCalculator(abc.ABC): + def __init__(self): + super().__init__() + + @abc.abstractmethod + def _metric_mat( + self, + *, + seqs_mat1: np.ndarray, + seqs_mat2: np.ndarray, + seqs_L1: np.ndarray, + seqs_L2: np.ndarray, + is_symmetric: bool = False, + start_column: int = 0, + ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + pass + + def _calc_dist_mat_block( + self, + seqs: Sequence[str], + seqs2: Optional[Sequence[str]] = None, + is_symmetric: bool = False, + start_column: int = 0, + ) -> csr_matrix: + """Computes a block of the final TCRdist distance matrix and returns it as CSR matrix. + If the final result matrix that consists of all blocks together is symmetric, only the part of the block that would + contribute to the upper triangular matrix of the final result will be computed. + """ + if len(seqs) == 0 or len(seqs2) == 0: + return csr_matrix((len(seqs), len(seqs2))) + + seqs_mat1, seqs_L1 = _seqs2mat(seqs) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2) + + data_rows, indices_rows, row_element_counts = self._metric_mat( + seqs_mat1=seqs_mat1, + seqs_mat2=seqs_mat2, + seqs_L1=seqs_L1, + seqs_L2=seqs_L2, + is_symmetric=is_symmetric, + start_column=start_column, + ) + + indptr = np.zeros(row_element_counts.shape[0] + 1) + indptr[1:] = np.cumsum(row_element_counts) + data, indices = np.concatenate(data_rows), np.concatenate(indices_rows) + sparse_distance_matrix = csr_matrix((data, indices, indptr), shape=(len(seqs), len(seqs2))) + return sparse_distance_matrix + + def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = None) -> csr_matrix: + """Calculates the pairwise distances between two vectors of gene sequences based on the TCRdist distance metric + and returns a CSR distance matrix + """ + if seqs2 is None: + seqs2 = seqs + + seqs = np.array(seqs) + seqs2 = np.array(seqs2) + is_symmetric = np.array_equal(seqs, seqs2) + n_blocks = self.n_jobs * 2 + + if self.n_jobs > 1: + split_seqs = np.array_split(seqs, n_blocks) + start_columns = np.cumsum([0] + [len(seq) for seq in split_seqs[:-1]]) + arguments = [(split_seqs[x], seqs2, is_symmetric, start_columns[x]) for x in range(n_blocks)] + + delayed_jobs = [joblib.delayed(self._calc_dist_mat_block)(*args) for args in arguments] + results = list(_parallelize_with_joblib(delayed_jobs, total=len(arguments), n_jobs=self.n_jobs)) + distance_matrix_csr = scipy.sparse.vstack(results) + else: + distance_matrix_csr = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) + + if is_symmetric: + upper_triangular_distance_matrix = distance_matrix_csr + full_distance_matrix = upper_triangular_distance_matrix.maximum(upper_triangular_distance_matrix.T) + else: + full_distance_matrix = distance_matrix_csr + + return full_distance_matrix + +class TCRdistDistanceCalculator(NumbaDistanceCalculator): """Computes pairwise distances between TCR CDR3 sequences based on the "tcrdist" distance metric. The code of this class is heavily based on `pwseqdist `_. @@ -522,7 +602,6 @@ def _tcrdist_mat( seqs_L2: np.ndarray, is_symmetric: bool = False, start_column: int = 0, - distance_matrix: np.ndarray = tcr_nb_distance_matrix, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: """Computes the pairwise TCRdist distances for sequences in seqs_mat1 and seqs_mat2. @@ -577,6 +656,7 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ + distance_matrix=self.tcr_nb_distance_matrix cutoff=self.cutoff dist_weight=self.dist_weight gap_penalty=self.gap_penalty @@ -661,70 +741,8 @@ def _nb_tcrdist_mat(): data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() return data_rows, indices_rows, row_element_counts - - def _calc_dist_mat_block( - self, - seqs: Sequence[str], - seqs2: Optional[Sequence[str]] = None, - is_symmetric: bool = False, - start_column: int = 0, - ) -> csr_matrix: - """Computes a block of the final TCRdist distance matrix and returns it as CSR matrix. - If the final result matrix that consists of all blocks together is symmetric, only the part of the block that would - contribute to the upper triangular matrix of the final result will be computed. - """ - if len(seqs) == 0 or len(seqs2) == 0: - return csr_matrix((len(seqs), len(seqs2))) - - seqs_mat1, seqs_L1 = _seqs2mat(seqs) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2) - - data_rows, indices_rows, row_element_counts = self._tcrdist_mat( - seqs_mat1=seqs_mat1, - seqs_mat2=seqs_mat2, - seqs_L1=seqs_L1, - seqs_L2=seqs_L2, - is_symmetric=is_symmetric, - start_column=start_column, - ) - - indptr = np.zeros(row_element_counts.shape[0] + 1) - indptr[1:] = np.cumsum(row_element_counts) - data, indices = np.concatenate(data_rows), np.concatenate(indices_rows) - sparse_distance_matrix = csr_matrix((data, indices, indptr), shape=(len(seqs), len(seqs2))) - return sparse_distance_matrix - - def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = None) -> csr_matrix: - """Calculates the pairwise distances between two vectors of gene sequences based on the TCRdist distance metric - and returns a CSR distance matrix - """ - if seqs2 is None: - seqs2 = seqs - - seqs = np.array(seqs) - seqs2 = np.array(seqs2) - is_symmetric = np.array_equal(seqs, seqs2) - n_blocks = self.n_jobs * 2 - - if self.n_jobs > 1: - split_seqs = np.array_split(seqs, n_blocks) - start_columns = np.cumsum([0] + [len(seq) for seq in split_seqs[:-1]]) - arguments = [(split_seqs[x], seqs2, is_symmetric, start_columns[x]) for x in range(n_blocks)] - - delayed_jobs = [joblib.delayed(self._calc_dist_mat_block)(*args) for args in arguments] - results = list(_parallelize_with_joblib(delayed_jobs, total=len(arguments), n_jobs=self.n_jobs)) - distance_matrix_csr = scipy.sparse.vstack(results) - else: - distance_matrix_csr = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) - - if is_symmetric: - upper_triangular_distance_matrix = distance_matrix_csr - full_distance_matrix = upper_triangular_distance_matrix.maximum(upper_triangular_distance_matrix.T) - else: - full_distance_matrix = distance_matrix_csr - - return full_distance_matrix - + + _metric_mat = _tcrdist_mat @_doc_params(params=_doc_params_parallel_distance_calculator) class AlignmentDistanceCalculator(ParallelDistanceCalculator): From e9c0642fecff7f9582a448ad6c1c2503f9adaf70 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 14:54:46 +0200 Subject: [PATCH 04/94] numba version of hamming distance implemented --- src/scirpy/ir_dist/metrics.py | 118 ++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 343e0c332..58dfed34e 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -343,56 +343,6 @@ def _compute_block(self, seqs1, seqs2, origin): return result -@_doc_params(params=_doc_params_parallel_distance_calculator) -class HammingDistanceCalculator(ParallelDistanceCalculator): - """\ - Calculates the Hamming distance between sequences of identical length. - - The edit distance is the total number of substitution events. Sequences - with different lengths will be treated as though they exceeded the - distance-cutoff, i.e. they receive a distance of `0` in the sparse distance - matrix and will not be connected by an edge in the graph. - - This class relies on `Python-levenshtein `_ - to calculate the distances. - - Choosing a cutoff: - Each modification stands for a substitution event. - While lacking empirical data, it seems unlikely that CDR3 sequences with more - than two modifications still recognize the same antigen. - - Parameters - ---------- - cutoff - Will eleminate distances > cutoff to make efficient - use of sparse matrices. The default cutoff is `2`. - {params} - """ - - def __init__(self, cutoff: int = 2, **kwargs): - super().__init__(cutoff, **kwargs) - - def _compute_block(self, seqs1, seqs2, origin): - origin_row, origin_col = origin - if seqs2 is not None: - # compute the full matrix - coord_iterator = itertools.product(enumerate(seqs1), enumerate(seqs2)) - else: - # compute only upper triangle in this case - coord_iterator = itertools.combinations_with_replacement(enumerate(seqs1), r=2) - - result = [] - for (row, s1), (col, s2) in coord_iterator: - # require identical length of sequences - if len(s1) != len(s2): - continue - d = hamming_dist(s1, s2) - if d <= self.cutoff: - result.append((d + 1, origin_row + row, origin_col + col)) - - return result - - def _make_numba_matrix(distance_matrix: dict, alphabet: str = "ARNDCQEGHILKMFPSTWYVBZX*") -> np.ndarray: """Creates a numba compatible distance matrix from a dict of tuples. @@ -538,7 +488,75 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No full_distance_matrix = distance_matrix_csr return full_distance_matrix + +class HammingDistanceCalculator(NumbaDistanceCalculator): + def __init__( + self, + n_jobs: int = 1, + cutoff: int = 2, + ): + self.n_jobs = n_jobs + self.cutoff = cutoff + + def _hamming_mat( + self, + *, + seqs_mat1: np.ndarray, + seqs_mat2: np.ndarray, + seqs_L1: np.ndarray, + seqs_L2: np.ndarray, + is_symmetric: bool = False, + start_column: int = 0, + ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + + cutoff=self.cutoff + start_column *= is_symmetric + + @nb.jit(nopython=True, parallel=False, nogil=True) + def _nb_hamming_mat(): + assert seqs_mat1.shape[0] == seqs_L1.shape[0] + assert seqs_mat2.shape[0] == seqs_L2.shape[0] + + data_rows = nb.typed.List() + indices_rows = nb.typed.List() + row_element_counts = np.zeros(seqs_mat1.shape[0]) + + empty_row = np.zeros(0) + for _ in range(0, seqs_mat1.shape[0]): + data_rows.append(empty_row) + indices_rows.append(empty_row) + + data_row = np.zeros(seqs_mat2.shape[0]) + indices_row = np.zeros(seqs_mat2.shape[0]) + for row_index in range(seqs_mat1.shape[0]): + row_end_index = 0 + for col_index in range(start_column + row_index * is_symmetric, seqs_mat2.shape[0]): + q_L = seqs_L1[row_index] + s_L = seqs_L2[col_index] + distance = 1 + + if q_L == s_L: + for i in range(0, q_L): + distance += seqs_mat1[row_index, i] != seqs_mat2[col_index, i] + + if distance <= cutoff + 1: + data_row[row_end_index] = distance + indices_row[row_end_index] = col_index + row_end_index += 1 + + data_rows[row_index] = data_row[0:row_end_index].copy() + indices_rows[row_index] = indices_row[0:row_end_index].copy() + row_element_counts[row_index] = row_end_index + return data_rows, indices_rows, row_element_counts + + data_rows, indices_rows, row_element_counts = _nb_hamming_mat() + + return data_rows, indices_rows, row_element_counts + _metric_mat = _hamming_mat + + + class TCRdistDistanceCalculator(NumbaDistanceCalculator): """Computes pairwise distances between TCR CDR3 sequences based on the "tcrdist" distance metric. From 68e0493ecfdb9aff19f35c3a9335c691854c13a6 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 15:28:35 +0200 Subject: [PATCH 05/94] hamming numba tests passed and reference test added --- .../hamming_WU3k_csr_result.npz | Bin 0 -> 6842 bytes .../hamming_test_data/hamming_WU3k_seqs.npy | Bin 0 -> 136528 bytes src/scirpy/tests/test_ir_dist_metrics.py | 14 ++++++++++++++ 3 files changed, 14 insertions(+) create mode 100644 src/scirpy/tests/data/hamming_test_data/hamming_WU3k_csr_result.npz create mode 100644 src/scirpy/tests/data/hamming_test_data/hamming_WU3k_seqs.npy diff --git a/src/scirpy/tests/data/hamming_test_data/hamming_WU3k_csr_result.npz b/src/scirpy/tests/data/hamming_test_data/hamming_WU3k_csr_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..7405b10c2f2958c2be32af81a90865d81426d101 GIT binary patch literal 6842 zcmaiZ2UJtb_cfq`6zRRUgsMV7dhfl9ARxU82!!5Lq<075~kSbs>fRsmn z^diy>pb(n>8_@UkeZRH7$-1nSn;GuRoW1wiXRd+vg^ScUI5?MqivuS-8Wu}^1qX+h z0|$o~hZ@J#)4|o=5qizjD+m{d{OmT+4+q-?+wvxZ#!3I0#qk;loBhrkCq^@2q>`Tc zim++o;9OlM^)5s$f0D zX)8xWWECO#Q3ia&E!bmm#k>)JzR?|m(^q9e#;IrNz~?q7|mj3XtqbcVSJ6`df7*fZYcIAdf6tEo-CC!IVA;Km=~EILc5}vdcJw_@XQ2Quo`A02MbGH&9k%|*OyJ<}X!P8cGL)zaK$S z3qel1c-J8@+3=4tE4vWlK+s5ylm*<%GokjQP~8cCHCjsBtQTVXH6ipr`izb5T1Kf8NDeus+S zKr(}n0Fya;kS8xe-5b?;oGIY;s5K~=%g%ZKD1~0?g&-w&XUhhuASf?aW(mZG;Y+2- zM1p%*xMtKih+@JM7kSfZzImuhKD47=Rnc2mGLr!b@b^$6y4Mhjnp)oAQa)w#Uc>xZ z%=o*|4)Y-ziCm~%4QHlAc5mqNIR8SBdXx26@DtvfatgHD6Tv z<~Amt>@;aq=4E!0&~sY~9g7=kh1FWzj|S)x2*tJxm9kKFe`6MgXkeS4~Z zp=!a65t7Zbb)F3SI_nXJDx{YzUuF)K#w}?HTIGP4Ge^wM&8+|no z+i-RfcvCFP@>!$2@@bgg6M_1nt65umWWhrTGkeJRNh9mSsFNx*`M^$u4}3xHa7zBc zLy`r5e5m?U>7Kv&W@x_1rd?~|7zz-{uqAQ6$i>hlJN?mR%bvuZq32}6@0NlbS**U} zn6}gXGCdv*`41|rc9gInIG>K=?WfkP%fe+MkM@@a^;lE12S~p4z|04~bz{Pw&feO6 zwdov~wF;;Ck~Y!YmVlZsH47PUPl@wwg7e50WOvrQp^L3H=-eK7X%I}oqrGoev-iBY zdtK0$@1t3NQ>6{)dEg`QpoIFMn7?Z5={`b}8pH(R*Ubi^4QS2R!u^XI_}<)W2+Qr- zpXea4^qe(CE6XiF{a)!#vxaPi$L-Z^XPin1xHZjVl#Vtx5ms*kj17ZA`NL^aVn-$HzJkq?4%XoQ7^$&-uC$-Jw>8Q5)9z>lF5`hu_yj5duThRcptp5hS?Ju zs)Ra7R!bs*M*(ksx6BO5_Onnut&%B&W`wc@BmD2xs2cg<>`_d%6k%)6Q5Wh&Oiy)8 zI*&EZm+kcFiL*v_3AMA)V@QT0$ui|y3-=yZinMb%ls_jsvp$TLhSIRW5Y#j)pgMaDJrBECmBW(( z8E503eX`|+eKd*2Qgb)Er?c17P8HGpCV}rV0c_s{J1Grk_i(uAk-S@kn~5PogRj*S z_CZD8CB)LUrbM>g%I>$yQ!I<;^8}`^=Cmz+!iUo*RD(~q%@}$ejW!T5IUOctXbs+b ztoBkOcKg%9Om9s zbnlvdlp3vjnB=at#5^D#vpe2F;C^M<@kh);)PnlFWS7T+zyC{E3MSHOW5rl*5&HeB zs$h7IM*ZfLaw~iFyEpr2K-8|x?ywwT^4WN^P?F-8gO0gb=T=pM!)Qit6JTyoykJ^@ zF5ZK!-56sVilJQdGdtezJf`VizD(G|1Ct!1NBf#WR=v@_@y$zMeRpX#3~UW!1~0x@ zY-0v9i0xqh=uJHz81JJ5;g4CcWnSdgTcOq~y;$xYOuyJk&y|G*Sk_R6ts4L2t>hhB z7Ntx5r1pd~Fv-~@XjiZ86>I`KcGzRaDqDr3u3OX%{MU_eDND*x5=+`qLQ6%fDw3WC zJ=q$$D-H3{mMq%VRNW4Y=fc%-7zsxDvN2BHE_>n7A|6KG%Ncr~!eTga<<%VR?IC`K zlbPr@ug;{4M{4*SN5hZu_q57Z32pmFW2QbcZ6yII%?5INJ?7}jc9#x5lpw%|PPrlX z35mS~fRv{WqV-MZNp0sAL^e86yR0WqNIjT+!(IupdhEhU1{&T)2+c%d^quaAm>=4f z3Ejxa&(_gaTW?bjN?b!}lpPJCylsy^%1lwoAdO}E>SU<2<$DD4o&6z<0uW53x2T9- z-#lmql7eYQ;oY(2NH9bFD^u|f!BzAtiDybB!0_$k_DiV|x?i9`erDQ+bd?%fv6 zvZE#)>jNu@Yyipn{!9M>6radZ>ncI`Qn}YPf`c4ui4WDsv_~goq-dTPm?6_~F4wX0 z!8jtbccPVBGwo-_Q&3DMyo*Bwb%)X`C{=Jvr0wa#z2vWC^C=NIt;8UWbWZW05884&#t$Nx zJI$$!gnE3KRVDI^5DLQaP);6iNVyk3NFr%i^3hL2Tbn^Vt79!=%Cw$`GB7H7; zfOi4CaM5L&97_J~Tga97S%>Z|HmpDG%DmX&P|7LEWxu|&?64?vD2LYDvb+4JH!1UB zv#0+kv9*fZE%J~c=Yz729fV8-l+`238rOA3>p5?yos0TA)cU(513S!h@V?!eJ>q8i z1ZZGfBg!7o`PfU%9UbE>UCq6BoBJvM6PlR6nm;m*)+T>*x8ceSOzCSLpq2aOPD%ZD z&Vmuz>m1t8+A^p8y*8!#;PJhKtnKlE{o}TjyNbTq^BHH7qa&l#QwlF_@kgYxjS(Y4b)c6^f)*~=LMCQug|Y~3VRpZ zusyIXZ+tnI{5>s!P4;+c6Exdu+$wl2om5dF6(3hqG1W}!ItM4m)x4pKk>35_` zjo)rJ{pgS#*bRs8qQ-J0cXQ3^?j+q#!joIWXtsVG^?3U_p7c5e^POj5PrgpK&wEI# z2c&ooRs^kn9`&$0%(U>`7*%VvXri^6G*efhxyGC+4GH-AVD3jfWoQvztg8D0dA34C zta_-O7?_(PPE!ez4vWYr;k@Chp)EB8dzk3JDPsOX>+-~9P~yfCdxVD1Pmzi%!nwq! zVjFLeVMkql5yaP3ex^j~LHq`3S|(*g1%{b!Cgu1yFcG&$R<4JPu0CtGss;lZL1)l&v%X(suo&;H{&x$)HdcZ4$96K$^Y$DhEFzYuS0x56 z0qg&0InOxOh`j0vjcJ-*lp_oLz{R>7c|WLEB!4$vJd}H7{xUmlO`NhN9w8?yT2n!Z zG7av)&2x!QQ#BZpnx^9bCN?JoN4iH+s&PvqRati^RdI=oBa5huB|yBId_b*Tn^7!7 z^Hc-GZyW{S0&H!NAT?6Tdc{@6{_&>_C(-gtS6#cr0cQc|%dMg{wSeGHuseHYWMOOS z_wjew05ourIMT}tA=nt&HPnQoYZC2csNf&C!<`_;@Lo}%qRBzoi9rDw54WW_D{gC} z{VJWQEGGI{zx%{;A?5{F8&Nb>K1u7-b8MdI2Qe}W!UKW*#CLY$$3nTC0twm0ykoW5 z6}h=7G&Qt*jMC#rN;od$e9)ndAmm_r5}B6AtU|7-=aUAUGw{UYQJ0WM@hKU`8zLw( zN~nL!SVKZHQZIU{)K8>@gB3!(=qwIck%4E$2A1 z8#-=QL17ltE9a2?nyN0d!gs!;==$bGS+)ZWR3Wnh^8%#?Ia))3Gd(rqXUN=!QnP$Z zc(`(Ytt^sC5eS&>v#^zmI`V3(@*C#xn-t@}eyVH5_$jsWFVUt*>J8@tXnWrqK?T6JT(Wca zIJ*w9ibvP@-m%In^X~K)s}Ywfy@#`N>E3u%(CS5K%~H(U4X#cD=>NmLF`V% zY3NX0$)TQ)Ye87PMn(t?D0oFp!SX(T0 z<0+DEq)5!#DCX@*G445r6IViYiEj_ZkG^`s!h%WG^DQGD+$-dH_xUX14@sPOv1O8) z$Un#hi$qoE0Ci7Jm-~uPcF=!B>|X{0TS5C~M_}q8X>-Gdr)j*8^?PlB`~mC&^AdTC zWQSq0Hv)!OX3lOv-SqrQCf?Y^vgMOM{Cx7pv*W32ZyQ}KORI>yDDm*8H6${&ZSR258&WKeo;2<_`MkzOHe?r z&9NBIps94Ad2Zf$UWfrYJ>Dx8ZZ9o8j=bhy7xV78U+z2fUy+*QE|=(II9(C^N2`_r zt!mfx^v;!@US{17YEx!K!+Cy&d**9aK=UegsoTG&H_Fu1_uL$ONDV!rb827xpPH?b z4zpYW|I}@O3x+=>?0;pqP(4341K90QO_nc4Ks{c_k>sL7)lvj-q<`#H;kjqpxw7dp zP!HIq*}=G=FH zf2F&BM4q)(sNj;I14V*)hf!8t`d^i{SApXde6iZNK6hJz*4m`m@pe7`x@9fo-`PbYGarksl1Y4PNA@-$q!Lc$OC4p>nURDlW{=vNkcw zfYTLNXQUHR256)B^e)BIIdjvN;9(daOA);$qQN*qC^Z(9C*d9W7aa{87u!uqlwF8?zIW68EyqMHwRVL8Xsd6>&j zFEHjQa6xcl9+*WaZE+Ri$0~MUkRDXt;Nxs>PkSd)w(G2g1yq{+z{8xMJ^(HnXbFtLp;i2*Rii=VqPZ>?GJ`FXz-}-e^ z*vL#O24>{N^<7`vI}|7$aPg>tnE&thF=rt^`}5!|{=dDQ`R~9#>yF#zRofBfga{Hs6z3%~fIKl|^0{TKE5U;np1`=5VNKmVP- z|A&A0_kREH{{HX%-`{&U{`Noq&2N75kA5Bhy3YUX*Y&@u^S}G`v(NtI*U$g7&hG!C zUq_vWBV5mV#u?MIuKz;AWO?ZD(O-G0BK`ON*(dL};57+Xb>Q#>(B>-4hE?&>$*yu^~51dD!V)yo{?4Sm|9_`D-5a^7GxkP`4c8%JnP` zK0kKrRgWCw%*AxXdi1SsxaWiRtex)UVx^8(y|}bG zd!A-YIkwk1tAXFmM#rTy+IeFx%+(BExwOy~F5Wk|>MaYj@)wt{K3nm%Zac$Nv{KKny@E%WTu?#$(BdW**NoxH{ORj;pHdetZz{mu2H2S?oGKrZNe zlC>)rCw;-+c#KUqZ>)zevz{JtN}`zed`Nc>vUK@4*kyN+hJezhKrYM#G9Qx zV#MVG)>EzZ=no&_1+CUD2XwCHpl&_JRd4b|bFMM*0t#19{uS{X4hiRtMd1d`|+cD01|F}NEOdjGd?v7UAE7$Y!)OojFn8uZh z6V|S$2k_T_&Ro2_i+Rb@r*mAnc)Z}VEW{H(n?pVM?mT-RUAfjSyWUA*<7JOqt>KkR ztMy|#=>2p*qf6GX(_7fq@JW7gd3V;!k8kKrFQ^$lUsv{ny6tUT{l!ge&wov@{1qMna|JsHm|IAqvP_!`#8Ma_wt#Gb?z_PaD7JKPA<+|-qDp_ zP&2;zd}r^i+u!yx+GOjTU)Pws{IlM0v#0UlsrDIH-QqXBaExO|oY{$j88Lf!frBQ8&|SpaZcG*|G;qn(V`OQ9@eC+y*%SX)kf&C4=>1S*?!&fit zJDqjQF-AV%;^FM}!vU@TbX;07v0XdU{i$9o`%dab|IQxK>AmLQQ?BP}b;wk_@cB;d zu6o0LlC7HhHAdWPzeRU6tKD7wMqKO7zP`hHiLHwfmj_RC5O&9vRXzQr*RZc#Il#02 zZ}wZbyvXr{Khm45nZsTY{h#C)D@XR9CmxUV;wHmS>(_DREIsFXH^&{ak;7_2S~GTRfIy z^T?f=u6z$3aq&_+*E_tr{TNqn_Axvf9Z&YO5to@=5jXAT|bYwo_EmBgI#7l z@}rNjFw<}T9Vz?7C-0$fQ#V?1vq!AvZR7(_Y1SKhk7itRt!nPLVLPXGJG!r2+Vlk$ z&T1djt#9MX#d4_IxsUmLm#w9c+Zc4~Ik zyW&U3#p8#wb(`apoY9LR2lDHh54*V->BWl1r&$|uX-+1b&Eve|@ra8d2k`+{zg+V+ zt6#@eF9&8a@@d`hvY+uc*L<*!l|AjMSKVnmd+?+;I&Ss|bEIA?I_vghjJRhePrlK| zd`PCOU(I|*^Cxw~%-)ciVeL2I=xT(Y{sR}!Hej3 z{v;2c$8^>o=?S0t`Dq{H>4EU0{nI|ivwl2$XX|&~@qARbb9SS9$AgmaqmSjEr}SlS zpda^>y5VBrsu4ELKGSzAdvDz~)Qd6VnP+ue{q_GXiC%U3mcOa_NblV{;=Got-sCjd z%KqbXb6Jbu-T8dx%9q;61K!hKt4H7J7A{TxU5d9pbHtz3tc~=_n1lB8o}bP04Q~3F z{vBrBKJ+0yL9b)CW_MhASL=^;9-Y5>OHVt0Q!lM*Bn#1uzuFs}UQFIT$yhzs)=j>i zYMpVj-{=qDrRa}0E8o#NAHvQ4BYS*S$Gg3uZfSkMz?3gJ#?E}Re2n}Jj~6n#%BovmW5&g#b!UGJxZ~pSm3FzL@%ZuIfMXFKnW5!ae=9p|3-WS^9)v!~enPCi!W8tKK|t$#i8Hbz{Y#s{9t zUA_~q)^NrRKehAj;EA8rbT&p@&&N+Sr1=~DhMRqXA1nP;x4n&#UjHU#z1cH&-|;5s zxIE+qAM{_|%QF1LUu(v5c>c`vnf$HP@2WRkzT!k*KA*kY>-N{^xcqgb_i$IU3*K>M z%X+b}!%rXPo!aHN)h);9xX~AO^1SNBh>Ia#>gJq-2e>QyaK}A+ zx|6S(_823s+FQS~UQ<8wxTaan?);c>J&Utc*J14rJ-6dQ-QIF#DjDXl+Qs7)KRO@m z3D!^RZV%TZtzKLjX_g5;ul&1Y-FA#lZ|cS8^hSK(%Uv!yZuS})X~s`~-0i(}YwH6R z4sUmK*6qjW^nQ{B;EpQ?*3QRo?K^nkI)5v_GUW)Lzr5bzk9i#q0T04d;eoNUh)bKjjK90S zW`d4OYxK&;xAbD-(d`h!wBRPIbYZQI5jRY}-uc+gGvlUb)hF6{edmF>I&L)MB@fuZuwRo@Z{?0J7c6bd#;|f4?oe|acRww@1>vSEFL(^06ymG zo>R9D>xP%Uz|8lF9e(EnuAHTxztI=;>IFG-Sii$uxxUNENH1-k$K$GmQ-|gLcTT(EX#EAV?}G-HssPqV|sbl+Zk8A&i!^*C#>@r z`9LqOdO6fz?>Vpj+eXLbrA+W5ePTXduV^1}J@>@Zdh9hu+<0s4^wO@5&a3;xjEm`@ z{TunEH(uHwR{X5n-bSZ4EP7>v2j)`4{H`wB(Q#=$?-RWCnPa8*W?cH7=3>T;#&71q zdS%Kvd*-SK^~iAF@w|i1CmE~9T5jI+_A3nA@BBWdeRUt|xa##ckuyx+SD(J)I=z^DNG`I+C(n29qY;;< zI1ZW66Y7}$;Bnr4-M-Xu`H)Pg7bfj1dvDz~%!B^Iv}OnHZhwsQp7#dabAQlm4!UQf z<0h-Vx4Y)cYR$~oan+1&tlf8fJu>8CTBo|J*OTw;BfZI3*1hU!NqCKpD-YI>efA=A zJ?RVjtY593XsySF{$i#6_6K>;*EIV4E@O4ekQ)!u8&C9){5^ANl?UgYUY>FFggRkn zKhJgY-F<)WxU^yN*ZCX!Lwrr`%yE~iy5((jdaaL6yu35a6CdvMkGQ;)B{{(MGe7TG zb<1mXTwcUm8fneHcRkH7eYko-#-8+Erx*L|uj}OZ>AsE`uJt>1_u!dcKJao!XWf2` z5tqJrfGu;o>={?>_`y$^2-_O&GBM)fV9IPVmb$I>-IeS4s2Qj^~952tH*|YocDV)udW?0W8_16Aig?d@9aGzuKLKAee$Uv z9oHV5p3__ULmgK=>A1YW#!){%rH{9&q zyiiBnRd&ZczcZis(Q(robUw*i$4%X2jlbD*R&UVM8kKUcV zrfz?Yj{C`;4nE@Ym|kn+0nN@keLdoOULVjbTY7J2Z>ZacdTCXc)W}({BxkgnFYhJu zU&lxfTQyy*TmHbEzW^Px*A0 z>k&6P;|Cw?^H}DTjb!dx!}Q#hOOv|$efZW+BPNgi+^xTUb&QTn2bOcZrkg+JbIfuw z;?hJLW;Cy`>h{+daq;*Nt#qqbazXsj4#HA^FS=i}~yZt5;bX=Y~czO2r6E6RF%U5S?2koD7vBOOs=$7Zy z&>UU;Mm}I-(aP%`opt*$MqEsFayG}4o||#4+nTLc-#N?n-5y!D4EG&6)d`bUemL`E zg;}??T+h?1cJX&76P*t~{iI$oWyDR*qIbj zlfD~q`I@@Pt!vtM_MeWM_q%iQmh(#PMqJ-&3$7N$MA`larb5E7$vh z*6i!9&5`%V>V7%m$^tGXjnT`O)%|D0jb=Krd-fSW{lSA3{)}tSwQl~>j3*P(|0HMhhRG9pcRcCzX3x3XziYbTm~r)+HOs>8 z9?@~LH{{*%t(=9+2TZzoKC4yz`ZZ>Hvp?+AE8vc6{c`J!|E>JuS@#pI^;p}-@l|%S zKl0cbRy^)}2zU2;PEC7^J8o)6}Obx_$za?WBSbX%;U}n&+_)$K18RpI?Db5(u=+0!5x>s-`pqpz=L@4 zB*UYeg)3LG%LmshxvkrdG2`Zaxl*U_a=znczu9HaxcSaQ_ip`WTp3Gm@OSmy#*2PP>o=1#B7%CUN-Zv0+(XVz`4zu9kSezK>{^kQRX zU%&g?s&0La8JDMWv724+JFZ&MoSwdWuVES;7f(;@?z@=Aj2pIda<}U7jO(4rV;<1G z!|%8}#lduzTUz6b{_|Ov=Z>5H+TCwzs$EDrN4Llji;YO!y@MSd%Ch55LNIuwaciumB`!PChJb0oNc*fNS zyvVV$ug$pfMX$Qi&ku9>EH&tQZpMwbIC$UEOK0*c3oAP7wqwln(kK%=2siJKt{)C) z{b$CdPjB%9o2Ndr4xY@|Gp=XxobhIy`t{rBxO~mNgRM`#(Qg<$aOU$j-a~xEp_67l zIDEe1ZQa(^EnK`Onmd2xORf02nt{6YH99U&`QVT<`ggpTapT3#d$^`Q#*E90_>qj! zE`!~Eb3p4q9hax^K@L*CWa24TcEjEAwH`Z0rx!PDdrfj2Kje6(x#OO{o8_Iples%C zpLt5}Oke%_ZH%~{kGFDc?X;$5{C3Z5bX=LGRnD*-JDKRXJk=NS8qI3uGiUS8G|#y7 zI>L3$AAa2RM#rTK59eEXus*(GhP~r$JvQuP+^ku?^auV<-*rChFlx#lW5l(7Ivu=y zGRKUI&jX(9^l;ri^f!9rvAoKbpIy^G%WnPpmYe-+bw9^6Iv=c?zchZ+U+dB%pW$Ki zI<;HfD>@%=Q@2%CJ@OheE_V1~&)z*RXkE;>@pqNA%FW(@FT>yH3;O_8n7iLiIv>tl z9{c(3Z??MSHSV}Od*cgQ7c(wD&&N(CW?bH?odZ98jjvZ`IqA5V`F@7WW2{w=XIy!} z^S#V<-XAM_a&$zVGnarhzEA{KRv{|>~$vb1l-R-fG@ES8NAFMg~O4e5X zjeW-T-6mf0VkHN4+c7$BG^!h(T+pkA)^WFAtZ@As-R?ed$EA(-Z*pZ#@A6_d8#BG? z=D^?awr+3tNjyD~eF57wKlpN&jZSa4>Yfa{Ud`K#8&7lo7T3DtgY!4~jSpCZtYHD-FP)B8*Y_^_I>x^$aqYRuSu)J)?D2fLtLKbs&B@yCJFep@Zte>=V|@wHD+9OO5T#QmA|pTJ8pRV zNao@<&3F1oT>my17RG9=>ekojxU^;8*~vk`9oPDw=)B{~!`a(!?2~HG-<`ZhXU-!Z zaL@ZMe;qPr9kL=1=I(mBoiwMP)$A?^p91M>oIuVc+QbtbTQj85dLCFnNNx`oGK0xTCxmdTDgr z=;Q0%nnx3L+_Rs!`Zu-He|Mwf@<5%QW^Kf!E&G|e(Mi7}n(y*9ea>u3D zJi8g5apg=V^4(^KJ>#Z!>5bLDSxU+<58>G(8S^_-2JJ)-041>A4$54@D8 z_^Wo`$nTkJf2Ud2c4{~B;mqalj<0q5F=kv$b9|G_TOQ-?^l#mMj85;F`^4iJ7aNn_ zZ`Casi@)~XPx~0_%ylipKL2DtJ9F`#{D0hL{^~JVvuE;o^>5?4b-3?DZ@4t8+qe7; zpTB(d^Xgtd^TD1Loh$FOy8SioxICbjPI_10^)qgm_J>{F9XB<5qP1kSdB}GLTw3L9 z$Ky_K-fRAT=Lh{eUfgk?es90y^20v-Exj^@$Jco5v(=vK^kO`D|I}m0=(zbVjjvv` zerx-I^*$bj#s$$-EZXKtk*S-xI65b-t+{2({rm{cU&3M6V9uw zy7e_W?zvvc2EN1Z(u46y9@RHv#?@QVi^X5huk7tJE*~7>VDtH^tv>oOX57?jXMfBD z9hXM>ctLaapH>;P&eyX*(*D)?_`*?CR4aPU-6>j>NU?N zXMX0*>UXp|t~DoTuJN7k_#OA0f#i$_=bow`+cD$nS^o0Ed8J-;+c7#WCY^NeGJ$7Y zb>nCD1wXH5xZ~2e!>QZf#*C{Uu%CF`ar6JfOOCC7$LASW{ppQnY`wSgJz~U-Rvfud zx1FA-+lP5LbM*rLPIm9O)^FWB-tpj$o4sAVzRAt^k@&DuFItUR-J+B3-3)i!d{5vp ze|PKGap?=&-jjEodAyVE_PCCltUcLtYTBW{d51me2TWt67dKhpEkC|dH(Z|bGVf*m z_G!*=X;dpZ@nzT=R@t2o=PZO9-<{28j&5{ZdpeEy4lH$C?Y%2Ee$vR(8!JO0emGke5M2CRMeJrQ=t#p8i~<}v+xWc9n>m5YJr`Rq0M z&a+dy6>fSsKCkq~h#RfR82y-^^c$Av<8Rpc4*WC^IQ*n9OxL_#S%2MjjFAtx4qTay zub<>M8h2`6Q~wxOy)tUe4%x-0clG~YQ@0+Y z_xEeva*U2^-L&$V#&78jH~q&0`C8r6W_q!{&CUCGr^iahYmD@!pRj3ulH-nxXD>I0 z>(zYKt;guN=?m;9{oHY_-66~L+B0ch^}FNdZ+YirST-Mv=ocQd?eunG0C%k&> z7@c11c)6Xu{7xPKEqWS?B^+vv=>(`(JvpX}yZM)Gc&`Le=|_8qTRxXHp!*6z63 zA9gcw$9?ct(@T|15Y)N^!m=A zZmAWvpPl*qv~IZ3sb=bh_YE$dOxSzQIlzC~7s*)GkG0zyBY)3aTF-h{v(|Av|0G}a z*fBaT4;-o6?)Rw~7c=i-XEnV0`>Jj^M#q(1KIk`f+pS&4#pFZ!j?UGL)vd#QQXX*m z5VmuAZl#_hz1eH9yf5S|`sLWXcX=CelUZxW#{7oAxY=`L;L5$zT8|9<#pUhO>~>rl ztv}hNJ=gL2?j17nH(d4iURJyKe8;O>k1^w7(2DO`9ap@q+lKx|uk&sWFpZhs_`2%p z?})$q?+RDWR(%b7#>Mj;CEk9zKg3@-kS}#}{f)Ziy9uohIm7q4ez6umJCF9d&Ifx7 z7Us%6Q@6d1j>`)k#9QZ1+Z*uL4|iOeG;{{%F%V+Gn?A9$)Zt|8I%I&9I?*)44&LL~&FyCE2k8;Lex|}hwq8Tr~;CG(P zxazgSsM~&wj;mg@o;@~?&&)+XuR1?wTpS*UgSlF}88@DWojuLxPy5*V>5XPO@w+B#iz^e>j=9ncb=zyq^zs0k54-Fcmu5^}IIrZOZaYTD z&3hATXJ7b*9qwnz7H`uFv%Mg!^LeBfHyY{wR=?rWDZ^LIe7@_ox~262Hy+1})jfU2 zwO6SbUsvC0cp06)wBgeFO}+XdKGT~y^v8-HonAHL0j=lS#hbhOb=+vBEnJ@a+3y_K zU4M66x;&G2XE>|xyAc;tMzOG0*Iu{2M#trAdd_*a<`?XAG2T0&H{e4uIz0y?yjdb^=sU5v3GO^+;P=N4zhkf&%QT>mvhJU?`Cv<^SzmU zSGLk~__X`XeC|lp3xBfXx{Yo5Wy;WMst$Eshw#*7cKp z>#w@yZFF2r`r@^pvtQh4H9^OXx6$n9Pjkix983qz->6%-e85fJayI9kCnF!C>XJ zWT77W;jfy|B?EF#Z*sNzH$Xkoa`CQu^VyvTGj4kH>;taDnykURUFNR-#)rHk;`M6( z)>OynxU^*5e5LPB;{p8jpBXoK!{>*3+Jjd1p&8fOF);0uJYW58*>R&ad*&{C#LfFh z4SA3n@|?%~xXW_KwJ&-;^~2-C9j|UVM#sg#rPqPmWwn0wW6ZdC;qU66an;VAizyp4 z`)oyb$CWEQIdfjESKWGyj+-pVhKwc4@%gTP9apw6o%yUEb~MkpG^HO_JgwVbW5%UZ z)-Y-1^D2MF%^tJrX-)eY9oPEP7f&)ViR$lizpzj%)o6 z+A*`Ixz2hpce%7;IOJhBZ=K%!4HTx{@Q!v~`R>>k>&BIxq4#?m?Gp_nS^`hgZkDt_x{5vi`=$5q| zpI5jtp=L7eJj-qU`WQ1Vt?3JSPVa?1t7pfxb}ZU(?)s~4c^e%Uk6v|4PdnpxdszMI zHSW0BbjRD(f0w%BX5Zxn{>;<*^=sU5`N0c3b42%DCT3ju!sNw{w{?5l7y9>{E!-V` zrxzDT{dc{sM_RpkN38S!&UJKp=}f+M)*t?i8_mw~+I%Z}+>DE{^KYHe>fCXYFy)czyv5zkMW>gxr#)Pc4)uBm*(Z0_jddO)z4@-l1N#C$R`;>6@3_g8Id*zv z#N|Qub-7K}cKTz)4HHN0{G7Q~zDw5a!@7mb({JvRxOD1;Z|lWWCv`KA%&ojH>$YQz z{LP-8e&aJgKFve+bi5osb6AJZ?&cWj&9nN$^^?AvalgGkM6Yx7n&WQmb<1mXdU+rR z@{oC=Km9k$N5{?gK7Z3k=T6o-E=F=e=Qna@?e-i0ALwMt&(=L#d;R(~MtX6reRt1& zLF;0~_0FWvk(#;Yhu>x2pK0&7*3VP>;wT`RL@REznf$y_B{*|l8JQrr_ch)UuPq^}h7nXDOv(;W1>BWui-F=}R zJ;oh3y6ufL+dF;cJo_#&<0c1u*nKD8akHoKV~0QE>J4j^XWE^yXZub4`WPcFMsnu7 zs};E8%4l+&InswKUUyv2hL@hxYd%}?V})x!(+_wn-qvlc5A^B{8gW0}ck&K)##8@q z)lF}y8!zdUVZCM!KR?~i&Ro22yf3U@jcCK88QbU1t3HqN5D&0v&)&1+bH_dRg?`2k z-+XtyG2&XUta=tB^)uJqd!lYR#)wN7R(cM5r-y+%u9~H8@!?Z{>BYp84cd4880ocs z+2^xs^El7mJN4_gG2+UqHOFI_UCF>1+;Ow+)Xp4^l|2%F#7(`z_VYLLfX4^C^S-va z=hS0Ez0s+5bnfg6n8wJ5aQPYCuIXLvxjXJ^-`13GW5%_92cKn_*3}HpxctD|%|boW z`b%Tp@4Mc1T;J6(@pA0cY{ZRD90#t?cD=az8*bL_jLGZO?<95W4}Zgz1H5l@?KgUu z&+>)udNt1@AM}``?ms~m~UmD7;)*$y1&`S{)SIqXu`S^x~#g zGW@Nah06o{-CnClhJBpBiLZMw&-L_*{*e!79v1!SY1;k%$-d(`&!02l8e8VA8-Z{^hfIXy5t%*Kw_XXaB0(-$uvfW&Ss!=;gy|=0;pzroZB|&s_6^H+TGw>vUGxoNgszRzq>l?kz;iJCS%Sy z%%PsM930U4&y1_jQa}5}&U>Td@)gs&l167-x&A~iuIH_NXZ>~iF*?2HeIZ`puV@AC zxar;W$l2SK_fp5@Y1q!V>213CG|Tdc8(y?N`F?W8#r#&k(VINziLS78Sc1vdRC_ReXiY#A2Y7? zV|=S_$-n)K?r-f6xLNydz3S0pjC_dB)Qr#e$(_FIxN_?_=gmB;?2apAyhzq~!)NoX z^xTM>Uc+?A!?(DpowZ|UZ|D6=kBs!nY}Svv+Y|N3kM{+gYD1s8@fh3B{Gc~`5udwp z)f*r4e#2C+l|A7+$B2tXv-PJxcG(>_+5I;6{Qr|P$8PO9z5GBYUi<9LIK?|iqvjkq-O zlqMRTefFsjSvy@%{=bOz*l=H$vFvBq4)ycilas7vwuX@pxY7L7iy1eyI`0pw9(Po%_2m069!9j%$CH)bt=nGXs#k7hlh$Naoz%-*t9#Fk8!w*J?v9)NfquGI zHFwikm(Q&i(RX@YadBn9=9?%<4lDF0OQO8v;`#?OVo5w3zsN04)3yVg%2-YS!`WZ^#459Ho%%r6xi?nUk7;y1(C46;ujWugnpgIS5!Z7V zG9|~jwC;SDnsKq{bgbsKZaqfFnH4vE5mA!?Hui^d(Vt(?Yq9#Bgg2t)*r3T z-^j1JCBv?>N3QlnJ<{gkNv-fQMn0&KwP*j44Y}LN+llKysg`Z z{?eH&?CQicMtZ}=-tC72dc(y(d$F@ebb6Bsn!`5FDtpAu`;xb~d2ir68 zkgxoWEW7x6h2~GV)=n>tSTy6V@;klp^y!|~acQJ)b-%4!-^Pf`KY4KAKY4eKxI9c2 zzUf1_wBoO3wr+imPA|5D7xo-;?ELN0aqVYVjwki&xYqv7zlZdj_0t)jdFj@KH??|d_3=$_ImoC{ClLKo>3Y@Uj;r=(X7+_u z_KeHdyq8?ln)ExvxpL_b#y5WBF`vX06 z@<30-hh3jLA8^kZSj|B_c8rc|Z_gf)p4ee`Tz;fp=@CDNeV4rvS8wq+`wSNCcdWYQ zH9D^RCTBc}FRSd1t7h3FU7N#PtJ-H=ES}=6>a1IjG2*69bn;ksv7J}Glhkd)_XV!~ z0Egye+4U!V93OEU_}H%bYu-CwMn2$%AzzrQ^{QKsG2^B$u=&Dk*`}W#Gno-)b7f2p_<%20GQ3*D zPA{F9-_%R1nq_~CW^*LVce+P<<3Tj9?g=&RYjj*19jp3Yz+V?LuDw8S@WWoDFFx6S zv)|x7$wECg%r7o&-|k~v>kfB!@2y9NdaXS@an8n04M%mujmB^K8*ci|&vd7c@7`^7 zYwLq&)h=s~{uSnoo1WO+Z!nF{-}IY4a4j451UYfO>yM7B*D#XhygPi(6MM&O@9DVS zoAGw`eBCx29aqhw@w{ic=85mE?xB11Jz~c7951kTHFsQpWAT=r-Hg>EZ64^916t`$ zPuzWfs#}lI=?(YW`Ax>09lQLQUi*S0e((Zc9^y;a*Bv+gAj8<6cV6wg85c**9O-L% zyUf9O<@b6}{@q*9k1AIE~MyJ>FyR7qEBd&E? zbB8fs}C>C!54G#{LYV#t7hjs$U@lhXEzUcIMHd2-CiE)#n1!sfj)EXyyHgP z{M#(f6J|X&tXuv!7aDPQ`WMp}=~Zjl;zK-EvmN~%SFPy8a}M8m<$F-al?ObG#)wOoBR#iL zrx)v_4`}~cfPM|iT z*)x}Je`oA6>yfvIalad_;mWmNc{Y0K40mPyb^9@{-2D5j9P`*5be+9$eK*I5%SSxx_I$KI{rFV)vCq=`0GC-E}afusuw;s-7DVKZ9no5UwIt8 z=D_vYr`%*>mse9CW8?#!*027qt=~GF@7}|8%P~e=S%_X9SpQDOfJfZyGxXt_V^{Zx z8?Ex-T844$&F13KZoly{+)v(}Bfa@9$qxs8{BSnsoz}YLhrjVOzNRa&got z`?}286L`M5H`J{|E+4Ze(5_zUxVlHytu0sWWHmnUI(b;(k9?pPFU;^(d!im4>XrXw z_?$OVI*>;J@GJsPgF`kiwe|5x;%^(L=)^d-AHyn5^ySMD`m z{0RSP9x!RnzZKA_hH~eO%OB^vck@{V_d>o=w(LKiYlfyyEp$Z@6lf?^SZPn~M>bMw)pYPjP*=Iy$b}SugJD+Rt-# zTr7H>`G9ZFU#;DU>sdXG$?qq+N8G$Od76EK=YBTViq9Pv#}Umkm}}>|tc|!B4t~g6 z{Kca6sorQ)r_{AgV$nsPnw>pf@wFZ~#)z9Whv}?O_`J#< zaWTS-*LU@=N8ZMjiy|a%6KH~C$&gka3xpwMz<@!En&9pjb&G!nQn`6b}D_36e z&UpxbCwEtFvS7`4*)Q+Dm)5Osd_R9fFK^=o{_1~WB zGJ$EXc(T%Kb^F`*)Nqrn_^{f$cp1@~^=@`7d=!;!1Z zhi#73c9z4AOE0a-+TDIzx4w;4?p^)gp!LI*o8GYIWMY?Tjw@FW@F1Cs$2)t&mFwM3 zXKE*3w8m>|+1(%X1#a?7XWm8T*{$E%2l~!DIvslClRVJM2dwxitFBl3p>93KSucjQ z^8=fo=Gyh39(l_{xY3Lqzg_dmT=I99y^-E47ss4f(Z0(~#&FI#xZ~9AZ{w;rHA{bF z{kX23S8{yi^1$I)964}p{c5_Jw-Gl?e7yWEyUMIbe&pet;dnx)4D;Y=-Dt&5rkv?c z-tf(L=k?VG_0qTD#OqJ}#g#D!P53+OuiK9?(#wmy_rFy)Jawlte`ASzTpH~G4m?`TW!*d3s@q?K z-ZNJoXpS#o-__{_{B?2VrdH0_crsuuGNy*JeATayapuN@aIkS4D?N4qfBomo^?de0 zbI{8ZUPPzs6;IAw_j@ku=QW*}eDM1fX5Dt=H+4&9F?g6>OXhs`L@%x*{<=<{Kk)%4 zdt+)ypU<6n^3(^<@_{#4yE^NUW1Q<|zoe57_}H|nlc5-~?${4OZ@PQ}lwd-v?^44$RS}Se1shfH5 z`MuK5SG~#BZmw#&w{hmu!Ux*;={t&RnyrC;=as*Wu3XuQW^*`rlD=E&d=bj~(M$ zx9E(&_;m8fd8LQzwjch+SKO!Gc3hernM2*=?P(sYlg{)5pB=k3t4H2^z~!gf+1KdH zUbO4Uxo*injq>^=xAoXDMqJt)v}4=XKJho%rS{ULg3@Es?f(!X23kzU;F6XtTP)~jwk z#)!*<^TAVIVz2JQ9hXL$c@e#z`b%r}9p0|wwQhSGBfZ&o@SkWkLC3|U_31mKPeIinGG zH)|c&zL32Ee^>K}izll*pfld?>}?$v1E0sv`8SVMKW1Fs?r5yrU!&vFDN|40LEtm4 zdi_ms`KrgT`73{FxY8S)URq^>AC6qF_D9F1Ek4M=&fZYBxBh0|;e~7LPxoJ(WLG~w zdFPLOu$QGTqMKg6sM*uH<=vzo)-qO3eZ;rU0&-$H{t({Da zxN5~iT3zq-L*0Iij*A%&a;@j0{VsDOt{TyaXYD*)^|fw2#+4gw`EJH*`M~$CchG&8 zwGkJ~I(M^GkG#f+t4?&&BxAC-Te}fAdENbvQ`5gj$JJvzrTd8=GcKRM$<<@m9;V+M ztNYJL@0p7)%lg5!-PHc}Co1H)q+{$!^{L9Pj7=E%LZ`Qv0_tupgp0k4&>bUc+sN3K2 zODB$+Ia~kJOkDNi(V5!u0edIAb^A~+50fkWZ*XOaMw;>X`i(r`>QR1RCKJ25ul{Bq zk7n$=Gj?>3xO$9EKDfr+&Bm3Br%(Biy^TeEufhYxa_y1AyAMjn5n7fapn^atj*^d<)~?3&h3{q;N_FlqgU zUUiDM&NAjR-XtS+xirpsz~Ci+WebmPKf7M}-cYw4W5iW=T50yT9Dnn@a<%8q-1EQ3 zsTb|mK=W=dU%7n5!}b2$<<%p{2p20ka7}ysrkA(oSnZ87*K_KVOyEW{-m15C>#L7x zq$_pDG0$#4)Fa0@>y18hIP$+oV((;jsAJVVZ9Uq*Z)-PPz(qpd86aFXrI+jDuXg0_0o=}gxM)b;9>gKG!c#UtK z-S=|GjSuk#JAJ&;cO&j6`KsxUG2-$$dxbfkyo;_}`wV}1%VR#ccC4(wZac=6D-T#S z%bER>=1=P;7kUBH5l^h)sa`erEM9cG<}beAt&Xz~xO~Lot+Tv+(r?y(=Eh&^$4++5 zv7&pVSKWAQ9-5zceC1-&nBKrmmOrVRHG3W(^PG*H`d#&AUwBfhlJOd6uJ!U%#%Se- zgJ$#j+NRWr|H^B8Z}+j``OzsVD>EJS;L=IhE_XTALM zd@?2r{KfLQv$d>b;>`6d7EkdozL7KBtdZs?I_t4xT=il*`1~|uz#}dnlCP(luiVs( z-sFup*?-LQq;AoN8$W1Qzj(jGKI_#JdO^=cud~m0^LFL(A^ng{?C7k=k8$N@{n*(n znB>T+<1`pnKLf_?w)qmi<|dD{_e)jUa_P7%#E)2 z>daT3m@|F7^7qV{o9}2C@dOjk@6^xSyZz=_&+x$W(fw&2a2;XlS9 Date: Mon, 29 Apr 2024 16:42:41 +0200 Subject: [PATCH 06/94] hamming numba distance calculator implemented and tested --- src/scirpy/ir_dist/metrics.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 58dfed34e..7fe021f43 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -440,8 +440,9 @@ def _calc_dist_mat_block( if len(seqs) == 0 or len(seqs2) == 0: return csr_matrix((len(seqs), len(seqs2))) - seqs_mat1, seqs_L1 = _seqs2mat(seqs) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2) + max_seq_len = max(np.max([len(s) for s in seqs]), np.max([len(s) for s in seqs2])) + seqs_mat1, seqs_L1 = _seqs2mat(seqs, max_len = max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, max_len = max_seq_len) data_rows, indices_rows, row_element_counts = self._metric_mat( seqs_mat1=seqs_mat1, @@ -509,6 +510,8 @@ def _hamming_mat( start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + print("mat equal: ", np.array_equal(seqs_mat1, seqs_mat2)) + cutoff=self.cutoff start_column *= is_symmetric @@ -528,6 +531,25 @@ def _nb_hamming_mat(): data_row = np.zeros(seqs_mat2.shape[0]) indices_row = np.zeros(seqs_mat2.shape[0]) + for row_index in range(seqs_mat1.shape[0]): + start_col_index = start_column + row_index * is_symmetric + row_end_index = 0 + seq = seqs_mat1[row_index] + comparison = seqs_mat2[start_col_index:] != seq + unequal_length = np.where(seqs_L1[row_index] != seqs_L2[start_col_index:])[0] + sum_of_rows = np.sum(comparison, axis=1)+1 + sum_of_rows[unequal_length] = cutoff + 2 + indices = np.where(sum_of_rows <= cutoff + 1)[0] + row_end_index = len(indices) + indices_row[0:row_end_index] = indices + start_col_index + values = sum_of_rows[indices] + data_row[0:row_end_index] = values + data_rows[row_index] = data_row[0:row_end_index].copy() + indices_rows[row_index] = indices_row[0:row_end_index].copy() + row_element_counts[row_index] = row_end_index + return data_rows, indices_rows, row_element_counts + + """ for row_index in range(seqs_mat1.shape[0]): row_end_index = 0 for col_index in range(start_column + row_index * is_symmetric, seqs_mat2.shape[0]): @@ -548,6 +570,7 @@ def _nb_hamming_mat(): indices_rows[row_index] = indices_row[0:row_end_index].copy() row_element_counts[row_index] = row_end_index return data_rows, indices_rows, row_element_counts + """ data_rows, indices_rows, row_element_counts = _nb_hamming_mat() From 0b15f8b3351cae5eede7acb55a8c95a0de081a9f Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 16:52:24 +0200 Subject: [PATCH 07/94] n_jobs parameter handling done in NumbaDistanceCalculator superclass --- src/scirpy/ir_dist/metrics.py | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 7fe021f43..440efa695 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -410,8 +410,9 @@ def _seqs2mat( class NumbaDistanceCalculator(abc.ABC): - def __init__(self): + def __init__(self, n_jobs: int = 1): super().__init__() + self.n_jobs = n_jobs @abc.abstractmethod def _metric_mat( @@ -496,7 +497,7 @@ def __init__( n_jobs: int = 1, cutoff: int = 2, ): - self.n_jobs = n_jobs + super().__init__(n_jobs=n_jobs) self.cutoff = cutoff def _hamming_mat( @@ -548,29 +549,6 @@ def _nb_hamming_mat(): indices_rows[row_index] = indices_row[0:row_end_index].copy() row_element_counts[row_index] = row_end_index return data_rows, indices_rows, row_element_counts - - """ - for row_index in range(seqs_mat1.shape[0]): - row_end_index = 0 - for col_index in range(start_column + row_index * is_symmetric, seqs_mat2.shape[0]): - q_L = seqs_L1[row_index] - s_L = seqs_L2[col_index] - distance = 1 - - if q_L == s_L: - for i in range(0, q_L): - distance += seqs_mat1[row_index, i] != seqs_mat2[col_index, i] - - if distance <= cutoff + 1: - data_row[row_end_index] = distance - indices_row[row_end_index] = col_index - row_end_index += 1 - - data_rows[row_index] = data_row[0:row_end_index].copy() - indices_rows[row_index] = indices_row[0:row_end_index].copy() - row_element_counts[row_index] = row_end_index - return data_rows, indices_rows, row_element_counts - """ data_rows, indices_rows, row_element_counts = _nb_hamming_mat() @@ -632,7 +610,7 @@ def __init__( self.ctrim = ctrim self.fixed_gappos = fixed_gappos self.cutoff = cutoff - self.n_jobs = n_jobs + super().__init__(n_jobs=n_jobs) def _tcrdist_mat( self, From 46bfc1406e0f8975348666ff70c113a88ba0163e Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 17:22:48 +0200 Subject: [PATCH 08/94] documentation adapted --- src/scirpy/ir_dist/metrics.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 440efa695..65706e558 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -350,7 +350,6 @@ def _make_numba_matrix(distance_matrix: dict, alphabet: str = "ARNDCQEGHILKMFPST ---------- distance_matrix: Keys are tuples like ('A', 'C') with values containing an integer. - alphabet: Returns ------- @@ -434,9 +433,9 @@ def _calc_dist_mat_block( is_symmetric: bool = False, start_column: int = 0, ) -> csr_matrix: - """Computes a block of the final TCRdist distance matrix and returns it as CSR matrix. - If the final result matrix that consists of all blocks together is symmetric, only the part of the block that would - contribute to the upper triangular matrix of the final result will be computed. + """Computes a block of the final distance matrix and returns it as CSR matrix. + If the final result matrix that consists of all blocks together is symmetric, only the part + of the block that would contribute to the upper triangular matrix of the final result will be computed. """ if len(seqs) == 0 or len(seqs2) == 0: return csr_matrix((len(seqs), len(seqs2))) @@ -461,8 +460,8 @@ def _calc_dist_mat_block( return sparse_distance_matrix def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = None) -> csr_matrix: - """Calculates the pairwise distances between two vectors of gene sequences based on the TCRdist distance metric - and returns a CSR distance matrix + """Calculates the pairwise distances between two vectors of gene sequences based on the distance metric + of the derived class and returns a CSR distance matrix """ if seqs2 is None: seqs2 = seqs @@ -491,6 +490,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No return full_distance_matrix + class HammingDistanceCalculator(NumbaDistanceCalculator): def __init__( self, @@ -511,8 +511,6 @@ def _hamming_mat( start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: - print("mat equal: ", np.array_equal(seqs_mat1, seqs_mat2)) - cutoff=self.cutoff start_column *= is_symmetric @@ -648,20 +646,6 @@ def _tcrdist_mat( start_column: Determines at which column the calculation should be started. This is only used if this function is used to compute a block of a bigger result matrix that is symmetric - distance_matrix: - A square distance matrix (NOT a similarity matrix). - Matrix must match the alphabet that was used to create - seqs_mat, where each AA is represented by an index into the alphabet. - dist_weight: - Weight applied to the mismatch distances before summing with the gap penalties - gap_penalty: - Distance penalty for the difference in the length of the two sequences - ntrim/ctrim: - Positions trimmed off the N-terminus (0) and C-terminus (L-1) ends of the peptide sequence. These symbols will be ignored - in the distance calculation. - fixed_gappos: - If True, insert gaps at a fixed position after the cysteine residue statring the CDR3 (typically position 6). - If False, find the "optimal" position for inserting the gaps to make up the difference in length Returns ------- @@ -675,7 +659,6 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ - distance_matrix=self.tcr_nb_distance_matrix cutoff=self.cutoff dist_weight=self.dist_weight gap_penalty=self.gap_penalty @@ -683,7 +666,7 @@ def _tcrdist_mat( ctrim=self.ctrim fixed_gappos=self.fixed_gappos - dist_mat_weighted = distance_matrix * dist_weight + dist_mat_weighted = self.tcr_nb_distance_matrix * dist_weight start_column *= is_symmetric @nb.jit(nopython=True, parallel=False, nogil=True) @@ -763,6 +746,7 @@ def _nb_tcrdist_mat(): _metric_mat = _tcrdist_mat + @_doc_params(params=_doc_params_parallel_distance_calculator) class AlignmentDistanceCalculator(ParallelDistanceCalculator): """\ From e339e14c350099679a4ec0c4efeff700d8fd4fa7 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 29 Apr 2024 17:29:22 +0200 Subject: [PATCH 09/94] removed unnecessary import --- src/scirpy/ir_dist/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 65706e558..805f54a3f 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -2,7 +2,7 @@ import itertools import warnings from collections.abc import Sequence -from typing import Optional, Union, Callable +from typing import Optional, Union import joblib import numba as nb From 7da451910f14c644163f46d0968bee7a4402bb57 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 15:32:34 +0000 Subject: [PATCH 10/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 114 +++++++++++------------ src/scirpy/tests/test_ir_dist_metrics.py | 3 +- 2 files changed, 58 insertions(+), 59 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 805f54a3f..b038767e0 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -10,7 +10,6 @@ import scipy.sparse import scipy.spatial from Levenshtein import distance as levenshtein_dist -from Levenshtein import hamming as hamming_dist from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix @@ -362,50 +361,51 @@ def _make_numba_matrix(distance_matrix: dict, alphabet: str = "ARNDCQEGHILKMFPST dm[alphabet.index(aa2), alphabet.index(aa1)] = d return dm + def _seqs2mat( - seqs: Sequence[str], alphabet: str = "ARNDCQEGHILKMFPSTWYVBZX", max_len: Union[None, int] = None - ) -> tuple[np.ndarray, np.ndarray]: - """Convert a collection of gene sequences into a - numpy matrix of integers for fast comparison. + seqs: Sequence[str], alphabet: str = "ARNDCQEGHILKMFPSTWYVBZX", max_len: Union[None, int] = None +) -> tuple[np.ndarray, np.ndarray]: + """Convert a collection of gene sequences into a + numpy matrix of integers for fast comparison. - Parameters - ---------- - seqs: - Sequence of strings + Parameters + ---------- + seqs: + Sequence of strings - Returns - ------- - mat: - matrix with gene sequences encoded as integers - L: - vector with length values of the gene sequences in the matrix - - Examples - -------- - >>> seqs2mat(["CAT", "HAT"]) - array([[ 4, 0, 16], - [ 8, 0, 16]], dtype=int8) - - Notes - ----- - Requires all seqs to have the same length, therefore shorter sequences - are filled up with -1 entries at the end. - """ - if max_len is None: - max_len = np.max([len(s) for s in seqs]) - mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) - L = np.zeros(len(seqs), dtype=np.int8) - for si, s in enumerate(seqs): - L[si] = len(s) - for aai in range(max_len): - if aai >= len(s): - break - try: - mat[si, aai] = alphabet.index(s[aai]) - except ValueError: - # Unknown symbols given value for last column/row of matrix - mat[si, aai] = len(alphabet) - return mat, L + Returns + ------- + mat: + matrix with gene sequences encoded as integers + L: + vector with length values of the gene sequences in the matrix + + Examples + -------- + >>> seqs2mat(["CAT", "HAT"]) + array([[ 4, 0, 16], + [ 8, 0, 16]], dtype=int8) + + Notes + ----- + Requires all seqs to have the same length, therefore shorter sequences + are filled up with -1 entries at the end. + """ + if max_len is None: + max_len = np.max([len(s) for s in seqs]) + mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) + L = np.zeros(len(seqs), dtype=np.int8) + for si, s in enumerate(seqs): + L[si] = len(s) + for aai in range(max_len): + if aai >= len(s): + break + try: + mat[si, aai] = alphabet.index(s[aai]) + except ValueError: + # Unknown symbols given value for last column/row of matrix + mat[si, aai] = len(alphabet) + return mat, L class NumbaDistanceCalculator(abc.ABC): @@ -425,7 +425,7 @@ def _metric_mat( start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: pass - + def _calc_dist_mat_block( self, seqs: Sequence[str], @@ -434,15 +434,15 @@ def _calc_dist_mat_block( start_column: int = 0, ) -> csr_matrix: """Computes a block of the final distance matrix and returns it as CSR matrix. - If the final result matrix that consists of all blocks together is symmetric, only the part + If the final result matrix that consists of all blocks together is symmetric, only the part of the block that would contribute to the upper triangular matrix of the final result will be computed. """ if len(seqs) == 0 or len(seqs2) == 0: return csr_matrix((len(seqs), len(seqs2))) max_seq_len = max(np.max([len(s) for s in seqs]), np.max([len(s) for s in seqs2])) - seqs_mat1, seqs_L1 = _seqs2mat(seqs, max_len = max_seq_len) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2, max_len = max_seq_len) + seqs_mat1, seqs_L1 = _seqs2mat(seqs, max_len=max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, max_len=max_seq_len) data_rows, indices_rows, row_element_counts = self._metric_mat( seqs_mat1=seqs_mat1, @@ -510,8 +510,7 @@ def _hamming_mat( is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: - - cutoff=self.cutoff + cutoff = self.cutoff start_column *= is_symmetric @nb.jit(nopython=True, parallel=False, nogil=True) @@ -536,7 +535,7 @@ def _nb_hamming_mat(): seq = seqs_mat1[row_index] comparison = seqs_mat2[start_col_index:] != seq unequal_length = np.where(seqs_L1[row_index] != seqs_L2[start_col_index:])[0] - sum_of_rows = np.sum(comparison, axis=1)+1 + sum_of_rows = np.sum(comparison, axis=1) + 1 sum_of_rows[unequal_length] = cutoff + 2 indices = np.where(sum_of_rows <= cutoff + 1)[0] row_end_index = len(indices) @@ -551,9 +550,8 @@ def _nb_hamming_mat(): data_rows, indices_rows, row_element_counts = _nb_hamming_mat() return data_rows, indices_rows, row_element_counts - - _metric_mat = _hamming_mat + _metric_mat = _hamming_mat class TCRdistDistanceCalculator(NumbaDistanceCalculator): @@ -659,12 +657,12 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ - cutoff=self.cutoff - dist_weight=self.dist_weight - gap_penalty=self.gap_penalty - ntrim=self.ntrim - ctrim=self.ctrim - fixed_gappos=self.fixed_gappos + cutoff = self.cutoff + dist_weight = self.dist_weight + gap_penalty = self.gap_penalty + ntrim = self.ntrim + ctrim = self.ctrim + fixed_gappos = self.fixed_gappos dist_mat_weighted = self.tcr_nb_distance_matrix * dist_weight start_column *= is_symmetric @@ -743,7 +741,7 @@ def _nb_tcrdist_mat(): data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() return data_rows, indices_rows, row_element_counts - + _metric_mat = _tcrdist_mat diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 56450930d..a1e512ff5 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -652,6 +652,7 @@ def test_tcrdist_reference(): assert np.array_equal(res.indices, reference_result.indices) assert np.array_equal(res.indptr, reference_result.indptr) + def test_hamming_reference(): # test hamming distance against reference implementation from . import TESTDATA @@ -659,7 +660,7 @@ def test_hamming_reference(): seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_csr_result.npz") - hamming_calculator = HammingDistanceCalculator(2,2) + hamming_calculator = HammingDistanceCalculator(2, 2) res = hamming_calculator.calc_dist_mat(seqs, seqs) assert np.array_equal(res.data, reference_result.data) From 82b02591d23d61a433c3dc91d1667c3c630cc059 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 2 May 2024 15:33:09 +0200 Subject: [PATCH 11/94] hamming distance with numba parallelization implemented --- src/scirpy/ir_dist/metrics.py | 64 ++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 805f54a3f..a320b6e52 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -471,7 +471,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No is_symmetric = np.array_equal(seqs, seqs2) n_blocks = self.n_jobs * 2 - if self.n_jobs > 1: + if False:#self.n_jobs > 1: --- only for intermediate version set to False split_seqs = np.array_split(seqs, n_blocks) start_columns = np.cumsum([0] + [len(seq) for seq in split_seqs[:-1]]) arguments = [(split_seqs[x], seqs2, is_symmetric, start_columns[x]) for x in range(n_blocks)] @@ -514,43 +514,61 @@ def _hamming_mat( cutoff=self.cutoff start_column *= is_symmetric - @nb.jit(nopython=True, parallel=False, nogil=True) + nb.set_num_threads(self.n_jobs) + num_threads = nb.get_num_threads() + print("numba threads: ", num_threads) + + @nb.jit(nopython=True, parallel=True, nogil=True) def _nb_hamming_mat(): assert seqs_mat1.shape[0] == seqs_L1.shape[0] assert seqs_mat2.shape[0] == seqs_L2.shape[0] + num_rows = seqs_mat1.shape[0] + num_cols = seqs_mat2.shape[0] + data_rows = nb.typed.List() indices_rows = nb.typed.List() - row_element_counts = np.zeros(seqs_mat1.shape[0]) + row_element_counts = np.zeros(num_rows) empty_row = np.zeros(0) - for _ in range(0, seqs_mat1.shape[0]): - data_rows.append(empty_row) - indices_rows.append(empty_row) + for _ in range(0, num_rows): + data_rows.append([empty_row]) + indices_rows.append([empty_row]) + + for row_index in nb.prange(num_rows): + data_row = np.empty(num_cols) + indices_row = np.empty(num_cols) - data_row = np.zeros(seqs_mat2.shape[0]) - indices_row = np.zeros(seqs_mat2.shape[0]) - for row_index in range(seqs_mat1.shape[0]): - start_col_index = start_column + row_index * is_symmetric row_end_index = 0 - seq = seqs_mat1[row_index] - comparison = seqs_mat2[start_col_index:] != seq - unequal_length = np.where(seqs_L1[row_index] != seqs_L2[start_col_index:])[0] - sum_of_rows = np.sum(comparison, axis=1)+1 - sum_of_rows[unequal_length] = cutoff + 2 - indices = np.where(sum_of_rows <= cutoff + 1)[0] - row_end_index = len(indices) - indices_row[0:row_end_index] = indices + start_col_index - values = sum_of_rows[indices] - data_row[0:row_end_index] = values - data_rows[row_index] = data_row[0:row_end_index].copy() - indices_rows[row_index] = indices_row[0:row_end_index].copy() + seq1_len = seqs_L1[row_index] + + for col_index in range(start_column + row_index * is_symmetric, num_cols): + distance = 1 + seq2_len = seqs_L2[col_index] + if seq1_len == seq2_len: + for i in range(0, seq1_len): + distance += seqs_mat1[row_index, i] != seqs_mat2[col_index, i] + + if distance <= cutoff + 1: + data_row[row_end_index] = distance + indices_row[row_end_index] = col_index + row_end_index += 1 + + data_rows[row_index][0] = data_row[0:row_end_index].copy() + indices_rows[row_index][0] = indices_row[0:row_end_index].copy() row_element_counts[row_index] = row_end_index + return data_rows, indices_rows, row_element_counts data_rows, indices_rows, row_element_counts = _nb_hamming_mat() + data_rows_flat = [] + indices_rows_flat = [] + + for i in range(len(data_rows)): + data_rows_flat.append(data_rows[i][0]) + indices_rows_flat.append(indices_rows[i][0]) - return data_rows, indices_rows, row_element_counts + return data_rows_flat, indices_rows_flat, row_element_counts _metric_mat = _hamming_mat From 249e626a31b96f516f1d087ee410cb811a74d468 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 13:40:44 +0000 Subject: [PATCH 12/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 9beeceb4d..5d33ff20b 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -471,7 +471,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No is_symmetric = np.array_equal(seqs, seqs2) n_blocks = self.n_jobs * 2 - if False:#self.n_jobs > 1: --- only for intermediate version set to False + if False: # self.n_jobs > 1: --- only for intermediate version set to False split_seqs = np.array_split(seqs, n_blocks) start_columns = np.cumsum([0] + [len(seq) for seq in split_seqs[:-1]]) arguments = [(split_seqs[x], seqs2, is_symmetric, start_columns[x]) for x in range(n_blocks)] @@ -524,7 +524,7 @@ def _nb_hamming_mat(): num_rows = seqs_mat1.shape[0] num_cols = seqs_mat2.shape[0] - + data_rows = nb.typed.List() indices_rows = nb.typed.List() row_element_counts = np.zeros(num_rows) @@ -533,7 +533,7 @@ def _nb_hamming_mat(): for _ in range(0, num_rows): data_rows.append([empty_row]) indices_rows.append([empty_row]) - + for row_index in nb.prange(num_rows): data_row = np.empty(num_cols) indices_row = np.empty(num_cols) @@ -562,13 +562,13 @@ def _nb_hamming_mat(): data_rows, indices_rows, row_element_counts = _nb_hamming_mat() data_rows_flat = [] indices_rows_flat = [] - + for i in range(len(data_rows)): data_rows_flat.append(data_rows[i][0]) indices_rows_flat.append(indices_rows[i][0]) return data_rows_flat, indices_rows_flat, row_element_counts - + _metric_mat = _hamming_mat From 2fccc6a4013dae011f8fc8daad7f4132da782d1a Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 2 May 2024 15:58:44 +0200 Subject: [PATCH 13/94] imports fixed --- src/scirpy/ir_dist/metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 9beeceb4d..dd9af90e5 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -10,6 +10,7 @@ import scipy.sparse import scipy.spatial from Levenshtein import distance as levenshtein_dist +from Levenshtein import hamming as hamming_dist from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix From a68ab538343c29f44150bf2ac85349ceb37dbf73 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 14:18:35 +0000 Subject: [PATCH 14/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index b22eef64f..5d33ff20b 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -10,7 +10,6 @@ import scipy.sparse import scipy.spatial from Levenshtein import distance as levenshtein_dist -from Levenshtein import hamming as hamming_dist from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix From d68a10b2d031846de3035737684ea25fc38f2d4e Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 6 May 2024 10:28:03 +0200 Subject: [PATCH 15/94] implemented parallelization with n_jobs and n_blocks for hamming and tcrdist distance metrics --- src/scirpy/ir_dist/metrics.py | 109 +++++++++++++++++------ src/scirpy/tests/test_ir_dist_metrics.py | 3 +- 2 files changed, 83 insertions(+), 29 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index b22eef64f..45fa184ae 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -410,9 +410,10 @@ def _seqs2mat( class NumbaDistanceCalculator(abc.ABC): - def __init__(self, n_jobs: int = 1): + def __init__(self, n_jobs: int = 1, n_blocks: int = 1): super().__init__() self.n_jobs = n_jobs + self.n_blocks = n_blocks @abc.abstractmethod def _metric_mat( @@ -425,6 +426,42 @@ def _metric_mat( is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + """ + This function should be implemented by the derived class in a way sucht that it computes the pairwise distances + for sequences in seqs_mat1 and seqs_mat2 based on a certain distance metric. The result should be a distance matrix + that is returned in the form of the data, indices and intptr arrays of a (scipy) compressed sparse row matrix. + + If this function is used to compute a block of a bigger result matrix, is_symmetric and start_column + can be used to only compute the part of the block that would be part of the upper triangular matrix of the + result matrix. + + Parameters + ---------- + seqs_mat1/2: + Matrix containing sequences created by seqs2mat with padding to accomodate + sequences of different lengths (-1 padding) + seqs_L1/2: + A vector containing the length of each sequence in the respective seqs_mat matrix, + without the padding in seqs_mat + is_symmetric: + Determines whether the final result matrix is symmetric, assuming that this function is + only used to compute a block of a bigger result matrix + start_column: + Determines at which column the calculation should be started. This is only used if this function is + used to compute a block of a bigger result matrix that is symmetric + + Returns + ------- + data_rows: + List with arrays containing the non-zero data values of the result matrix per row, + needed to create the final scipy CSR result matrix later + indices_rows: + List with arrays containing the non-zero entry column indeces of the result matrix per row, + needed to create the final scipy CSR result matrix later + row_element_counts: + Array with integers that indicate the amount of non-zero values of the result matrix per row, + needed to create the final scipy CSR result matrix later + """ pass def _calc_dist_mat_block( @@ -470,15 +507,14 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No seqs = np.array(seqs) seqs2 = np.array(seqs2) is_symmetric = np.array_equal(seqs, seqs2) - n_blocks = self.n_jobs * 2 - - if False: # self.n_jobs > 1: --- only for intermediate version set to False - split_seqs = np.array_split(seqs, n_blocks) + + if self.n_blocks > 1: + split_seqs = np.array_split(seqs, self.n_blocks) start_columns = np.cumsum([0] + [len(seq) for seq in split_seqs[:-1]]) - arguments = [(split_seqs[x], seqs2, is_symmetric, start_columns[x]) for x in range(n_blocks)] + arguments = [(split_seqs[x], seqs2, is_symmetric, start_columns[x]) for x in range(self.n_blocks)] delayed_jobs = [joblib.delayed(self._calc_dist_mat_block)(*args) for args in arguments] - results = list(_parallelize_with_joblib(delayed_jobs, total=len(arguments), n_jobs=self.n_jobs)) + results = joblib.Parallel(return_as="list")(delayed_jobs) distance_matrix_csr = scipy.sparse.vstack(results) else: distance_matrix_csr = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) @@ -496,9 +532,10 @@ class HammingDistanceCalculator(NumbaDistanceCalculator): def __init__( self, n_jobs: int = 1, + n_blocks: int = 1, cutoff: int = 2, ): - super().__init__(n_jobs=n_jobs) + super().__init__(n_jobs=n_jobs, n_blocks=n_blocks) self.cutoff = cutoff def _hamming_mat( @@ -618,6 +655,7 @@ def __init__( ctrim: int = 2, fixed_gappos: bool = True, n_jobs: int = 1, + n_blocks: int = 1, ): self.dist_weight = dist_weight self.gap_penalty = gap_penalty @@ -625,7 +663,7 @@ def __init__( self.ctrim = ctrim self.fixed_gappos = fixed_gappos self.cutoff = cutoff - super().__init__(n_jobs=n_jobs) + super().__init__(n_jobs=n_jobs, n_blocks=n_blocks) def _tcrdist_mat( self, @@ -686,36 +724,45 @@ def _tcrdist_mat( dist_mat_weighted = self.tcr_nb_distance_matrix * dist_weight start_column *= is_symmetric - @nb.jit(nopython=True, parallel=False, nogil=True) + nb.set_num_threads(self.n_jobs) + num_threads = nb.get_num_threads() + print("numba threads: ", num_threads) + + @nb.jit(nopython=True, parallel=True, nogil=True) def _nb_tcrdist_mat(): assert seqs_mat1.shape[0] == seqs_L1.shape[0] assert seqs_mat2.shape[0] == seqs_L2.shape[0] + num_rows = seqs_mat1.shape[0] + num_cols = seqs_mat2.shape[0] + data_rows = nb.typed.List() indices_rows = nb.typed.List() - row_element_counts = np.zeros(seqs_mat1.shape[0]) + row_element_counts = np.zeros(num_rows) empty_row = np.zeros(0) - for _ in range(0, seqs_mat1.shape[0]): - data_rows.append(empty_row) - indices_rows.append(empty_row) + for _ in range(0, num_rows): + data_rows.append([empty_row]) + indices_rows.append([empty_row]) - data_row = np.zeros(seqs_mat2.shape[0]) - indices_row = np.zeros(seqs_mat2.shape[0]) - for row_index in range(seqs_mat1.shape[0]): + for row_index in nb.prange(num_rows): + data_row = np.empty(num_cols) + indices_row = np.empty(num_cols) + row_end_index = 0 - for col_index in range(start_column + row_index * is_symmetric, seqs_mat2.shape[0]): - q_L = seqs_L1[row_index] - s_L = seqs_L2[col_index] + seq1_len = seqs_L1[row_index] + + for col_index in range(start_column + row_index * is_symmetric, num_cols): distance = 1 + seq2_len = seqs_L2[col_index] - if q_L == s_L: - for i in range(ntrim, q_L - ctrim): + if seq1_len == seq2_len: + for i in range(ntrim, seq1_len - ctrim): distance += dist_mat_weighted[seqs_mat1[row_index, i], seqs_mat2[col_index, i]] else: - short_len = min(q_L, s_L) - len_diff = abs(q_L - s_L) + short_len = min(seq1_len, seq2_len) + len_diff = abs(seq1_len - seq2_len) if fixed_gappos: min_gappos = min(6, 3 + (short_len - 5) // 2) max_gappos = min_gappos @@ -736,7 +783,7 @@ def _nb_tcrdist_mat(): for c_i in range(ctrim, remainder): tmp_dist += dist_mat_weighted[ - seqs_mat1[row_index, q_L - 1 - c_i], seqs_mat2[col_index, s_L - 1 - c_i] + seqs_mat1[row_index, seq1_len - 1 - c_i], seqs_mat2[col_index, seq2_len - 1 - c_i] ] if tmp_dist < min_dist or min_dist == -1: @@ -752,14 +799,20 @@ def _nb_tcrdist_mat(): indices_row[row_end_index] = col_index row_end_index += 1 - data_rows[row_index] = data_row[0:row_end_index].copy() - indices_rows[row_index] = indices_row[0:row_end_index].copy() + data_rows[row_index][0] = data_row[0:row_end_index].copy() + indices_rows[row_index][0] = indices_row[0:row_end_index].copy() row_element_counts[row_index] = row_end_index return data_rows, indices_rows, row_element_counts data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() + data_rows_flat = [] + indices_rows_flat = [] - return data_rows, indices_rows, row_element_counts + for i in range(len(data_rows)): + data_rows_flat.append(data_rows[i][0]) + indices_rows_flat.append(indices_rows[i][0]) + + return data_rows_flat, indices_rows_flat, row_element_counts _metric_mat = _tcrdist_mat diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index a1e512ff5..87feb956c 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -645,6 +645,7 @@ def test_tcrdist_reference(): fixed_gappos=True, cutoff=15, n_jobs=2, + n_blocks=2, ) res = tcrdist_calculator.calc_dist_mat(seqs, seqs) @@ -660,7 +661,7 @@ def test_hamming_reference(): seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_csr_result.npz") - hamming_calculator = HammingDistanceCalculator(2, 2) + hamming_calculator = HammingDistanceCalculator(2, 2, 2) res = hamming_calculator.calc_dist_mat(seqs, seqs) assert np.array_equal(res.data, reference_result.data) From 0005e637fee9e0b1ecda43c339466a3c5a64dc91 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 6 May 2024 17:46:14 +0200 Subject: [PATCH 16/94] performance optimization for hamming and tcrdist --- src/scirpy/ir_dist/metrics.py | 79 ++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 45fa184ae..c717baabe 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -553,9 +553,13 @@ def _hamming_mat( nb.set_num_threads(self.n_jobs) num_threads = nb.get_num_threads() - print("numba threads: ", num_threads) - @nb.jit(nopython=True, parallel=True, nogil=True) + if(num_threads>1): + jit_parallel = True + else: + jit_parallel = False + + @nb.jit(nopython=True, parallel=jit_parallel, nogil=True) def _nb_hamming_mat(): assert seqs_mat1.shape[0] == seqs_L1.shape[0] assert seqs_mat2.shape[0] == seqs_L2.shape[0] @@ -572,10 +576,11 @@ def _nb_hamming_mat(): data_rows.append([empty_row]) indices_rows.append([empty_row]) - for row_index in nb.prange(num_rows): - data_row = np.empty(num_cols) - indices_row = np.empty(num_cols) + data_row_matrix = np.empty((num_threads, num_cols)) + indices_row_matrix = np.empty((num_threads, num_cols)) + for row_index in nb.prange(num_rows): + thread_id = nb.get_thread_id() row_end_index = 0 seq1_len = seqs_L1[row_index] @@ -587,25 +592,25 @@ def _nb_hamming_mat(): distance += seqs_mat1[row_index, i] != seqs_mat2[col_index, i] if distance <= cutoff + 1: - data_row[row_end_index] = distance - indices_row[row_end_index] = col_index + data_row_matrix[thread_id, row_end_index] = distance + indices_row_matrix[thread_id, row_end_index] = col_index row_end_index += 1 - data_rows[row_index][0] = data_row[0:row_end_index].copy() - indices_rows[row_index][0] = indices_row[0:row_end_index].copy() + data_rows[row_index][0] = data_row_matrix[thread_id, 0:row_end_index].copy() + indices_rows[row_index][0] = indices_row_matrix[thread_id, 0:row_end_index].copy() row_element_counts[row_index] = row_end_index - return data_rows, indices_rows, row_element_counts + data_rows_flat = [] + indices_rows_flat = [] - data_rows, indices_rows, row_element_counts = _nb_hamming_mat() - data_rows_flat = [] - indices_rows_flat = [] + for i in range(len(data_rows)): + data_rows_flat.append(data_rows[i][0]) + indices_rows_flat.append(indices_rows[i][0]) - for i in range(len(data_rows)): - data_rows_flat.append(data_rows[i][0]) - indices_rows_flat.append(indices_rows[i][0]) + return data_rows_flat, indices_rows_flat, row_element_counts - return data_rows_flat, indices_rows_flat, row_element_counts + data_rows, indices_rows, row_element_counts = _nb_hamming_mat() + return data_rows, indices_rows, row_element_counts _metric_mat = _hamming_mat @@ -726,9 +731,13 @@ def _tcrdist_mat( nb.set_num_threads(self.n_jobs) num_threads = nb.get_num_threads() - print("numba threads: ", num_threads) + + if(num_threads>1): + jit_parallel = True + else: + jit_parallel = False - @nb.jit(nopython=True, parallel=True, nogil=True) + @nb.jit(nopython=True, parallel=jit_parallel, nogil=True) def _nb_tcrdist_mat(): assert seqs_mat1.shape[0] == seqs_L1.shape[0] assert seqs_mat2.shape[0] == seqs_L2.shape[0] @@ -745,10 +754,11 @@ def _nb_tcrdist_mat(): data_rows.append([empty_row]) indices_rows.append([empty_row]) + data_row_matrix = np.empty((num_threads, num_cols)) + indices_row_matrix = np.empty((num_threads, num_cols)) + for row_index in nb.prange(num_rows): - data_row = np.empty(num_cols) - indices_row = np.empty(num_cols) - + thread_id = nb.get_thread_id() row_end_index = 0 seq1_len = seqs_L1[row_index] @@ -795,24 +805,25 @@ def _nb_tcrdist_mat(): distance = min_dist + len_diff * gap_penalty + 1 if distance <= cutoff + 1: - data_row[row_end_index] = distance - indices_row[row_end_index] = col_index + data_row_matrix[thread_id, row_end_index] = distance + indices_row_matrix[thread_id, row_end_index] = col_index row_end_index += 1 - data_rows[row_index][0] = data_row[0:row_end_index].copy() - indices_rows[row_index][0] = indices_row[0:row_end_index].copy() + data_rows[row_index][0] = data_row_matrix[thread_id, 0:row_end_index].copy() + indices_rows[row_index][0] = indices_row_matrix[thread_id, 0:row_end_index].copy() row_element_counts[row_index] = row_end_index - return data_rows, indices_rows, row_element_counts - data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() - data_rows_flat = [] - indices_rows_flat = [] + data_rows_flat = [] + indices_rows_flat = [] - for i in range(len(data_rows)): - data_rows_flat.append(data_rows[i][0]) - indices_rows_flat.append(indices_rows[i][0]) + for i in range(len(data_rows)): + data_rows_flat.append(data_rows[i][0]) + indices_rows_flat.append(indices_rows[i][0]) - return data_rows_flat, indices_rows_flat, row_element_counts + return data_rows_flat, indices_rows_flat, row_element_counts + + data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() + return data_rows, indices_rows, row_element_counts _metric_mat = _tcrdist_mat From 6f16a3e75301750a3216b266c598156396e945db Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 6 May 2024 18:07:16 +0200 Subject: [PATCH 17/94] more documentation added --- src/scirpy/ir_dist/metrics.py | 68 +++++++++++++++++++++++- src/scirpy/tests/test_ir_dist_metrics.py | 2 +- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index c717baabe..2e4d017a8 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -410,6 +410,20 @@ def _seqs2mat( class NumbaDistanceCalculator(abc.ABC): + """ + This is an abstract base class for distance calculator classes that compute parwise distances between + gene sequences in parallel based on a certain distance metric. The result is a (scipy) compressed sparse row distance matrix. + Derived classes just need to implement the method _metric_mat (see method comments for more details). + + Parameters + ---------- + n_jobs: + Number of threads per process to use for the pairwise distance calculation + n_blocks: + Overall number of blocks given to the workers (processes) + """ + + def __init__(self, n_jobs: int = 1, n_blocks: int = 1): super().__init__() self.n_jobs = n_jobs @@ -529,6 +543,19 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No class HammingDistanceCalculator(NumbaDistanceCalculator): + """Computes pairwise distances between gene sequences based on the "hamming" distance metric. + + Parameters + ---------- + cutoff: + Will eleminate distances > cutoff to make efficient + use of sparse matrices. + n_jobs: + Number of numba parallel threads to use for the pairwise distance calculation + n_blocks: + Number of joblib delayed objects (blocks to compute) given to joblib.Parallel + """ + def __init__( self, n_jobs: int = 1, @@ -548,6 +575,43 @@ def _hamming_mat( is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + """Computes the pairwise hamming distances for sequences in seqs_mat1 and seqs_mat2. + + This function is a wrapper and contains an inner JIT compiled numba function without parameters. The reason for this is + that this way some of the parameters can be treated as constant by numba and this allows for a better optimization + of the numba compiler in this specific case. + + If this function is used to compute a block of a bigger result matrix, is_symmetric and start_column + can be used to only compute the part of the block that would be part of the upper triangular matrix of the + result matrix. + + Parameters + ---------- + seqs_mat1/2: + Matrix containing sequences created by seqs2mat with padding to accomodate + sequences of different lengths (-1 padding) + seqs_L1/2: + A vector containing the length of each sequence in the respective seqs_mat matrix, + without the padding in seqs_mat + is_symmetric: + Determines whether the final result matrix is symmetric, assuming that this function is + only used to compute a block of a bigger result matrix + start_column: + Determines at which column the calculation should be started. This is only used if this function is + used to compute a block of a bigger result matrix that is symmetric + + Returns + ------- + data_rows: + List with arrays containing the non-zero data values of the result matrix per row, + needed to create the final scipy CSR result matrix later + indices_rows: + List with arrays containing the non-zero entry column indeces of the result matrix per row, + needed to create the final scipy CSR result matrix later + row_element_counts: + Array with integers that indicate the amount of non-zero values of the result matrix per row, + needed to create the final scipy CSR result matrix later + """ cutoff = self.cutoff start_column *= is_symmetric @@ -640,7 +704,9 @@ class TCRdistDistanceCalculator(NumbaDistanceCalculator): Will eleminate distances > cutoff to make efficient use of sparse matrices. n_jobs: - Number of jobs (processes) to use for the pairwise distance calculation + Number of numba parallel threads to use for the pairwise distance calculation + n_blocks: + Number of joblib delayed objects (blocks to compute) given to joblib.Parallel """ parasail_aa_alphabet = "ARNDCQEGHILKMFPSTWYVBZX" diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 87feb956c..a735cb551 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -208,7 +208,7 @@ def test_levensthein_dist_with_two_seq_arrays(): def test_hamming_dist(): - hamming10 = HammingDistanceCalculator(2) + hamming10 = HammingDistanceCalculator(cutoff = 2) res = hamming10.calc_dist_mat(np.array(["A", "AA", "AAA", "AAR", "ZZZZZZ"]), np.array(["RRR", "AR"])) assert isinstance(res, scipy.sparse.csr_matrix) assert res.shape == (5, 2) From ad13f522989c9ffa25b4c9e1bed72d6a5aad35a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 16:08:23 +0000 Subject: [PATCH 18/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 19 +++++++++---------- src/scirpy/tests/test_ir_dist_metrics.py | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 30fac45cf..82d4d923d 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -422,7 +422,6 @@ class NumbaDistanceCalculator(abc.ABC): Overall number of blocks given to the workers (processes) """ - def __init__(self, n_jobs: int = 1, n_blocks: int = 1): super().__init__() self.n_jobs = n_jobs @@ -441,13 +440,13 @@ def _metric_mat( ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: """ This function should be implemented by the derived class in a way sucht that it computes the pairwise distances - for sequences in seqs_mat1 and seqs_mat2 based on a certain distance metric. The result should be a distance matrix + for sequences in seqs_mat1 and seqs_mat2 based on a certain distance metric. The result should be a distance matrix that is returned in the form of the data, indices and intptr arrays of a (scipy) compressed sparse row matrix. If this function is used to compute a block of a bigger result matrix, is_symmetric and start_column can be used to only compute the part of the block that would be part of the upper triangular matrix of the result matrix. - + Parameters ---------- seqs_mat1/2: @@ -520,7 +519,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No seqs = np.array(seqs) seqs2 = np.array(seqs2) is_symmetric = np.array_equal(seqs, seqs2) - + if self.n_blocks > 1: split_seqs = np.array_split(seqs, self.n_blocks) start_columns = np.cumsum([0] + [len(seq) for seq in split_seqs[:-1]]) @@ -543,7 +542,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No class HammingDistanceCalculator(NumbaDistanceCalculator): """Computes pairwise distances between gene sequences based on the "hamming" distance metric. - + Parameters ---------- cutoff: @@ -617,7 +616,7 @@ def _hamming_mat( nb.set_num_threads(self.n_jobs) num_threads = nb.get_num_threads() - if(num_threads>1): + if num_threads > 1: jit_parallel = True else: jit_parallel = False @@ -796,8 +795,8 @@ def _tcrdist_mat( nb.set_num_threads(self.n_jobs) num_threads = nb.get_num_threads() - - if(num_threads>1): + + if num_threads > 1: jit_parallel = True else: jit_parallel = False @@ -821,7 +820,7 @@ def _nb_tcrdist_mat(): data_row_matrix = np.empty((num_threads, num_cols)) indices_row_matrix = np.empty((num_threads, num_cols)) - + for row_index in nb.prange(num_rows): thread_id = nb.get_thread_id() row_end_index = 0 @@ -886,7 +885,7 @@ def _nb_tcrdist_mat(): indices_rows_flat.append(indices_rows[i][0]) return data_rows_flat, indices_rows_flat, row_element_counts - + data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() return data_rows, indices_rows, row_element_counts diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index a735cb551..4b9bbdfaa 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -208,7 +208,7 @@ def test_levensthein_dist_with_two_seq_arrays(): def test_hamming_dist(): - hamming10 = HammingDistanceCalculator(cutoff = 2) + hamming10 = HammingDistanceCalculator(cutoff=2) res = hamming10.calc_dist_mat(np.array(["A", "AA", "AAA", "AAR", "ZZZZZZ"]), np.array(["RRR", "AR"])) assert isinstance(res, scipy.sparse.csr_matrix) assert res.shape == (5, 2) From 08ad83876637a83465396b1f07dc14e0a2a62a8d Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Tue, 7 May 2024 09:15:40 +0200 Subject: [PATCH 19/94] documentation adapted --- src/scirpy/ir_dist/metrics.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 30fac45cf..178d53064 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -408,12 +408,15 @@ def _seqs2mat( return mat, L -class NumbaDistanceCalculator(abc.ABC): +class MetricDistanceCalculator(abc.ABC): """ - This is an abstract base class for distance calculator classes that compute parwise distances between + Abstract base class for distance calculator classes that compute parwise distances between gene sequences in parallel based on a certain distance metric. The result is a (scipy) compressed sparse row distance matrix. Derived classes just need to implement the method _metric_mat (see method comments for more details). + The code of this class is based on `pwseqdist `_. + Reused under MIT license, Copyright (c) 2020 Andrew Fiore-Gartland. + Parameters ---------- n_jobs: @@ -541,8 +544,11 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No return full_distance_matrix -class HammingDistanceCalculator(NumbaDistanceCalculator): +class HammingDistanceCalculator(MetricDistanceCalculator): """Computes pairwise distances between gene sequences based on the "hamming" distance metric. + + The code of this class is based on `pwseqdist `_. + Reused under MIT license, Copyright (c) 2020 Andrew Fiore-Gartland. Parameters ---------- @@ -678,7 +684,7 @@ def _nb_hamming_mat(): _metric_mat = _hamming_mat -class TCRdistDistanceCalculator(NumbaDistanceCalculator): +class TCRdistDistanceCalculator(MetricDistanceCalculator): """Computes pairwise distances between TCR CDR3 sequences based on the "tcrdist" distance metric. The code of this class is heavily based on `pwseqdist `_. From b86030c1ffa008349815e7bc8aa0f1fcbb9ba592 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 07:18:55 +0000 Subject: [PATCH 20/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 35ca3a3a3..89b212de3 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -548,7 +548,7 @@ class HammingDistanceCalculator(MetricDistanceCalculator): The code of this class is based on `pwseqdist `_. Reused under MIT license, Copyright (c) 2020 Andrew Fiore-Gartland. - + Parameters ---------- cutoff: From 2fb82547a761f803c7785f60dd13dd3d55c75d2e Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Tue, 7 May 2024 09:29:57 +0200 Subject: [PATCH 21/94] documentation adapted --- src/scirpy/ir_dist/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 35ca3a3a3..43985899b 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -442,7 +442,7 @@ def _metric_mat( start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: """ - This function should be implemented by the derived class in a way sucht that it computes the pairwise distances + Abstract method that should be implemented by the derived class in a way sucht that it computes the pairwise distances for sequences in seqs_mat1 and seqs_mat2 based on a certain distance metric. The result should be a distance matrix that is returned in the form of the data, indices and intptr arrays of a (scipy) compressed sparse row matrix. From 80ae27109cfe573236ef557660b3e5f3526e58ab Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 7 Aug 2024 18:27:43 +0200 Subject: [PATCH 22/94] signature of _calc_dist_mat_block changed --- src/scirpy/ir_dist/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 441f82517..02030164c 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -482,7 +482,7 @@ def _metric_mat( def _calc_dist_mat_block( self, seqs: Sequence[str], - seqs2: Optional[Sequence[str]] = None, + seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, ) -> csr_matrix: From 91c1dea2cb551ea2493ce3b90a247b8c05f98c1c Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 7 Aug 2024 19:42:54 +0200 Subject: [PATCH 23/94] the alphabet for the hamming distance is now the unique characters occuring in all sequences --- src/scirpy/ir_dist/metrics.py | 68 +++++++++++++++-------------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 02030164c..0c2f864cc 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -434,16 +434,14 @@ def __init__(self, n_jobs: int = 1, n_blocks: int = 1): def _metric_mat( self, *, - seqs_mat1: np.ndarray, - seqs_mat2: np.ndarray, - seqs_L1: np.ndarray, - seqs_L2: np.ndarray, + seqs: Sequence[str], + seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: """ - Abstract method that should be implemented by the derived class in a way sucht that it computes the pairwise distances - for sequences in seqs_mat1 and seqs_mat2 based on a certain distance metric. The result should be a distance matrix + Abstract method that should be implemented by the derived class in a way such that it computes the pairwise distances + for gene sequences in seqs and seqs2 based on a certain distance metric. The result should be a distance matrix that is returned in the form of the data, indices and intptr arrays of a (scipy) compressed sparse row matrix. If this function is used to compute a block of a bigger result matrix, is_symmetric and start_column @@ -452,12 +450,8 @@ def _metric_mat( Parameters ---------- - seqs_mat1/2: - Matrix containing sequences created by seqs2mat with padding to accomodate - sequences of different lengths (-1 padding) - seqs_L1/2: - A vector containing the length of each sequence in the respective seqs_mat matrix, - without the padding in seqs_mat + seqs/2: + A python sequence of strings representing gene sequences is_symmetric: Determines whether the final result matrix is symmetric, assuming that this function is only used to compute a block of a bigger result matrix @@ -493,15 +487,9 @@ def _calc_dist_mat_block( if len(seqs) == 0 or len(seqs2) == 0: return csr_matrix((len(seqs), len(seqs2))) - max_seq_len = max(np.max([len(s) for s in seqs]), np.max([len(s) for s in seqs2])) - seqs_mat1, seqs_L1 = _seqs2mat(seqs, max_len=max_seq_len) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2, max_len=max_seq_len) - data_rows, indices_rows, row_element_counts = self._metric_mat( - seqs_mat1=seqs_mat1, - seqs_mat2=seqs_mat2, - seqs_L1=seqs_L1, - seqs_L2=seqs_L2, + seqs=seqs, + seqs2=seqs2, is_symmetric=is_symmetric, start_column=start_column, ) @@ -572,14 +560,12 @@ def __init__( def _hamming_mat( self, *, - seqs_mat1: np.ndarray, - seqs_mat2: np.ndarray, - seqs_L1: np.ndarray, - seqs_L2: np.ndarray, + seqs: Sequence[str], + seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: - """Computes the pairwise hamming distances for sequences in seqs_mat1 and seqs_mat2. + """Computes the pairwise hamming distances for sequences in seqs and seqs2. This function is a wrapper and contains an inner JIT compiled numba function without parameters. The reason for this is that this way some of the parameters can be treated as constant by numba and this allows for a better optimization @@ -591,12 +577,8 @@ def _hamming_mat( Parameters ---------- - seqs_mat1/2: - Matrix containing sequences created by seqs2mat with padding to accomodate - sequences of different lengths (-1 padding) - seqs_L1/2: - A vector containing the length of each sequence in the respective seqs_mat matrix, - without the padding in seqs_mat + seqs/2: + A python sequence of strings representing gene sequences is_symmetric: Determines whether the final result matrix is symmetric, assuming that this function is only used to compute a block of a bigger result matrix @@ -616,6 +598,12 @@ def _hamming_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ + unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) + max_seq_len = max((len(s) for s in (*seqs, *seqs2))) + + seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + cutoff = self.cutoff start_column *= is_symmetric @@ -743,14 +731,12 @@ def __init__( def _tcrdist_mat( self, *, - seqs_mat1: np.ndarray, - seqs_mat2: np.ndarray, - seqs_L1: np.ndarray, - seqs_L2: np.ndarray, + seqs: Sequence[str], + seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: - """Computes the pairwise TCRdist distances for sequences in seqs_mat1 and seqs_mat2. + """Computes the pairwise TCRdist distances for sequences in seqs and seqs2. This function is a wrapper and contains an inner JIT compiled numba function without parameters. The reason for this is that this way some of the parameters can be treated as constant by numba and this allows for a better optimization @@ -764,9 +750,8 @@ def _tcrdist_mat( Parameters ---------- - seqs_mat1/2: - Matrix containing sequences created by seqs2mat with padding to accomodate - sequences of different lengths (-1 padding) + seqs/2: + A python sequence of strings representing gene sequences seqs_L1/2: A vector containing the length of each sequence in the respective seqs_mat matrix, without the padding in seqs_mat @@ -789,6 +774,11 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ + max_seq_len = max((len(s) for s in (*seqs, *seqs2))) + + seqs_mat1, seqs_L1 = _seqs2mat(seqs, max_len=max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, max_len=max_seq_len) + cutoff = self.cutoff dist_weight = self.dist_weight gap_penalty = self.gap_penalty From 899e2eb8a00bc626c2a0f6589291f507397521e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:46:36 +0000 Subject: [PATCH 24/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 0c2f864cc..27e3b7946 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -599,8 +599,8 @@ def _hamming_mat( needed to create the final scipy CSR result matrix later """ unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) - max_seq_len = max((len(s) for s in (*seqs, *seqs2))) - + max_seq_len = max(len(s) for s in (*seqs, *seqs2)) + seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) @@ -774,11 +774,11 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ - max_seq_len = max((len(s) for s in (*seqs, *seqs2))) - + max_seq_len = max(len(s) for s in (*seqs, *seqs2)) + seqs_mat1, seqs_L1 = _seqs2mat(seqs, max_len=max_seq_len) seqs_mat2, seqs_L2 = _seqs2mat(seqs2, max_len=max_seq_len) - + cutoff = self.cutoff dist_weight = self.dist_weight gap_penalty = self.gap_penalty From d5dbe8e36de53489c7980741831c90ca5bed227b Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 9 Aug 2024 17:19:06 +0200 Subject: [PATCH 25/94] normalized hamming distance added --- src/scirpy/ir_dist/__init__.py | 2 ++ src/scirpy/ir_dist/metrics.py | 6 ++++++ src/scirpy/tests/test_ir_dist_metrics.py | 12 +++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 223fec74b..4bef13e5b 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -99,6 +99,8 @@ def _get_distance_calculator(metric: MetricType, cutoff: Union[int, None], *, n_ dist_calc = metrics.LevenshteinDistanceCalculator(n_jobs=n_jobs, **kwargs) elif metric == "hamming": dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, **kwargs) + elif metric == "normalized_hamming": + dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, normalize= True, **kwargs) elif metric == "tcrdist": dist_calc = metrics.TCRdistDistanceCalculator(n_jobs=n_jobs, **kwargs) else: diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 27e3b7946..f9bf403b6 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -553,9 +553,11 @@ def __init__( n_jobs: int = 1, n_blocks: int = 1, cutoff: int = 2, + normalize: bool = False, ): super().__init__(n_jobs=n_jobs, n_blocks=n_blocks) self.cutoff = cutoff + self.normalize = normalize def _hamming_mat( self, @@ -605,6 +607,7 @@ def _hamming_mat( seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) cutoff = self.cutoff + normalize = self.normalize start_column *= is_symmetric nb.set_num_threads(self.n_jobs) @@ -646,6 +649,9 @@ def _nb_hamming_mat(): if seq1_len == seq2_len: for i in range(0, seq1_len): distance += seqs_mat1[row_index, i] != seqs_mat2[col_index, i] + + if(normalize): + distance = int((distance-1) * 100 / seq1_len + 0.5) + 1 if distance <= cutoff + 1: data_row_matrix[thread_id, row_end_index] = distance diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 4b9bbdfaa..3395208da 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -305,7 +305,7 @@ def test_fast_alignment_dist_with_two_seq_arrays(): npt.assert_almost_equal(res.toarray(), np.array([[0, 1, 5], [0, 5, 10], [0, 0, 0], [1, 0, 0]])) -@pytest.mark.parametrize("metric", ["alignment", "fastalignment", "identity", "hamming", "levenshtein"]) +@pytest.mark.parametrize("metric", ["alignment", "fastalignment", "identity", "hamming", "normalized_hamming", "levenshtein", "tcrdist"]) def test_sequence_dist_all_metrics(metric): # Smoke test, no assertions! # Smoke test, no assertions! @@ -667,3 +667,13 @@ def test_hamming_reference(): assert np.array_equal(res.data, reference_result.data) assert np.array_equal(res.indices, reference_result.indices) assert np.array_equal(res.indptr, reference_result.indptr) + +def test_normalized_hamming_reference(): + hamming_calculator = HammingDistanceCalculator(1, 1, 50, True) + seq1 = np.array(["AAAA", "AAB", "AABB", "ABA"]) + seq2 = np.array(["ABB", "ABBB", "ABBB"]) + expected_result = np.array([[0, 0, 0], [34, 0, 0], [0, 26, 26],[34, 0, 0]]) + res = hamming_calculator.calc_dist_mat(seq1, seq2) + assert isinstance(res, scipy.sparse.csr_matrix) + assert res.shape == expected_result.shape + assert np.array_equal(res.todense(), expected_result) From 5a6ef24e9d531907dc86df3e30d56377b7e24bc1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:19:37 +0000 Subject: [PATCH 26/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/__init__.py | 2 +- src/scirpy/ir_dist/metrics.py | 6 +++--- src/scirpy/tests/test_ir_dist_metrics.py | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 4bef13e5b..51181753b 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -100,7 +100,7 @@ def _get_distance_calculator(metric: MetricType, cutoff: Union[int, None], *, n_ elif metric == "hamming": dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, **kwargs) elif metric == "normalized_hamming": - dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, normalize= True, **kwargs) + dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, normalize=True, **kwargs) elif metric == "tcrdist": dist_calc = metrics.TCRdistDistanceCalculator(n_jobs=n_jobs, **kwargs) else: diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index f9bf403b6..15553525a 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -649,9 +649,9 @@ def _nb_hamming_mat(): if seq1_len == seq2_len: for i in range(0, seq1_len): distance += seqs_mat1[row_index, i] != seqs_mat2[col_index, i] - - if(normalize): - distance = int((distance-1) * 100 / seq1_len + 0.5) + 1 + + if normalize: + distance = int((distance - 1) * 100 / seq1_len + 0.5) + 1 if distance <= cutoff + 1: data_row_matrix[thread_id, row_end_index] = distance diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 3395208da..f28013a49 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -305,7 +305,9 @@ def test_fast_alignment_dist_with_two_seq_arrays(): npt.assert_almost_equal(res.toarray(), np.array([[0, 1, 5], [0, 5, 10], [0, 0, 0], [1, 0, 0]])) -@pytest.mark.parametrize("metric", ["alignment", "fastalignment", "identity", "hamming", "normalized_hamming", "levenshtein", "tcrdist"]) +@pytest.mark.parametrize( + "metric", ["alignment", "fastalignment", "identity", "hamming", "normalized_hamming", "levenshtein", "tcrdist"] +) def test_sequence_dist_all_metrics(metric): # Smoke test, no assertions! # Smoke test, no assertions! @@ -668,11 +670,12 @@ def test_hamming_reference(): assert np.array_equal(res.indices, reference_result.indices) assert np.array_equal(res.indptr, reference_result.indptr) + def test_normalized_hamming_reference(): hamming_calculator = HammingDistanceCalculator(1, 1, 50, True) seq1 = np.array(["AAAA", "AAB", "AABB", "ABA"]) seq2 = np.array(["ABB", "ABBB", "ABBB"]) - expected_result = np.array([[0, 0, 0], [34, 0, 0], [0, 26, 26],[34, 0, 0]]) + expected_result = np.array([[0, 0, 0], [34, 0, 0], [0, 26, 26], [34, 0, 0]]) res = hamming_calculator.calc_dist_mat(seq1, seq2) assert isinstance(res, scipy.sparse.csr_matrix) assert res.shape == expected_result.shape From 3060314aab26d334e9f9f160be5b0319acf46481 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 9 Aug 2024 18:02:15 +0200 Subject: [PATCH 27/94] renaming test --- src/scirpy/tests/test_ir_dist_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 3395208da..22e911eb2 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -668,7 +668,7 @@ def test_hamming_reference(): assert np.array_equal(res.indices, reference_result.indices) assert np.array_equal(res.indptr, reference_result.indptr) -def test_normalized_hamming_reference(): +def test_normalized_hamming(): hamming_calculator = HammingDistanceCalculator(1, 1, 50, True) seq1 = np.array(["AAAA", "AAB", "AABB", "ABA"]) seq2 = np.array(["ABB", "ABBB", "ABBB"]) From 6b2b025b7102037f573726ff674977b96063ab22 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 9 Aug 2024 19:34:46 +0200 Subject: [PATCH 28/94] histogram creation for hamming distance added --- src/scirpy/ir_dist/metrics.py | 49 ++++++++++++++++++++++-- src/scirpy/tests/test_ir_dist_metrics.py | 8 ++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index f9bf403b6..99ef596d6 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -12,6 +12,8 @@ from Levenshtein import distance as levenshtein_dist from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix +import matplotlib.pyplot as plt + from scirpy.util import _doc_params, _parallelize_with_joblib, deprecated @@ -554,10 +556,27 @@ def __init__( n_blocks: int = 1, cutoff: int = 2, normalize: bool = False, + histogram: bool = False, ): super().__init__(n_jobs=n_jobs, n_blocks=n_blocks) self.cutoff = cutoff self.normalize = normalize + self.histogram = histogram + + def _make_histogram(self, row_mins): + if(self.normalize==True): + bins = np.arange(0,101,2) + else: + max_value = np.max(row_mins) + bin_step = np.ceil(max_value/100) + bins = np.arange(0,max_value+1,bin_step) + plt.hist(row_mins, bins = bins, histtype = "bar", edgecolor ="black") + plt.axvline(x=self.cutoff, color='r', linestyle='-', label="cutoff") + plt.legend() + plt.xlabel('Distance to nearest neighbor') + plt.ylabel('Count') + plt.title('Histogram of \"distance-to-nearest\"-distribution') + plt.show() def _hamming_mat( self, @@ -608,6 +627,7 @@ def _hamming_mat( cutoff = self.cutoff normalize = self.normalize + histogram = self.histogram start_column *= is_symmetric nb.set_num_threads(self.n_jobs) @@ -630,6 +650,11 @@ def _nb_hamming_mat(): indices_rows = nb.typed.List() row_element_counts = np.zeros(num_rows) + if(histogram): + row_mins = np.zeros(num_rows) + else: + row_mins = np.zeros(0) + empty_row = np.zeros(0) for _ in range(0, num_rows): data_rows.append([empty_row]) @@ -643,6 +668,12 @@ def _nb_hamming_mat(): row_end_index = 0 seq1_len = seqs_L1[row_index] + if(histogram): + if(normalize): + row_min = 100 + else: + row_min = seq1_len + for col_index in range(start_column + row_index * is_symmetric, num_cols): distance = 1 seq2_len = seqs_L2[col_index] @@ -658,9 +689,15 @@ def _nb_hamming_mat(): indices_row_matrix[thread_id, row_end_index] = col_index row_end_index += 1 + if(histogram): + if(distance>1): + row_min = min(row_min, distance-1) + data_rows[row_index][0] = data_row_matrix[thread_id, 0:row_end_index].copy() indices_rows[row_index][0] = indices_row_matrix[thread_id, 0:row_end_index].copy() row_element_counts[row_index] = row_end_index + if(histogram): + row_mins[row_index] = row_min data_rows_flat = [] indices_rows_flat = [] @@ -669,12 +706,16 @@ def _nb_hamming_mat(): data_rows_flat.append(data_rows[i][0]) indices_rows_flat.append(indices_rows[i][0]) - return data_rows_flat, indices_rows_flat, row_element_counts + return data_rows_flat, indices_rows_flat, row_element_counts, row_mins - data_rows, indices_rows, row_element_counts = _nb_hamming_mat() - return data_rows, indices_rows, row_element_counts + data_rows, indices_rows, row_element_counts, row_mins = _nb_hamming_mat() + + if(histogram): + self._make_histogram(row_mins) - _metric_mat = _hamming_mat + return data_rows, indices_rows, row_element_counts, row_mins + + _metric_mat = lambda self, *args, **kwargs: self._hamming_mat(*args, **kwargs)[:3] class TCRdistDistanceCalculator(MetricDistanceCalculator): diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 22e911eb2..af8bfda35 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -677,3 +677,11 @@ def test_normalized_hamming(): assert isinstance(res, scipy.sparse.csr_matrix) assert res.shape == expected_result.shape assert np.array_equal(res.todense(), expected_result) + +def test_hamming_histogram(): + hamming_calculator = HammingDistanceCalculator(1, 1, 100, True, True) + seq1 = np.array(["AAAA", "AA", "AABB", "ABA"]) + seq2 = np.array(["ABB", "ABBB", "ABBB"]) + expected_result = np.array([75, 100, 25, 33]) + _, _, _, res = hamming_calculator._hamming_mat(seqs=seq1, seqs2=seq2) + assert np.array_equal(res, expected_result) From 5670a84cc98ef629cd9c6dfcf5883b871a39fbb4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:38:01 +0000 Subject: [PATCH 29/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 41 ++++++++++++------------ src/scirpy/tests/test_ir_dist_metrics.py | 2 ++ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 247ba76b2..c13bf368f 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -5,6 +5,7 @@ from typing import Optional, Union import joblib +import matplotlib.pyplot as plt import numba as nb import numpy as np import scipy.sparse @@ -12,8 +13,6 @@ from Levenshtein import distance as levenshtein_dist from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix -import matplotlib.pyplot as plt - from scirpy.util import _doc_params, _parallelize_with_joblib, deprecated @@ -564,18 +563,18 @@ def __init__( self.histogram = histogram def _make_histogram(self, row_mins): - if(self.normalize==True): - bins = np.arange(0,101,2) + if self.normalize == True: + bins = np.arange(0, 101, 2) else: max_value = np.max(row_mins) - bin_step = np.ceil(max_value/100) - bins = np.arange(0,max_value+1,bin_step) - plt.hist(row_mins, bins = bins, histtype = "bar", edgecolor ="black") - plt.axvline(x=self.cutoff, color='r', linestyle='-', label="cutoff") + bin_step = np.ceil(max_value / 100) + bins = np.arange(0, max_value + 1, bin_step) + plt.hist(row_mins, bins=bins, histtype="bar", edgecolor="black") + plt.axvline(x=self.cutoff, color="r", linestyle="-", label="cutoff") plt.legend() - plt.xlabel('Distance to nearest neighbor') - plt.ylabel('Count') - plt.title('Histogram of \"distance-to-nearest\"-distribution') + plt.xlabel("Distance to nearest neighbor") + plt.ylabel("Count") + plt.title('Histogram of "distance-to-nearest"-distribution') plt.show() def _hamming_mat( @@ -650,7 +649,7 @@ def _nb_hamming_mat(): indices_rows = nb.typed.List() row_element_counts = np.zeros(num_rows) - if(histogram): + if histogram: row_mins = np.zeros(num_rows) else: row_mins = np.zeros(0) @@ -668,12 +667,12 @@ def _nb_hamming_mat(): row_end_index = 0 seq1_len = seqs_L1[row_index] - if(histogram): - if(normalize): + if histogram: + if normalize: row_min = 100 else: row_min = seq1_len - + for col_index in range(start_column + row_index * is_symmetric, num_cols): distance = 1 seq2_len = seqs_L2[col_index] @@ -689,14 +688,14 @@ def _nb_hamming_mat(): indices_row_matrix[thread_id, row_end_index] = col_index row_end_index += 1 - if(histogram): - if(distance>1): - row_min = min(row_min, distance-1) + if histogram: + if distance > 1: + row_min = min(row_min, distance - 1) data_rows[row_index][0] = data_row_matrix[thread_id, 0:row_end_index].copy() indices_rows[row_index][0] = indices_row_matrix[thread_id, 0:row_end_index].copy() row_element_counts[row_index] = row_end_index - if(histogram): + if histogram: row_mins[row_index] = row_min data_rows_flat = [] @@ -710,11 +709,11 @@ def _nb_hamming_mat(): data_rows, indices_rows, row_element_counts, row_mins = _nb_hamming_mat() - if(histogram): + if histogram: self._make_histogram(row_mins) return data_rows, indices_rows, row_element_counts, row_mins - + _metric_mat = lambda self, *args, **kwargs: self._hamming_mat(*args, **kwargs)[:3] diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 49eb4107d..bd955266f 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -670,6 +670,7 @@ def test_hamming_reference(): assert np.array_equal(res.indices, reference_result.indices) assert np.array_equal(res.indptr, reference_result.indptr) + def test_normalized_hamming(): hamming_calculator = HammingDistanceCalculator(1, 1, 50, True) seq1 = np.array(["AAAA", "AAB", "AABB", "ABA"]) @@ -680,6 +681,7 @@ def test_normalized_hamming(): assert res.shape == expected_result.shape assert np.array_equal(res.todense(), expected_result) + def test_hamming_histogram(): hamming_calculator = HammingDistanceCalculator(1, 1, 100, True, True) seq1 = np.array(["AAAA", "AA", "AABB", "ABA"]) From 53d542e1dc302c6ae9b2ac8f799fbfd3693612c7 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 9 Aug 2024 19:47:38 +0200 Subject: [PATCH 30/94] refactored --- src/scirpy/ir_dist/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 247ba76b2..4482bc31d 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -564,7 +564,7 @@ def __init__( self.histogram = histogram def _make_histogram(self, row_mins): - if(self.normalize==True): + if(self.normalize): bins = np.arange(0,101,2) else: max_value = np.max(row_mins) From 59a3c9d077542b2d01fb4614ccf847ce70f7b91c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:50:10 +0000 Subject: [PATCH 31/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index def8a59e2..e8373278c 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -563,8 +563,8 @@ def __init__( self.histogram = histogram def _make_histogram(self, row_mins): - if(self.normalize): - bins = np.arange(0,101,2) + if self.normalize: + bins = np.arange(0, 101, 2) else: max_value = np.max(row_mins) bin_step = np.ceil(max_value / 100) From c1d9d514b0afcb44f9e8755fe188fdb7636dd3b3 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 9 Aug 2024 20:57:30 +0200 Subject: [PATCH 32/94] hamming histogram adjustments --- src/scirpy/ir_dist/metrics.py | 1 + src/scirpy/tests/test_ir_dist_metrics.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index def8a59e2..d23c5cfa6 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -627,6 +627,7 @@ def _hamming_mat( cutoff = self.cutoff normalize = self.normalize histogram = self.histogram + is_symmetric *= histogram start_column *= is_symmetric nb.set_num_threads(self.n_jobs) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index bd955266f..2b4afc6cb 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -684,8 +684,7 @@ def test_normalized_hamming(): def test_hamming_histogram(): hamming_calculator = HammingDistanceCalculator(1, 1, 100, True, True) - seq1 = np.array(["AAAA", "AA", "AABB", "ABA"]) - seq2 = np.array(["ABB", "ABBB", "ABBB"]) - expected_result = np.array([75, 100, 25, 33]) - _, _, _, res = hamming_calculator._hamming_mat(seqs=seq1, seqs2=seq2) + seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) + expected_result = np.array([50, 100, 50, 100]) + _, _, _, res = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) assert np.array_equal(res, expected_result) From 5419d0b886f32a5d5fcd66704f5bd89227d6b16b Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sun, 11 Aug 2024 19:08:04 +0200 Subject: [PATCH 33/94] reference test cases added for normalized hamming and hamming histogram --- src/scirpy/tests/test_ir_dist_metrics.py | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 2b4afc6cb..fca24c4f1 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -681,6 +681,35 @@ def test_normalized_hamming(): assert res.shape == expected_result.shape assert np.array_equal(res.todense(), expected_result) +def test_normalized_hamming_reference(): + from . import TESTDATA + from decimal import Decimal, ROUND_HALF_UP + + seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") + hamming_calculator = HammingDistanceCalculator(1, 1, 1000, False) + res = hamming_calculator.calc_dist_mat(seqs, seqs) + + cutoff = 50 + + normalized_hamming_calculator = HammingDistanceCalculator(2, 2, cutoff, True) + res_norm = normalized_hamming_calculator.calc_dist_mat(seqs, seqs) + + lengths = np.array([len(s) for s in seqs]) + + for i in range(len(res.data)): + if res.data[i] > 0: + distance = (res.data[i]-1) / lengths[res.indices[i]] * 100 + 1 + distance = Decimal(distance).quantize(0, ROUND_HALF_UP) + if(distance <= cutoff + 1): + res.data[i] = distance + else: + res.data[i] = 0 + + nz = np.nonzero(res.data) + res.data = res.data[nz] + + assert np.array_equal(res.data, res_norm.data) + def test_hamming_histogram(): hamming_calculator = HammingDistanceCalculator(1, 1, 100, True, True) @@ -688,3 +717,17 @@ def test_hamming_histogram(): expected_result = np.array([50, 100, 50, 100]) _, _, _, res = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) assert np.array_equal(res, expected_result) + +def test_hamming_histogram_reference(): + from . import TESTDATA + seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") + hamming_calculator = HammingDistanceCalculator(2, 2, 100, True, True) + res = hamming_calculator.calc_dist_mat(seqs, seqs) + res_dense = res.todense() + res_dense = np.where(res_dense == 0, 101, res_dense) + res_dense = np.where(res_dense == 1, 101, res_dense) + row_mins_ref = np.min(res_dense, axis=1) - 1 + _, _, _, row_mins = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) + + assert np.array_equal(row_mins_ref, row_mins) + From da07affee803349960133c347cb659e1ab9a5742 Mon Sep 17 00:00:00 2001 From: felixpetschko <48593591+felixpetschko@users.noreply.github.com> Date: Thu, 15 Aug 2024 09:02:43 +0200 Subject: [PATCH 34/94] Update src/scirpy/ir_dist/metrics.py Co-authored-by: Gregor Sturm --- src/scirpy/ir_dist/metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 8e208cf42..ac1f1730f 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -554,6 +554,7 @@ def __init__( n_jobs: int = 1, n_blocks: int = 1, cutoff: int = 2, + *, normalize: bool = False, histogram: bool = False, ): From c0e03812f000530b925ffc17253aa46aae948582 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 10:03:59 +0200 Subject: [PATCH 35/94] test cases for normalized hamming and hamming histogram adapted --- .../hamming_WU3k_histogram_result.npy | Bin 0 -> 12528 bytes .../hamming_WU3k_normalized_csr_result.npz | Bin 0 -> 152600 bytes src/scirpy/tests/test_ir_dist_metrics.py | 49 ++++++------------ 3 files changed, 15 insertions(+), 34 deletions(-) create mode 100644 src/scirpy/tests/data/hamming_test_data/hamming_WU3k_histogram_result.npy create mode 100644 src/scirpy/tests/data/hamming_test_data/hamming_WU3k_normalized_csr_result.npz diff --git a/src/scirpy/tests/data/hamming_test_data/hamming_WU3k_histogram_result.npy b/src/scirpy/tests/data/hamming_test_data/hamming_WU3k_histogram_result.npy new file mode 100644 index 0000000000000000000000000000000000000000..a776acf9e651487242eba4e748ebb144ca75e13d GIT binary patch literal 12528 zcmbW-!HS(#5Cq`k)@R6Kn9ZmRV$@)aqh5bCVoX9nTx2D|OmHDaCgMhf`wqU4k7jPB zzJ}B!2ximt>F(<4>hu5i&P@Kk|HHip58qgStUpg4pFjKQ;^fxR$=Q>eC#OdzPo7@< ze(~tnAD>=4K0ln_e)P+;^XKQ!etz`Z`E&oR>o;z^cl!3}(O*aZ_urfA#lJ6iFWasB zPcHraF6W=V+FxHK{(0HdFMZxg{KMefi67jx#fS6t%l`1tPds&pH z4<4hg4qrd}5$BuRIXu?6doQ1lpFZO1&EXuL_pv&A&K!Hs?Q!<6 zj?FzBz2LZ?`pc&WzT!87uU=do>)bqiaX98=^-2EB)jZ|F6Q@UgZZ3K5%@>&i4xV_$ z-^-(ie0b6Q%Dsn2C;Xh-U+Z!$yyW2%eWp)%%s%O-kNXwc<}7k~IVug{gm zrw2S%UFo~%Y7RVu(|zZ?*ZKPNynI%kIqC0xXf z@h#6SJ>aYJUZO8O@irfPasAx``2F5+zgYc~KRVMzUwHaC$47s>MY_sHioJoC{JU*~xAxqG7zJpK4=>t-(ajg#+#OT9e2%mK$aUh3d_ z@8N-`I{D_5uW#4s3x|Hr<$M0+iSyk9*8auuD{If*Bm8wQ_t8A&2(CPKem~$|(T}AE zerLfqpL4!>@fB~2pL6|Lad^({laARB9TwjBz>!baJ{M={CQls;-#yj0^P}JJ>wD{d zsiU{^@R^^B%`d;Ld*~jSC;aB~5bo}BpTkdIb5`CwkKr+O^yu%Cg|pgMhbNy7?h$?E ziEp`&e0ixG-RJ@z4}AIh(GM=G|5^G*|0~avJ*k7=_XW?~i~0EsPw_kHXAbXueehz< zVLm+M<0Fr+Z~WP>db+rG$)EWfN58_Idte?`eV+IE9%U|h`uo1Z_df7*viqRU9PseV z`8*%Z*K;9XAJ4OUNYCOkKQH28P8KhH`TEJTPyF%f_ahv2c=72X4==p?Ts=OlJnv~f z9JuDu7arSr@ZqS-`RqxZJb3Eq6TRj>%cqC^=Y6;y=6R!YTYc4gF4)|M>4y(}@RAQF z{O7$TpMH3__s-$5^pcOS&)e2X9-e&t)y1FZQXl)ri@tte?KuhV=wg2T;NeFXakzMS z?vgjW#PhsNKX~+jYi?G5R-XQ>xy9R>TVMEm`%{OvzIe)KeU7N3oBhLO)%jinU!2un zUw+s5eXlq@yr1CVr$1ch>e!rL*%!X*`kwXkraxToN9S}T5@veq(Z`b_?7@F)JdxtupIy6Nxll->*a zh3E9yb_bSg-lyB{;I4Td%*E0Rp84XB&UwE~ANsl{^>^(n{kGkK##%nV&DT8qzS%im z&68f9gYS1Q;FtFsyXD-^v+&f@6HfP`f9%YUzx~O(n||VK@@6mQ%liQ?-RAo_UYTQ0 zA3B&t^62Hgqd$wEy39p4xZ}H@J|ESy z`nk_^g6n?N`AYwue|~?}^?1++p1Jh*y@A!I`ND%kkL1n#_7;6T&+NLtTQC068y+7I z7Jv5~FX!et4zB+4>>=^t;d4?PkKkU>Ik@H=+xwE|oNlq6JMX38Lq~O6=3(*0lNG0f zejSfL{r$fbd-6PZUg-x{TtDw2xc1?GCvV;#_QJxE?>(b0TzZ&`RgWK^{+@rl=pmmi i-R*V0_jr%uK?ivDf;V4WAG*5d&h=;E%TM0(z5NU6zfW)g literal 0 HcmV?d00001 diff --git a/src/scirpy/tests/data/hamming_test_data/hamming_WU3k_normalized_csr_result.npz b/src/scirpy/tests/data/hamming_test_data/hamming_WU3k_normalized_csr_result.npz new file mode 100644 index 0000000000000000000000000000000000000000..92f22331853fac029b987f8a040bd28729fd26f6 GIT binary patch literal 152600 zcmW(+2UHVH*H%%nAc#nlW(5&xQbUW1%4-2cq)0E42%#ec2m}S`O+{%TL`CTkF;YVk z=>jPtHMGzp5C{-RfDrik{yk^U%+Ai9b7$wyo#(l8@8gGjhtD23aNyYfec?cmne8w4 zhX)TxoZvrj^1#^xp1#jL-93U-eEq`?9ytB~&HMch{IARZE(Y%SPOy8w5uO6z8oc@P zej$-gp>yCbmy1@z~w_Pm+?hE6O6B z_ac$O%iM8p9+RWUr0;G29KS_j@$xa8Nc5!LV8ehSGi)zT%CAR}tt99W+JR(`xehlg znLBPSVU@8fZkFQXVN6dsWX+h<$g-hN2XciuIPRMs3H=d8U{;PqK3CxG(SGzj=~l+f z{7qK=G1M1Rl*XYvisiA6V``yEQvi%OvF4vxnZ<`8F8?RqRt;jCBnHC1SB0nE0|B+T)jS@R;Y{{n z2Vp9fnEC%Jp+hv<5aW)<&h}tVWZSL~H!ArVN$kc*JHr?RLqzs5?qF229sNQIsL7yj zytLXJAMcuKsvroPEwe8HW{bhW4|M*b&=B?g|k)AsRg=Ky~)ZZFaYlt2T;y5@fl;B4xH> zwe7q2CF0~Py{IXob`aS*`mGEN5K4<%x-X834z`owilOBiy0fFHh$RgbubLEU0H_Gz z2ns<^-9a+kJhUdPnhGImYmN+f>tHm9`8r%IRf)kO40tPIpt%*0ntEz1$R>LX%+|*M zb9rryFiiat4ByZ20VN}p-D@hTFj9?a(yo{FR8F?COHCbB9yF4> z*cW+K47ROP!4~@~f$jTwWrEGy$q8L6OgZbQ=V&-FsP#vh#MO zoFJXAlfMx>8MgO_VFDt&1@FbL?ChNzZ+ZxFrJM$;P$d2xBBcHLgQw7C6W7-}pSF8HS{J2XNrP)8nhMAk;@FPX>c} z1KoWd%8c@7!jHlR(CRSvXULc%#PPELp|X07D)5xUs2}_JOhn_AymSv@=1KZt`m`Aa z7SO?X@KLYjx z?w!LBax0z#r;#AqF+6u_0t@ts+6%#$G*g~YyfY`T6HswY`xb^KhDB6S#~z`N!RH{P zOwE1ezJuwAi5mOs4a1mc%beX|>??)_H=k-wl+}U}%iU3~vKUdKbt_$+t4L0b%;#jw zK-fYUn-O}$UxwGF4VQ=3;v%ThZ2c$~ARVmR&E}YCV1JkUMbTS8bcCa$^{y>Jd}r+k z$mA$3;>k)kC)d!GT{Ka!Wk}bs|K&YDQ9FsLJ*GZ)7GoJB1RP=9!!UDrUFCkLMXRPF zwAjuzKQZJ6w~qRGa_bkxSv-Q*+eD+duuD(F_6O3$FvG|pDxO)dzF7Q`bOak0>i>os z4Dzc{jR+znp|){@rvhFwPXo<^)Xy^6U=Zb@K^q6v>s`4bE#a(_1^yJk_Q#mVYrM|p zQv6jiZ@{qB(lar&X|0n%SH$)-^4?WG6Jl!RWqE{K<$+R+O!9kGNuX*~qbw@6YqBoV zGhs}pDSf?clc$fN<^fTI-oY55>?Plt_{|k1BHn{7g5i&*#G&)g?J&Lmr|O#M;thbf9Z#@jArfxt-M1E!rvRM zje(?ceY|QgR0X1-rhX^mI;NZGs|`C!>B8^DftXq&RL0BwB?_m?MaZ+>(4#lmt8a}X z^jgt!vGgYJ=T$GsFcs$e%^8&Nl+|_lEpW|?1d9YylmKN}5Cg2w)`W$og@Um0n?a8f zZWXI;-^eN_B5sC-f8lyoX-+=vL=zs_(cl|wJaLE1$*cUKMM7EaFQW(*- z<}KBp7@|2c!g!8RMMU;95FnjoRmTJMIO;VJ1mPpMvzQf~&MVxcQV>W;NIumjnuhzu zkZRg9a$&#OH*}6%Gfb|s8(SN5g*c!EOQ*(@j6fk-R7((M1R9Q`p5+P-4%5SrqG)jD z2vJshd0CnWZK=yX;Zl2~cjofQK@o=-;eoU@X9uk^8=EC;jvZB-wox3}W z#JTsZ@7Jf&sk0!04(uSb1x_9zjk^HBk{B!ExMxin6%Hb3!lbuE{+{7(oy@85+Hdj% z31yRf34XGeE2Le8o!#jra8x%V9D{hLFht+|uy5Cas%13pJ#Nnathv;9)`>TYnCDce za$Gz7$K{w-bP=#`@5J<+*dAL9b2sT)7Faf2_j|B(78n~Cwzc)xx9IcN zuKqrcJkWxGA{cEmtx9JtU(y<9Uj23cCB0hV_c+FX>;MlKw; z(+_(%>CmWWb$bz<@}<4g;~Rbc6q^_CL{{+~fxqt>3x?ecEec2Hm`~yG_r@==y1q3> zf}HBZLQ&#zoX)&|__yM`Zx@8ZEkol&qsltDBNs<$?#);|rBO$`5?%p^CdO3q)p$_6 zk^amyZER7o+Pi$itosh>R@cH8H$aYfCUGL|ebmzHM&*OP>(ugdQU!^clDHb`XEI zj2k=kG)CAw%7gu(mD$FCse0BlQbRyl2uDZ=YJVj<4xfw3hCqa|x6 zXT-7|z^jsHBu);YcrJBvS+Q9E;5wm%D9e_xj%S2wOXQ?}G_bio0*WS&t;#ZwfrarN zvUJt76Yz;C|2?oE6rKjRnvS3{OxgOi3s%9a(RXA0yqyKyrIY*)jNIfs@aC(#C)j%v zd9Fp{an!ZKsvPR_!NJfV3;IX+I9hKJWsYl-kD)?o{N zUDwQ!>N6#feH6nO1L^OBO12i}B+2)<#E}(VZ1Rb$3-?w(e~}qThyHW`pJvf|yo<=^ zfJJS9uL_~R(K={YsIF0LbxFGuY+291UO^cKzKO^jT6e?grF-mdH%-`Urf+OP zu0cJhY0==K^odyoTf~xv)JkQ3qR;u8q<+{WISQ?F}(dcR%t` zylaAfhY=ubV+^+C*}XLE8>%ZvAsgw-j>N!-yRO&J3uDbroKsA?(Z?zs+Eph4X1c|V-u)FrvAf3wr30NwwMCU1ZyP26$MHXko_$WFNB zZ&Pf|t+`dL{@PiM=Zrqqoq}Acm}UMuHKki`v^46`2rR4W;dstNC;i+FNA$ZjD_eGz zVYMmg|KNqzww}hQ;{ji(0O4>Zv_y8<>w5 z5;Pigr>|6`$78x1Ak+znxgee$Jz+m-Gim8eL$9py9cpiQ6JhOEQEZV|D=WP`v3J(W zG%5VU&|)<5MASy`yop|nmZJZ&zHh%k&Wygz@`0~%>evSR{x3qo{OZ%Gu$WC{=7LgJ z6Eiheq;9$Ee}L{q4eZ%;Bk;OXxj4S47p+Dcj0N8?|AkD~h7EbgDO%;ZDpz?YsHZ5S zRF8(~R>9HC)%?GSoDi+TnQd%}6oYxudQ(;vhK1al;tOwJ&{e>wprX^Nz;kZ1C6h42 zv-l1|QWIQxdjJA0g)a$ymff~XNG*=}lY#4Jyk4QMI!eMBL?f4dtdg@&QJD5WoCkH!@ms6Lts1oQ zEv+ugxiu9TU*EK|a2J!sE+s`NWY&UL7Jn`JilPLa%@~4pV?UaTF#JR(d4K(>Dom1` z0-WDjurY0DkvfG>l4Nv8zdtx{_hopNn$7E~8oExB0n(3>eYDsoTQRPsVXZ+_1#hq0$liSu*D+G5~e*cQj59fG6acBJP8_A7p5k|FSY==3zPtR-v!(+Iyz zevxv^`=G4s^qJMs!p%Fnh}t)-EtktR-*d$^>*c=->^c107E*>Tb8?L$kMuxU>gwG-eo@7rf*Sq=NZ&G&aq<_|*c7pf7c0})*GH1Aj+KSV58hR^p6 z+T-_@L~S9UPRC*Jg`=Q=qRq&8Gvc|0Tp&As>ynXn{^Hcs^xfExZBx^kXGGis-c||O z4jG=|6W$woXe@1Tx%C#CTq&fkCbp-NCGmT+_B+*iJD6c9J7%8*BG=Ky)l~a!C6Z%m zMFSok+VWBnToz4?RZJta0|VC*IN^=ap{kLlDUeTLzACcsp%e8ptxChKCP2$yvR~GI z=j=7>2#bJ(^y4fxb?>LucPQ${9M~DB$TW73_w<4YwRBvjVU{>au$ICT8m+yN51fMY zJshuh)}Q?|U2$2;w~ z;iJA~MPDudW3x7w%)W)^>b$Xy9R{}79=-P(uAqL06FFl2-4rk+b;SC>j+AW^v%A!!q1ceVk>ph~WpmdsKCb=*@k?Y6qXX%M|2$X$ zx;ntL4iYJPxd_lAKR#cJXfx_Q15(QI87cz}so7Trp>xZAJ!-|^yzp)zqgogeY4@2m zOSo*ktX_K#n(gE6%kOLo=bIb*4RMia+N!O;?R$?*uDZZ%el)uC-ex{xG~=B?(iSez zPuC=VXTEhJ`K^@j$aAAlh6zJc^hosqt7r?Jo z;bq+G*jP%?rwDmA`(L`809*o<(YR!K2_8yIphceFbZXpZ_brjXYP+~S@ClQKVDr=X z?KjJ6dx+7C>qmQHwr;IVtE+6981F*=3msgi4IY&fq~W|*21C=h^5fjuH9Rw;(LWBp zL7b#C2@wjNH$+fOp|!WK7Gq$2!O-{BV#^m7t|5WA0(;7*W!Ys(-MeT-ez1k=H6<0{ z&^zGVF~6EBUb%VBzEcg1p)S!Y-GGbfpg}6dmb|+|=(^P?5@s;hT6t}42wcg2+SogY z66eHMeY6$2-T{7^cSKzy6FjFTzTPB%66%^_y>lceSx9^4v)x`aYa!^#iiv?4#$=^U zZgHSmeeK{thdR8|YOKN9%vG-(`H}F6r-PzggXZQPr6BW(F$g#k*+-VEkI4IstbI;B(cP#Qt!p*Y{r+L>HY?%c_#(c(c;OKn z`wa4hnQBzER{BTPsmPM^tLaS(`f4$kg@mhrmRUB-d7jbR&$!Gr5U0v>< zfA0!p`|UpP0$0j^0!_)H3y;@Jd(8Ko(96#XoN;%aZ46pOJpuCJyjIYE+)XIkbw5h) zTEs^uZ_oqZrpFrXoIHr|aEg!c4Z{@a#YikcuQ#@{^_-ALbtlooJSM)@K-r z$HJ0zOiUx8sjUA#^ zr=l9D-}b>Btm1IRqGj&!KMy}yH%E`zsZNSvMzAd#Mxlp`mw(TSZC=-6n9_g(0haS!Un;b~nI zbM=k~8zixhja{5>H}!ac)wQ`7`FHSfly^IMqEkfUcha3K9fPBH1UvL^!&?C_+p4t0 z)CB|5p3v#K8Ep~A!u~j15TA0$cpH}gyK;ZH4Gzsx8Li`=D&J2!(Lbc%Pl>!uJjmWk9RDtO zB;v9^<-^e;HRFqf`Fzc7!Zg zvk=0j2ilGZU>+o6IiDSZ8sppaL}}8zf;{qV5z5Y2c$<>*_u}x4addIWDU?EF9BSkt ziP5&r3f0y_$Ae$v4Z_!E%Dpe4`0Cl6lRW{%-&_tx!}DhUR+%_qQsdFf}= zqcr`UBjGQM#6;)`tD}gQzjMDgo-IlmauCRZ)vuK0hIje|BuWn=ZQpneDymdDsM|_; zrN7tEfyHYO!};i&j4ywMZyB&R+)``=P}TQ~ro=I3ca(3ZtSve`M!|fue0}7!lotBy(JKBgG&Q1z;X1d7ry~3NHvo;IiF$vV#Yp6 zw2~eFUA%h?VMd?LWOue=9n~9Yrnj$8>Ei})HUqqfqVkROOsE7b$}gCiUJdKzPQm`# zo6H|C8~K6~Z2$!&reSn*R7 z-}0Nl_rE@WVw&0p^#P$f6frwp#FkNmd!`z*t}_QY;dhu zZcI?n_L!gmjh_~?MM|ILetr3MS?WSZlYq<|J$vy$`i5=!+R5R*kpxb{M%Ad?s(sMz z2hf-k^5?*@0NaXW$#fhXiH_~)`OXS09sG{H+G^1!-12J8ufl>A(1Qe@N*Df)uSRm!tf?>_e=XBHqM3X6l` z)ywTrt1)XS;{Q6Tyo{#74o+nnfRaY-5He!Tt}UCaIy_qo6ySVGol~iE@4NBalli0e zmVj5@s7@j6!?ewhbP~51Q04X%fcLOWE6;nF?ix0xFheoF6ZnE{)=nyL4}aW6Y2ET* zrZ+rxJ`Ggwi9i^7qa~NPuYV=r(oU*ReaKAiNbZq(vpo5HzG1XqhTvDsdZK8W#oCT( zP}w6+#Ndxw;J*%RwEp)v>TyP9q!Rjmx83D9)B&lK)hmW+hw{$HefzF*%jltuL>pfY zAv|LC{oCH%Cu_EChYf|!RL6B3ZHiwH6(HM`^1st_)(gtgckjjOybfyLeM6pYBM+xc zmRBs12NyMHdw*J?X_14qzd8K?W3QLK=V>LWDhgVF&mDj?p$XU|{i*LwM!rtKp+L?# z%I4T){k16JFrxtLim>9Px81B@_#HsXOfsYr9cC05K~Xli8OU|M52uXE*6%j>Vjx3wXDgOHZ1_8xxDZcbsnV2nzKEZlMju}wV2+_3HuMJexl*MNCnzGz|KqbMMkLV@A?MJdn&Cw$cf6#N~R=s_T*rlN< zv)W0caccoK|8n}MGyBj}Rx0nnX!lTL-#LXMkwU0x@K|FeV(@4$_e+DzuG2r{3iS2x zk{+uKvu6+3V<5^@B&xwyPzks_{ZI)YuO)JLb2?8IUm`K^H4;)J@mJ1lzGnf;YJ`ww zM8uk~HuIk3QKTaz`;(C&BlVEBV2wW z2vn5S?|mTjA>rPQKJ?~88pB2(mz`D9Bi|^RG=bKa_PjeWvLK=}OZxHyfez|MjgOlkGUzaeJG)PS8kQYzd#k%xIL#=HoehKk>9n4-v}*~|iBXji-4 zvA=E1+ReATM)D)DDID~4x%D>A)q98n-_e0Kw1JpGp_M55f3Eczucyy+bh>)^1lKF@ zwfGct@+iVg0-SahwDEF^Ta5%$KeciG?zsL?`+L@DXk8@=u|-y|2wv9m`fCp?AhH@t zJZfqhog2-`6@?({PQOdw7x7=_RAns$yfj}M2L@TJj=suxy?AuT@Y{Ve-(%IC?m;pW zZLVmk?`mt)-`$0!h1O<%&3+mX;gmg@e#blP7SVS`_AaZ3@?-_}vq$9U(ce^YAm>c4 zsBeLy^vIE+ucF5*Pp;hZy5}pBe=5}EmmcBFI1K|r*D}>>sW!yQg}%IeEuxV}NFucb zM7V*V!s%tC%=R$#k-y&d`~F-bzmQ6*AxI;uUz!9a`D$YO86A^BucI)x2m^CB9R_q6 z(<)v=tmN?cw1v>~@}-$3UgBn@sD?+i3nj*U&zF3^@kPrnyuKh`DtpXLYLocS^NIKy z+<$1H=#<$`fQpdsku>a()Ro<>>u711#VT7?4R`OQXa~f3wk+shG}gZ%q1EBuwv zs8@(RVO(Rj-pDk}OOTt0%#dD=3Hri4XQ=v39H817Gu|tiSGKC574ADcez(5_qGtlP zcBZde+Jz(;&dwoortt+f(-sZW*S=wLr*t*vE85jcwYpsMytsjHX%hfF-Soca8@gZJ zR%B}r`O(W?Il9J4p3ZFVCneqTfiEFEA6Z4h$!`~Su1?S2`MEXHZjSB{%2%Wt*X!#P zHtog#$lYiqesK%Uv$+h9ufI!FTu7Oc)9cT>=x?|jaui?0MJ8?>q>mKaLmuDe#&Gi! zd68xI^RIWl??co+m2hWtuIJ8k^k3sbwz@yi1rE(tmQo{Faz9$@ZUnr79Js_+Dcno9 zaoo+82l^jJcMB=Q7np<)z5Yi90}n*ToHc9vOv)7s&ywsSQ$|`(N))LI!LxEtcfVZ7 z-Ayrba;Gf*LkcG*qL=!sx8`sSV)W%)+D05yMZ1644&0l5UDiz!;=#4qlOupr-9mTo zZyB!36<>0bmSd0B2ibETb!-|a97Q$DMtEn5R8xN4r7XStyLELfWJ2ks(Fy!*C_vGK zzB&PP)3EG{^d5dVISo(R8!F_GHN;1B76OWfjFRk4bTf>0FGj(>>gNt5u-mCZT^k+s zMQ>MaDdF~{4vY0W5`LD3E7>Y$BI%C<6#CD&3OH33FX}iC2Ij{ENCX4i)EDeVFD#eQ ziY=L59r@eIw?`dsPL+766wjg3KTni>+HG|Z@G|YoNs?kb7Fdv-Vj(U2c5)HPUduC8 zU#3<`rvr~Hd$Wd`QUgsAg^9vOLf;PoP2#CSQ+%vCeNpVgs>WyaEfd?((7>B3y?;J! zr&4KpF_`Ic1>C^dg-#Z!(mOydRD5o!pU0SOk+}(cKIp&PiMtzhcKEXS?=aQdyPM;I zZ)ktXKoTN;AW~*sV{<%SFcHHNn3JcNjvHOtODT zCjXAw+-vrHzb-%Y(RByw(f^1WE@tO2`hRj>-Mbv9-Jku~=Xig{r_94anscDO_jQf- z^Z(6gUV0a0q4Om8%2Si64Uca!$7PQHGr9e6ip9HD)UIJeN|yiy2dYO9n92(>a&V!{ zxnV`+(@4U@+5X?3-Sw^~-6D-i4LJ9N>G$)wrkO^)=5)x|v=*<(e(p{+aBFf?OTQ^t z1$~F&DC=(!Rd=N6aDa0*f@eC9oN}m?2t0&5nO|KFj!{*Hz$JQpkZrVjRD0E5wu{#uecp+Xcgnx_APXCq30moYdr*&Ht;)5crbi0d{o?dEWK08Qx zCC%3-Rj#Fmp^V*uh*h&g(`<4VMhcuM_3AXg)cKDEoKi!ZS;X8=bwJS3Q~-OSr^tk< z)pq|%(Zg!KKQa5RJuNkA{Dyx1Z|857S2qq9!#a)1=LQO1Ktdf)p`6pdd{eiONwv_@ z^dAhA-Zsk_=t~HoBVKB~f?geXw;=T$tma-cu3vY51@pTIbZYl$WQmu9qg+JEGXK}3 zq8mj|!Cp3)MeYXvzcJByx~gTPH^JmRG^Y{e6`2?tB%ijb5s`j)vv$qu9W$u5V!`IS%00?`&RBbB86$Wmbb6-hnogdy^+02n z``rq})T;vWhJ!U>TDyJzp3EXwqAO=*)N0LIh$+N1L%U{`S_XxOl)d!AHHjFjz`QPJ zeXEq&sdQ@_h8X@FF&j*4IFX?9+c^gtsH(&ofH{nBli}Zv==M0?E0Gz49d%wKCjfH< zcHO;%p4;!8nKpa^-pQVDHkmc5RNkH2OrX>&O+owU#dA%|WSjKv^giS*xodHd_pGGJ zv0Lg(7sfCsHwR) zC&4~*9x#$A#~V|C@!-QwN5jAUSpk-mud2E@mu)C9AP&~f1 zIBD^_Nn0#9q%{=Xho;ZHTm=rKPH+3H?)2r^3$zsFzd{91kD4qv8T3MyYZqmD3>8gB z)3ZTY>_fV$H@Xb+m`D80wYELe^>+3S!OGD;Ih%SIEHR{-zEe&;yD7?*WNL78sg9ud zAr#@y?#`v(q6~_!OGqI#oe26#!2H|~F+u$xz@n*AUAL6=XO!~@3T)l9{4qLA3vABH z?(+@zI%0N5;q5s+@evBP*V~v2*gsu^Xg{nSZVS(&^2Wjy7C-zL?rwPaj0xi9Q={2I z);H{X8G#rn;%;9$H$gkN^QyE;JNCn_Y`ue?rHNoxdhmkK6O9nR(|e0rcR!T%7s5N+ z4CPoa!S)D!@=y`S7QFGKvEnH)xeYticx`H2I&qkOk{9l49%StDQtn#m&L4Vgl7*j$ z7eYQ|nECc_T)V+LSWUuxW1&>QrSRTd4KKW|i3La6pQaVH6Ddr&peg{ktlOyWbakPG z8g6BlXm_pS)LqpgMf7Fs{6)G$Z%McwgYLqg&7{ADv^ber?#X{UC|J&Y=n)$Xd@`Un z@sIBBn{Po;4wO0#Lpb_n%EQm+j+mZ% z>;pZQ`omuD+pRxem)2LEBH#)($tf7HUZIIm1(9x{DCfxL2ZyGe0DM^ZS79`T{0f() z+<9P3p>QJEJK=p!(&n+Lp!>!GY_Brhak@Jo-8tnP;E6^)OOrLCy+h z&U;Px_T`LTWbQ{BX0EzZb%V+c`amVGTC4FNl7G%hvsH^}_J;(;h| zuBL1qGkWXZgG+CBeeD3{0ryj`W+GbWI%!hNGU0MFk1n9%3U7oNRm7#lx(%qPd4*OsUllk~g6)jimFkOuUiWvwKStHYjV9sj4l1@EQ4;{*g* z^t(l)zSj^9IGNLix%9W$Mfd_@}t$xbPZ z-7*q~8g{S^3KZhPVDk?JKB4%at}})chubOrI&MTV-)T_nCJTZjC(NC)MX@P_FYEki1lBQqJ{Q@mYtg zcrpWb1q^;7yPCF6FJE|Ubgu5j-m%q`-II-)*s-HW z<>oFhN&c=`%fzIUA@r_tyu30W(N%c>y)K}Z-2YXH0aZ3#Pg!ILPd5f$-bjH}K3_;d z&ym9rtg)9hga7)u=EM2mInNRngYpB|5!AIcOGVl}-Er=>>+;l!MnnVdcxgu$d}&C{ zGfxSy`UN@cd?fE4QDtsDs}9CzkjV^*tH16v=KYzSE7+8xy|K zNrq*(g{ zhV>E+NR)<-@IdHG6WZz+XgN)*kwW^E+S7*}#AOz&uP6)?(n)`~_)p|s7_UPrM$HMBc`SfCjJSX=O%wKV=&OaX zSwZH*#%7T=%Xj|lt_8lwdiQ!}p-_GySgO$F^e`)^zfE)L*releSlrH)YSq^`Z@1G8 zHd-75K-4V#c6RUi^jknlo*(hecYgeX&;h0ZfHbUvZo(fS^#Uc=5R?aL)8K}=B_VP! z$(zC*IFy(;_Rg$7c{iYF;TEA8-`nR~vGM&qbz$s8trLwQHP+HVVUbBJCJ01bZ{OQ~BY3X=0 zA6VTq7;4RPIz-uMu6OJl-&z6@u9fc2JZkEY>#5L${jq5*w;}yju`Aq4D-Ia(m$eYl z-j5x->6EbP8wD3{_2hEIvq^4yV;df%c!!z{`khKzS?D`wP3+Gsx=C8g21f!mqE}fG zhsfUyf3R3Nuks|Td@1sIg|-TOxhlxHKv0KNY&;Ewvh&0^1JhIW*fQP2 z0QuoET9w31oJDUQj;jrL?2Rr+7ga=p(=|3DwHy4JAV3l5ixxS{(OaL{?QnePoU-yqvB=bBb88m4!VHtIx=F@N_b7>IMo-H(pY=fHzNR^A~E=nWU9bD`q zs*l^7x|Zh2T&_GZRxeSr9pf8ea>GqQ&6-$lew3WsY|9h4=yt>3n%KeziSz)Z)b6u$ zzMwWU(88^MI|g)7!=sCI(bw2T)u2&>exSIP-4RgZ91}@)A|+M9NA`wEGgdtj?VrFsDt!P^gE*Z zrjh5SVg{!jFV28Fz{~v1xP{A4oL#{6C%JKXg3trJBMVoj1PyX^_rk}mz^jPB0jr>Q zu zT~GX=&-bV+$m4^8$n!Bxzmm4T%Eje$+=_gfv-KV)s>a)yHAPWJPy`gvl8+M z?AriUtteOwixS#6y73b6^QD%yhAh`MFR5kJuc9h@hXB4kdWGMvo-zbE1wBe$ySI$H z5!I-?s#0$}%-oZ&WeJ3VXI=*SX6iRhLP`K870(G7yq)D0JaTr{5m%;D7X-EmvxzVd z)q(O8q(?5KHQui$Hk8Rwa(B9dqW$o?29^x|K_2|KJ^LhnlV3QB5%e)fqrR&SkVIQ8 zg5{b06J-8d1^idqQ5M$m)5&4_74~3`v%$F3@-*3Aa6#tlT9WydwRlwg=q+x|ka&XZ z8>Zpc?Nj){o2W`$tgZ7w!&nEo3q}21@a9uvrKr=rz6HKgp+*zeZ#zGLmcQ>VWd(Jm z-ULpLI~iK?$bC-wM;QHqsigY8%D-A#2^ zXgF9D9U{MR+iLn$)mw>*DQm*~z^1n#BGq$g?q!JO#uMNf5y_K42f;um_w8@uA7X}v z`c=%H44%y`)L2(?v~2A%Cgn2+hZiQ$OHZF_MEA9F8ebw!EX~5bL5~BphysOke?OF& zjZ67!6AR|A=aPjBK|KahGJ5CrG75uU;6wNXO;oZMlGATwWd2R2JDgiIn1GH>9~=|z z3oBkNzQ6#BH^s_%z=az_`!(qTmjZx+v+CUmfO|;S)}4!BBQs5yv3lMmv}2#{@1`kW zduL*2qF;tc$IcxU1$p_uZ{lY{nYnw{WY_AQ@@YmrJ6XI;huv=3hiD)%2spf-trJgj z64*>!$Ve$u=!i(nam5Y5a;4jJ%=);XK;bTeYlSbm21^dsa#Sgx1y|{1szx!j;o_Yx#+iZYs zT$3po+2_bM&}bLAW5M6*MBZ$lvH9?^{jhH1-gPBG=50{fx{?YLk-TZ(GJ+)b!i-${ zsILmCK%mqoZfUT#OZ>%(`jm*I+}K}V4e4^B>%iON#%YAL&3E>XvvL{(k@iLSN$BNF zjW``ot=cx6?WN7_^J1)LmqM=-2l+EP`&}IUZEKg`eKI^m!HWdgJw!M@HME z3+|76*7+!R>E(Nf?F(68;^{i+qux`HELRCMlQxY2#lPvVrYD$fyL?@1MA5tZ02j%SA2~??(xZ zN{wR!{Z*a#rXsz@I&vvYyZbmf=MmSemW#ZmNI@sR!`0Hz=dZ>0HErc${mwI2rj{`GziF&=BrJ%t}$Obf*y-Gh+*L&=CpJ_yp=yIa?Ka`tjvxV4j$Zy| z-XEja_dtm4X6D;n^8pFPG^NFq8m*(D9;)$reB0ALUa6zkm)1*_cSnNndO7_ENj{rp z4@>b`NNO=Q6;Ae;Zm&vg|K6G4oa9_MB_7bFt>h^BJ2^cZT0BOx9k6R= z{-E(RjpBLv)eJiRMMxbF88zYfi-|QFY`oh*_=ckZajn+VOJqL=@*;rO`jfZoHw|eg9gDLDdC41y;wyQJ8da*>SqZ9^a)t1+igr(_ zaXIdt(%0vv_ik~Xy00B-dtoOYUneoz+T@w5E~z6}wEelDb<$Qq)&}b~YFoxL?-x(r zxan@UUX}fDe7ZP${CVrx2eZJW-i+K!iKNH8niPnlCE@n8TD=kHV*b!G?n8^EuXR5} z2GbOT8W{oufP{mrSY7|Fmq|zZ)E~4RUg*vkT;{CY)<3Z!4F%>q_DMt8^i3I52D7ul zH_r&^6;bG=*LS9T{`a@%KH`gGv+I{Es~yUn=@0-7{jmRKl@W3v@E^s~oIDupWcf6> zbU=V9fvYpmO`R4 zw#tU$QZ(nO3!J8dbTu>}b#k?51IC1-C|M`L<3Z7ea@lliPG@^)lMkQ?=|nmxzo*l| zYm>Z87YC==UHm0{Z*uP%M;zz$;>+tj*yS~%VoZZ=kV0UrZQ7@Ewx-qLX&STDfZ>@F zK3aOiMDvc9mGYJ}uRObmzKBoCQP6Par@zwYJLY}N>Qk{;is&b-4{!Yo^N|dHP_-qo{+6tq3p0su z*288`o13#8Q~?(YR=%z7cFd-30?x5RU)i^63I}I`F6IeS#Cwbc=@16Nx5DOu*^u%D zW`x;P#agrGa{KyB37>W)T+vMfb{N3>`ys^BtoB2mO23M!XXJbe8TTG~!N@!=L@}h< zl5kdkQp*fh9;qx|;NEcC2jm%;`$3URe%|ri3$#_2*PAcxK?YqMo&Y#zSWLalP^oLD zhuO8xp4HB8ZqqWtXj429FMRTIaj)5YU_akeo4Oz>pd&4Gw=RZQf|FV=voLq#4UzM}g5^Y*K6m?Qsx!>ZE=hZ-h+{?38 z#=;E_e;!{JN|wIymSq%+j{Z)7Jmr^ko@uRT$8>S##&wE>#>y!HHenFXjacRxpJt~?i1&Q_V$qr%C5D2}tB z%Ci|Rl`#e;^orsf!wP4TW7y6I=o=_ssFwiUO*R#?o9~}*m{wZ9>JTUvuv^@c#)7)& zRPu3$osih6NJ}B)w!98|?c8&Xk1edRTpRdT=^0yT#{s~V0K2)wq`_grxniAnu&rEH z!k1G0&l|6XS?@366k@ir(Z)?^Qvc`H*YDB;z$t;l!!Il^nXu}rD3okO+uDz_mC|MQ ziHeti(}Hu_;r-D=R)`X4`@E_89JkUL4y%`Lf%jY>#v$+cYL z6S-d&!(5WwL#14IQOYHkr8Nw5$^EjeGWX5g=el7SX8e2~kKcdi{PF(pJkI;}dOlxs z&zryGW^knr?(vTPkH17jyb1AgpJ<^?bSaLv}&CahCggQ>KKiB z_8AsEP-Xv9v)uhIUut$=qj%HvAz+kx*D;+OI$Krz0}wpoF1aPYuZ>;0Hm?0Eq1(w0 z1YP|%w)!XB!h;v$-f8X_7me}9QTaBHYAv*xbeSO4V9JZVg^&d26WE0bwb9NVdRJ>S za51d?cEk1)cSjo+A1^q@VB6nsHwP;frkbYQ#4^V;y0PhxpSnyhWAdxYeJ%z+9HXQ3 zmBpb;yRlX&5j^b0uxwok`dP+F`Wt%8dOb2wfnRZ|9vRGXk64b%b5@g4+v1_=9ufv% z8QLT^S2-{Ip-ncIfDxzt>5Xqgo#JOOLW{*Y<}!;{CN7pIKH>>?YW#QsiSn^0wyB&O zlq^@?|F_yGpq#)KEM+wW&N*3t8DmX8FDg36F)_q_Dg9JXHL;g<+hci14H$`x%oY)f z<1p*BZ6rr94qg|{s;E_up5WvBY;Dz^K_+!aiS|9qQ67n}hMvU)*rG9gSvAO-rbD0B>(6l)N8uzJBmCugOWP2KiJ`}FxQ*|4RfNoB0_`c@4$%Q$TgS)0yobUEjr zE-RPkc9K_8eRSluf+=!p@1Imju?KX;w$snB2bPC#D(QOr-q$VPlNrFi#m}pmcLYz^ z!${=p@aFql?YC?r5E&Vtu$&*7p*}Z11oabN^eG1Qcbd+juu(t6bu?4j%;DXSq|D|j z+61u2vS8AUCX~v<=a_Y8)r^T5siVLca zQ0S&Ct@1TmpkFrW3Pwnqo3(W<1YHPCfyKJF0#-dQc6{;!^J~0k>GjkHL@*i$XX=Yw zGH^>!K?Za%0U`~hCSoF~8NE7s^#_blM^!6JFJ)h^P)ExJHSJ2A3F^{=9( z*S!Xua&-b^Jqb z=0Qf=v}C4V6c^QIXMRSCPro!x@L#;IZl#b40f0jL3x>G*qO2(-dA#-F*A9=X2!Z{7 z>@G4dC7iOLX4Kgb51BePsRVNijs6=gQlPgqQ){&ZgI!sKK@Ri(+_-y-up36FJjFmQ zoLiovyEGSuEwkazCP|m-HVIRbA(Rs4>EeVUI&5-eCY%Hku7Vq@VZljgqy2Y?dav1w zE=f8!D^JlDO9P9xjGTP*BAlVRz4@Sl($IHY;M)J^WR!j~w!S<4XOdg-;*cU=d+u~2 zXv@GIFujeJYYQT@vPWyu5M1DTd!VOcWMf+hVSqY(w*OPvw0r$jclkihQc%a?HfKcS zx0GBW&)PwPKdd*iU*}K5E3J(b++Asln?}u1o18D!(ZhyA{F|3(UO5V1Jnk!Bv(1lr z9Go3U z?>~+tFJS907yT{Wb@02r2y#)@1i$Yu8G>I?-FM1&x}qIGy{*ThBH&E$)-=gK6JVFI zn^w0a2+ubOB9D>+uu_GBaHNdE88n{k=a0% z&FP~U%*TUc!pa$VL=EleA!vg$UNQbD(mh^HRZAb=?8HY0Y(0*tHDW+Zu!ud`NwAlU zGmRUd9M%|lfdWa_y~l0a>4X5+IcU3gc9+(Dbb-o_I&Mdbl?rlC# zJBTVMXVY9hMEZ=p^MQMB!CYUJSdGqWL8C>ea(EUMOvt2^r za?8a;=uh?GkG=2|mzl|q+ejlyK77g#>QpVEwx(DtAUO73RczO{hvHvGypRfQQ{gn! z4tX3yoI+@*zdtJwu#{08i}+{wuYQ_#MHND0HYgU4PCtTfr_<2nY^|drawdFo5q_6%T=@Z=^`JO!;i%q3KP5%t z6(C}{)wvfzF~?nZX$205;U4O90Y>F;DI!1YyRg!Vo%vP@*$JZhZ`q70?%APvp!O>F zK-r1G<{f6W8m-OsZ-(8?SoGYVc&>yyW};JNRRHkDQ-BY@qNv-q=ngV5`xxtTBBD-k zV)hz}*>efM5(!w<&SlMo!Mk*5{IGh9H{|5TJ)t!UO=OdfT1tsvX+_>%Tn>M_{frSK zaB|;{)=#1>j+4#XsEap8Z?y~sdx}8jiz19SidVly2V9PtTPN7jN6VnbBHViO><1Ry zNV|_Aip$WvRrfac$(olmz}~rC12&I8`Sqj{%8~?iBHp-eE=gZiG-yw?QdGb-Nz}7b zpo^QA0*$(3+{(GOSt4x9d%>(mzP+q0*|f}!yrHRB=#!R3I45C>PkiQePn zPuTUxTM^^#p$dEX@KQd!u7Bw3k!<`0G>c`~5ChdcyrCg24|yo(Rd_bh)Se5=*nqp2 zBqGedVRSH+=fU>dfZ3A8z3+rdUFp5U=u|XohmZ19$JqS}*siG`wimb<`GS9mmDBH1 znl-sUTY&gXzh$HZebh9l+IBN--J&S;#=f-5or#!(BSCiR9Gjyw& zxS5%FS7~lZbzWyF`7gLGQmK?2FBH(isPfA6StNf9{?+VWFnAtJR8Pr+M);{}LG_)f zWIB8b-mX=0nDafow)A0jIoOdT3|-q1Z$p7w)nqj{L6h}#n?ayWBudCmSiA@<{3*JB z%7B7&LW`1#McU?M*;|Xw)g*q~i4ea~O^cU$4(7exjKD2+P2NiYMw$0!Ra&LsYDabb z$SVU4FP+aBlKphwvIU|3_*v)bBDk-rlfts?f>~+g{Fj!1-*&+$^y2j}Go3@gvoqj> zvQ9e6_j|ETy^@W^4MvC%r|x5n?cMW+OnhNNbMI_7@=WgAkV-pT}-`;BiAq2AJa zkWo+T;OJfUjt+~5B?*c{pARlwN%2NjY$>?2To~ITGtqkI=^Od=>K19BUYR%SuJJ33 zOVFHs^QbH2LZeHNlMrrBA7&ZNa6MEP#mLfDNKk96<|pizxnyS=x*`MC*PudED1D;# zzTr{S^XoDjz{!PNo@z?Jj`x0XeVEZ&au+M>BtV9BDe&N;=XQ`1f!FgMYnfx__I#N_ z3tbG(c-?p~Y<^-KSq(jnEqu4;rzAprY4&?_7Lv9-~&F}y7y#7 z>5c!Q6K7*TtVvV4o>QQj$XWYy`Zq%Fg;!8E@HWkS#YS6*#JwIiJ$Rn_O2%1!Ee)5c zVJ{nQFzps!IPvD1xuv(^Dba$d7(d97j;ZPUKN=+&mH3;PRN-M;i2--$HOBy_Wb`$l za>6ub{BCf=E0FK%>_4Km}kVIbD!q_R60R*g@v0A&?bQLY^4c>BW3@j9+&Rg&{V1% zt`}8Q?(%788ar7G)>Z*b_2QDp`F^PA)f{)^MVqvR>sgbaeLwU%mYr@d%IT#?uvVhkQRbb1cg*)G;ImY%&*eY( z;K|m0>BIb4)oi<8P1)1hzbNscQJx!7FEX* z+qj^g+n$;2pkq8Q9i&kX)>zOwqL9!27hhcgZ1osl z61n!RLhpK$ZhKj$&Ld8bXeXj>yJvv|?uz{*P@Hgu-7v!{s@>{>bkz)bT2!JAcYx@x z0Mt62>O)_1CY|?g25aWfI&KRZ;uNk??rqV3g|1^BKScVghuhVvL3?y0 zY|#9i9Z_Vg_JVWFGwisl(^qlz86T^{4%$!FuUa4H)^thriYPZ$8ttMj>qw1$gWa-! z+?U0J0PkuCOZTw(gF>o>R1p5$E84X|(CKs@@&!>p-qzk{kYWhWnvN?vrT1~1xIv&~T3#!1WR zKprv7DN6>>$#(b6p$CtI@|GUd#_k&WU< z*xB9PYv!I0MN04p^BE*!=_@E|CV%wsla%2^QWngdvWLszgxr!W`-h3InSE7}yi7cK zXvEOv<~PN`&@Fz>v^6zM5_Fm{uI}|NDE+H(>TkxJZ|alI(4YviczDuQw!0bxD|Q~W zaK=|jL;n_Nqi5L1Rx#zHvzX`ooX#I8*uO%=z!s}7Q9$}%u_l%At!`H(G1g_{|qjAU+Tf0gp4JDWKcG8mdxz!|;qP=z~{&ak&mM;6OK+yzg? zu2ChKy=FRGM}g!+ei zhYm3R-YzbfduXaWMl9Lg=|5H}_H{}s-AAZ_^x zu^k)eZYh07fJtv@UrH*C#bNu`P&*>FCyMbge1F>{Z{=l{4$Y; zc@=e@MZR{_6Zfz^amanzu1fFtrS9-3u;x?MF zF?mG!5c4nf1cJ1hDEk)~EPu4oyO^iCKchT<*ws09ncWr{!2Q={WWf(8B~WD{;a6wA z5ryNl7y8V)pSzn+d#;tR3e=76-FB6so8yx5}dqFi3nmO<}{y_Rc z(Hsv#9r9{g%+F?!ebTdrf$`k-?@^FX++KG3p-f;T>SX?VowAndKKvEwJg{*3#GSwjJSMp12Npg`$cCYs~~S4|~3oIgum-PVvldxwy>yy7UXDY3rvrH)1-ycXd3=nmf)`i_+yPvGuEO zU@@JO=?>=I4&>J{6|n5F~z2(#9f`#y2M3Jj00RMjjPJ( zTED(aacQx5f4`W4iRZ1|GM9yK&~k_YVgJ55c=W}?5)Xb%u+m8gR~|skRIEcw(<1X}ao)ZabhQo++vjXtvHN1#vk_<#_maT*xURj6|#?AIuyM z$gkRY(taBZM(5}!P3AWfIOfe{nD6Q7jR+xF+KK8@W~ZA3--&wi%DI$@l5R|fW53J- zCdCdHJ~2wc=;7}JB{Y3v0urnHQ=$_K_H^4BpNI^h1Aid@eKq^{)!;XTAbl#x?|{Hz zZ!xqJ+>f=+)n5sXvTYiaTHBHHI1ADXYdRE5XmtwM`mgREo{;dGui}zn@Aq7muRW#* zyk+$^oP#C-S*-r&c?V@=D!fmZDnnLFsI7BSEuR%iEso}S)=#hMCdsaa2u;g_4byVp zesaz(O&fpE@B9;%)pZrx4WwPgjwV78#(zCu#w%-u!0)1$0pA%ByBx zS5t^?5kSv9<`K9tfC`DrnnC z2V)(q^y;oTXL@G!{2itH589jIKadF?6J<3*m}u=l8+hTd>hHDQyjE#mMS>1jiWu1W z<>cAjXY=u~;5X5y=f*$AnMc~NomAl=Q{`>;fTfPctf8kogWt86NcKGA=DwfE<F-lQkJ7ndfXm&-NzdtOSrTp2v3!2pB`nTf`-hY z`3kN}ErS)>oiryYSAQu3&H*!w~VD*L_9} z1e{6kJBNAzoc=@sk9Wa~r)P5UNG*$Kk$H4*r5o`@OCR1!dBGW8wr0=bZRd)jS$^CoTz-BIus&ZDRGJGe&^y_Whh8T1M1s~txJ zGiycu+nJ!M9{#CObK+6z+na|zm$Y5a*Lp^RCM<0~@jdkJb2{mW!bu&3t$w-}lH&&K z|M{Epr_c&Flsw-s_v=jO-_T9W(O(x8zO40#AKiUlicJU#DK9P#{X8h)_d^R_>R0B} zmFMm#^?{^RZW;0*he-SNAX4hXKe@XEhPttR{;z?{chl}UeUQR$sj#0m{ZqgS5ZiSb zp7_R*(q9h>^p8fhxa4sPB+F;_&=e!Y4Ww7Tl)G{^$IWlk>Cw_cdX>&9e1+-j-sF=N zoP!<+4Sj``i(rz_%=#;wy;t6@GaDVjIA{~HlgR%yVn>%}Ah*k&zenuB#&&7$xg{n$ z_vpI8N60LDUv*pX#}SJiIT`Qq%ccWU?iKhciAlK7B+Z318~2aCm}YRMd9 za1Jp{@iiPY9pO!=pok&Pc`++L*?ndHD19@B1UAI@|nPuagsJ;wFL`6GY)gx9GP1VU(3~H({DrJhkkCmyhD%Y zyOB(f`cZ1GTg?6)v`h|;xIgaU;i6XGyZnN!rZ$WPbnLTecxk$I1TjB<#cfMx+msCr z&>Z`0=XIym)5*owQ&Z4Xoyt^6h=BPzrT$k~`1k{?nl-3&M(+lmz z2U)QLtqBf-RKKNL1c(VfFjL@R`q;e|i}t4fHwuhv6yz&L6{#y`2pa(M4!I=4x zp%J8@(V5kH?<1HyM*rqi$J=i zoOl_7Z}IzcbJ1gjz2Kok8!VONm_r}&MaM;x;K7B3#vGw46b+q<%+7VEoCKDRog@fi+$7VMBW-U`I|457*lW(WK+LOiG zEBLXh7PS)+%PJu16)Vmmq4}P4BwE*{DMohdtrc5%V}!7P@bS&Kymo9md#CrAB27b^ zFr0jZ<1w!>7L#zhCTK2JWCz#nw^?IFgm3?;g}w(#fBZ)oWZmVPES$mpqNFafoag%r z8;gZ6K?>5OHK{TXN8f=d-E9i;PgAVC=4NfI;C@|6)jiiY%+2eOXKN?MR){h6fXC05jMXZ?aIaz^|N(QS(mgMEo_>B-+7ezft7N&YMn7l3*US05YBMZyntCk#1SG-D?rNA-%>{MO z>l!uR$=3*y?_7!`fq`G$9B&cReiJUQ7QV}`nWVORoYAwmB}9fTPX8r~Eb_gT~MIoBI7T*#uWygT;H6uQ|?XIzjIhQRObwq0^| z_jZ~F9P{((Et2-lr@X|Hb*l5tmp)fxA^8-r-y#tft+8{dk=hd>gsmmEgb-mvPYekh z!XxDGV)}*1tJu1nou-lHWOGdavjqQk)f3}CBWm*Suc>>skasR7dMBN4uAW~u4}0F! zfpi}RpN9CS0<26<(@utn|3WmnRU6hU{yiGgFMV@-$a`|6urvH4hl0w9>R)CV#Am#^ zp$94chnOO^I)6*WSDw~mTX|%^yur^#opp#9pv%o4Q>2uI{_-6u3$>LVEZc}}w(8s$ zUWVT{JHEdo()+M!Z}8{8@bJp=(EA-5;$0%?c*-}eA+u4w=6}G9oL#=QilmAlJD#a4 zsv3bmlvive-C;cIZyC${3`UX{=wgu@BylLR7SuK*Dg(WS7Vd4KNU54{-HT1F04_VE zv|vwef1Q0z^lCMcOILT%B%XNj@3 zr#B{R`>QCwZluU`f`%wh0poW4N|u|9I>?RcgPWZwSypOT1GW2Q=)~xjPAh;0(H_vf z6NszZGA(*p2K?ygIP+O%*)|6h3OyL~X+Z@m5B|41nAqNQ zDVs~FjNSFPc3G=7%$L?W3%?r~`Xx+j?_s}xqqXxC^ZoOPR+vggGTTYIR4xB^s1+HN z*qVDe`YV*H!1!;Q*d>=F$In5K<6mp!F&6K%J@cb^c41l;K-ys}|$jvtwFnBa=K z(zcRwAf;XmFZ{-!XivvXLS#;@$tQ*cvTjbDsp4-FYu9;Ueyw{!C_hmwuxgF%^PESlQ*l~Z|WQ^mxvD8Xl21EzfF&TVUZIaFL-3nRTQ z|6AGterMbgke7+ge*VR=&r&O=-$&C#f^yKkEEuLYM!HduVT0f5o^g<-N1W=x_AOkX zI+D=pY7hLeg)@oYjoUs6-d}cb0;_&zc`sB!zp9}!?{pr^@#!uTl4J_~N3|lE&n1G5o77%M<5^l( zc|IFKf`6R~{dsn#NlsRJMcUHQ#rMr)DvY6gx(`_lVK;F6^EEad!LLPF%oKwB*c( zM`})MBlb$Eb+DbAfpT-d_H`zq1t_D;ps(;wtIZ4Eib>FO=jZs=vCDQTUb+fmDC8u~Bi-w9H`R0O7EJhr$7)1D*vhmYh!#m~MRYNgy z(WZD!*N(OnnQJ#1!f=6!uM+8(AoIh48DTL=iNTB`1lcNC*LyHMwxd>dDKs=@CjFa= zInGmh%z9s1m9SK=PB1ooz?dqc>t3h%biC8N%8j)FB!Q%b=+_7;gtG(!AQ#NM?-^*P zMZoh08~%a1W_WSBa<{W!rujkL&=~OHTV}AAwc1cW?e^>s`c?14^ z`DCZ0Co-x!qst<)0*0_$@u@yXgLcJ&tg#ZPeZGg5;a@=8{?Xu5ZsPJYGvSm8c}2m2 zgw!?uW=~@;Zz%lPI=f>)j9Xd0-ROv2k)pygA*alu9sXHaSH#uMQ;&3y%;|K@Qm!Kq zO9!2i-`4|hS$ERU6olVR$tY{Q5Pr|QNUojO?wCj`y1|bU1x7r$W`4(CI=|tSebd$W z_PFdZ#t>M1`HU>%59}D`9|*n`ZuDL6&G=a;`A_S5ls|RaFhBw=EQmKb8{jtj9X8Xj4ejonA>v5QY8IepQf|y;FP8f0n(bv_Bvg$@87vJB(^L24u_q zMmIe=v`{ih!8ty+TuMgNx2RSm$7dap^J$Ecvy5}JQZ+(YWtAhQU917HDcgwzNHCWY>H{AWGJVrH--us1KXcaCzn$AVYVT90oM)t16kxnSc1-K zihTKRM){tDLb~18m-^WYgLw?;vempVs*(4?vDcz3r%OsyAJbg$sfy$G;J8yJ+-;VeGbhK&p>9?}e=`G{>-ZGKCA`L?ccZknm@Mga`o?!Mk zIEo_gTdAm|d5$kBBP21X&T^5D9B;YX=)QP)N zNjdw+*tjYHH^rvUS$R%u>%DGU#lO_6j_H*PKg_`g4Q%z>)`g+{~fL|eo zleFvRG|vklqik%Ll0avqIyi`!Y?*H7Tl5|sB=Dv|0#GwIR_f^BDGpG;!}8SvC#K95 z_fZ7I132h=^}WhLbNqnWuR(tYcj{wnpv|K4_i8cB1xq<7p3ZEgq%K^&Uagga7{41# z9i`Xd*Peo(j+aBa16Do?cM7vi0hMi+hm>x>kD{{Y0Vro`{!^O+7Hmw9W{F(8yAF%0 zIP{0ww5y<`o}Rwu54Bnmz@H%H`auQj(2rD)P>Wym#<3*B7C#)5Sal5R%jS~l3YrpD z{xi-E8M`C$8$1J6d!dQ|hLQ^yRlaX-9N4w|ePV=;Cd}59Pe1uF6*Vv%|wUbfitMhJJQoC`aCEwqR%{<45{CS9p_*hRPGJmAbJoIQ&T%&@@A*&@a z%!qmJ`ACj5{brAM8&2YNDv>0eX(t|Y*{h&Coo7TCg~jweG3>Ss)_Lb9J{K8)g@k4_ zlwt2Vw*Y?k&PL%bGJKzOb14Zh0+XXR8#bP-amwAQKaB`e@=!FIYx4UJk5}@e^_x1_ z2|>HV#Hrhh#WCG;k>hW1Er&N){lPNFSPey?0$(G=B6E_io=p2W zA$Ys#FF65<)a;(oIAg5Kq`V`Rc!+G)(UX9htAZu!hSI&S*B!0FzA|g^Ve&>VdBFO8 zMvqf@sqe-A+eZA6EgZ~V%$Q_b#=ISMw@Pix{nXKTyzHW8xe88Cu5UVYTV=7>Dt5 zQ|F-e7t;n!Jbrg(oj7T}KXW=X2K3C(sj}a%el4g#8X2B)mrsAgXYXbs zqkrDthyqnz1nNcrdQd%UHY2^V0qUI???Sl560=z*eE$=z{tIJzGbB0F`JdF_NE)6K zM_ute2#HDXzFIVQ5R3g{@pZReRqB;!Mp?y$=H1x>dF5TR=KtIyj!evUY={5?DzuMG zK%M@gJX~HPb=FVMzFW2@ftBZ5j|t9@&(O+3w&KxIJ=gw%7`g)8E179{+z)C09(_Z2 zxGeJcX3!Wmy9k8oQq`i}BD_tdTJ-K@6-i+P;(>2q?~9@^ed;fZzO6Kl_V+|=rL)g* zD#Hor0wO4TRWCH_$%VeEy>BI|22)xo&mSiq4ZBp;mXR%t$SKP#0L9q$MkLplhXydfThTtSIZ@ zTe~Und&^B~4z%FHV}!9)<7cMEa`&yKX0qtgH+i!xp@YM`Xu0?-Joq*|c((9e;_Jv) zjzZq8v8i7;qT5EZ7m*t9$R~uvou{AE1ehFkoM7=Lca_9A549;)09#kSt1+RIiOTs` z@~n}RfXeANdQld>tbT5E{%>0Z-!;WImTPc(-A(G0;w_eUt5e^me9{R&dhkzJ<1No0 z9e?c+yiv2{%J0W6CrTt{&u@MT5ahoU+H}*N#5Sx^{vL%c79KEfVy1Wt@UnKF(ax-1 zr%cwE6;F{R$HZX9<^u%?u;BFApQg?qm6BBMeJ2*{Z*d%?UqCm%eiwMa-5^1qohDeU z`!!>@oIC3LGk(sUx`$qo@j%h-pw@>FLG=M6tF0+~UqU#J98G^3bI1E8$+j$HG2%3a z#1%Yb6~Wc8gf^gues`YY9QHOApZk*qMX|{uO9Pnq>x)4=-QSF)-iRk{3i!y7B}f`i z`zMqlFOj(hRmvS6p$nY3Ae_GaRb8mAyLY^xnw~zwho0N9HI{j!m9xc;_YD1SHAM9$ zJKaCcMOh|XY1OQ)WXG{Y*xWaKdO~npGD&fo4Q>L>K}*gov7f^h?qZ{&#AwBYP&!t9_$Xi=U87U@EQ*rxWou5J6e0w`KoKS6bXSu1-B#&avKjRC?o#bGHfQd(2yk+4ocR zhvDz8m=zkxiR2kKx}cmVw4f#{NMLZ_A3jJ&} zqEGeU!EMP7$0ebGIDCN<#M5DESl0!@-7HegUD`4P! zE;yaOxG%~*LBB%`9m6Nk8{9h<2xf)a)Q0*@NNXp|ncsW(T@6P8i1d%Qq`Pg^`(rH5 z)csalI?ufPdb@)PCW}1q+q0LJ5~F*LAV(9* z5vfUos=ZvoLe{(Y`s!o2cPU~+fuThY_eQM<0$WerY+u1{wJFlkKv1N@YWI%^jM4oG zSbbbgv=>w znEq-EEcO}E22(D;2e_fqKeTZ}s$w(cnPP?J%LG7vW_WSOq-Q5x05IC1cg;_cT$l=N z-Sb6Nw8MExLv>%vH!;nBxfQ^lntPAe@PzYYfn7m4nijT%KF6Kj4{~Y;cm3bOSktnM zsB7e)?XH28$tPFv@wg^yzYn4ctkQGis0{rovR1YT6n7qK9!BslYLWQS7zcxV^Fe_&MpQew6R)SGJ++$0nR^3C{aIvgkUph`} z$YbauRaJcibJw3+Q)K$FTCMKTTEv&q5r_6yIjn6_8WTA!r%L}295@#`8|~z9Hbz$9 z7auqCQIZ4^lO)ybV!O=irZN-jFX+~#bY91tckMOVB27@*A$j4GId{=NQmPeiyf`%J zMARBP0k4~|vPPMz7@{v`L7tln{EQ0($D%vI(cRuLMj3Ot6pGgDJCZ3ViM z;WO{wDO%kUCAG}+!Lx7Ajs@8c|0*;X3(UdYah00Do?AM?gH^k-k~lVhcr3bV7e92r zFH7e%;!NX!dsSVb>jB-h(%4Zr{K~31#pn(WL8j|OD=+#i5p3wKafSk{iED`qgXi2H z3TjZQHS@{_z-tDZvet457%i+=m3_aVoZ;GY7l;}?QfWfwhOqh|1EUG#e?971W9}R$ zXTzFj`o=MoMZ$|3>D>dsHlQ_Fe@SX;_bIKQgR6+R|G9H>#Kd~}7=|CC^=QfW8c`b8 zq1O9raJV#CI>z(>G-%X2X+FKh0zXwC$p)lqdeJu?hw<$1usO7H)nv^baX}({b2eqx zH7)!#Y*SdW@t2@h+tyVjje0@sVIz-VU!(7#llkV>&IL7g$zi22Yg%lWp*^^qdYI}r zyG!@3P92dwtR6g^Ltby8vo#&Mm$_F3z9OQ`O1x;-rs@bf5BbjEuh7>)+K-8g=H1CJ zfar87BU)~qs2@#UUOR3V`Tbec%0sDQM^0Xu6UKj2O;GSJ#MZfHutS40L=~UW^Z@Y zb1HSxaa%l)f&NAL$w-oA_Fj_JO?$$ zO>-zvTRxtRgsL(gM!s6U@?m4o#kMB|ZYgV$I3LUjtJG$yr|@-+=%cJM=LOS`v?8=` zqy&|UU6LV=(%0^kKg6H)GMxgR!wB!0{>_vfe5BUmb7ebc0&`weLd)k(PV*BvJbYEx zrZAXH_9EfP{c0JH;k}lSpe8c&Dy%6RtQ2B&85sefJ+4!(sh}6lSvM*6Lvep)a?B!* zlV~KzrH|2(ogMwj;-0W^r+y0Pn<#3_m|{n5N~JuZ)~5~tu?WgR7F*k#>QAcIe!{`# zbolv;knXcz3gXHHOQCy&d{z$}c>Nz5YP}m8a}uizy_k#WD{e) zYpd66YAi$EJ{C_W1?DCt6nYze+nRpua|(WSUGa0T31N1OEzCF%>$OJw`Pk2fUCipV zm}}ITU2h9!3B}N6KOQs9073jVm&O7Qg~i2FQP&kb%awb=balOz2$T#nN6Fy8#y@9= z)Qfxw6>E;n<%-K$dW-b#0~vZ3>VSc0@MRc#TX^T!2T#4MRc=tw_VQA2=D}R!tDxJ{Ip|ko@h}7C zCp*J+;Q`!b){P>FF@MgYv$Mqdr%pM*nVn~OC>0CKs9s4=(-0+8DE3U)f65;*gICr? zR=Wn(4@(R0qG{~7JmB!oSIMj*&Z?@S^s>fOdf2kYZs7=;5^*3DrZZCu8!Zb{BHuND z+s&@;m6dO#Cgp@tDFq8cbR2BbI281*m9E)Q#@kD5;^n(cNMPAFle$h~iBzwGvu zQpWruPMYQQlG{11Po_VUTs+Fx>T7LMcW$DQtQ%$?Gagks7h1K#iJ`m@Ge$0PD5tpn zWt)0{JE5aKd(;Yr%^zxiOKIBFj;qUYh5)mAXJ6iO;b#T>+?Fl{oEUE(Mg+tt;GByX z=vkFV3`cP(yrZ~~G7%pwlN`@<4D(%e zzh6~XhZz5&w0_0BG!?k(v#7DTan=5yw-h1$jYnls=~P99(ZZ>>b^X_DeTug>KX-UU zQBG|h4!J40=)1g@uh8~9t+g-1G9%h%?+I^jgTl7T9FBBlPzyN1<~S2}{Jd!{<^l-q z{%C=V;HPx14XrEgXgb$_`{H9fqN$xJRuL}q)2hsVd!XUXE!<8{%zwPLP21KmNr?dA;u$lzr;`(J?c(gICV?8x zYW(Y;@l(Ph`=+9tJKed##6XQ&3j;gOjnDZin3e_%_2Jk_mS6u)am(qSU^$;a52FFg zha^ENc%&n4W4?=vmrf~x32KZ00Wb*1@ z+>8B+cM3(rb$}Zs34FuNrUnLlVmio=#HCMBS{QEPl@*0#3%@{r_`VN(y7BbvXX%%2N=5xwt)fkP6CdV1)NX(2_5t~++SIu( zG4Se*iMLYo3Uw5B(_H-m}K*lJFNph+V0i%Ao}lSGajRFJ|Oi|E#|?n&}g9Y9sJW{$M# z_K5c#UKM<8bgIUj*E*@go@!^KU*vx5I1mr4zlAI!evGFGg?|MaiqJYHro*Q8D&{Yi zXjqNJ8B4;af^Z7pl*-+haSJyYv3=ve#s6A#sbhl*xeo2?)peSWd>$v9g;eKqJUWU( z#jxTD)w!*XfXwZ*!OB-1Z~E`{9}Xs6i{VC}g^dG-gMgmJR+NYOnt0X=1xn^`Xie8& ziVCo`3+9A%L+CS`$8BVWElRr2pqwjmexh$ee*m%ESRZ!mIL3T}UfzpXbQ&2TLHsP{ zOgjq!0!;Otz_xEvbtN$Fu6!v}Mz4;vV-cw)xR2&FS<2mI5SxBaW_dthFfG)wxJws2 z(=nJ1*uT-Y@U64_vM>!2+{ujnd{O}I84JrJe7KX%>*1%Il?dboQAGQG|CfE5m_dDU z1saKBU#l^6&>y*o8-As6rqTZE2nScWz)!LTZ8I<`jUedVD!S#8OrU~hPQxYLq`>6tiiTyDv_bu5McNKaTx2oI0GpCh>}@VYD` z3wpN3W5i_P$uJ-%3iT0iqitnH{G?CphO?}nzXDdwaP&o8&I|PND(-3R=<^vN_mX5v z#7D@-8h0B7JD-1~p?Aw!{3OxxzyJ)_vl!sQY}tOY65=_93CEA$P z^IHb)?#?y>g$*1Q(eQC9w0rXeU$G}d`T#2j||e#Gd66sa|zL>=A8@GbH z`ucCno<O5+a1NF4pnbE0!%@xd>-5N#qoSj0gwhb%*@sWZA`ds%w+-HH zSlb3Kx{YP#3TW|~|8smWWwC}H558D=>o(`iDY!5M?i&3oEyn=G&b-Xvs?l-uX-BLTN zS=yrH-)N?oALXzRD2(I4MWUJ84La4D<=U|tGULH*JT0JA;<;&9YdnI-@&w{YbkSb@ zJaqXdX{c2QXIN;b+2S>>67!a(JF+nE5ct5-@GL3T5v)`OeG}+83_YeZ>Oxb*3$rGc zJW}*fy*G9ELuF!YVXL3D4h6i@w8F7ffU%(T4+6Dze+c^*RmSgy8?L!|`N5nB{a>yF zp7&mrIP^|=@nj(m2{U)%Eu9q>R@rW?X*0UZpMwN1->)cuT%#Z3`tC@e$6#;gj zOw2itLN^PlJX8-YXGBF^z_lt<4<8Vfk%Uk$h%+}Xcj9AhHBxz1E-n!X)<*;@?zY&z+!bJaT zxs~Ak-)_+|Zi2-qKTY@8ut(}cB6ub;am-#wfF*W<=(1Zdljh?Fm3v9X#wfP%D$&1n zB?M~PyNoRU4U#2b{?!uPW__~gfI3u1oKweKgrr6pprijD-gwrJ`>?cS=>5LEdL%QC z{IBKoacbAMub6*fZ!e3nA-O!H2nDW?UhO25@Q2VdP{E8`f2ZcmYKQR4IBbRt^AgjA zx=G)Qs|t!^F4j7tWy(~d19ywC1w`hIOv^BM!7iFQC%t9U&3st5sj`J=#}hM>WFnv( zBfO3q0G!q${eAyjS{L(F`fq{g+MWP@n+O+6v9qKDx3moea4jE4S3BNQ10QTE8*_i6 zDLoApV)KZsH0tG0B(hmALcdXo+)9Xo2&0YZ^e+ZI^>>ynOt(?6tPeF|LZB$v;bI)7Vn=>1hTFshrz=o5_sUH!cD;j_W zO^eCaMl@$-qfTv2IINAo&7=wx0lDBtz7Tqj-^cPz#8<|xF2EP1u!M%INS&9U=y;=1eKG2|4SdT)g`xH56;23whZp5D#JcSx@c<5f`j;}`$LRTQ)lcQ z4!rxfSU(ciAI1Y;!fX1|vdqAK+!lC7r6nP0L8068~-3v=+(c!hh=9vk8R4RQCtvti7x=xL{GBY#Q zbDY7D&~59e)n+0dWiER!&F88d%)yd5yixIc}OG82*!_E&pH8s36;xbq0Amnl1_g_M6tpK#3 zNtp^8&d=NrU2V$T?%yHgYoZ&AG0*5{Nn}57xP4ZNeA?eWu9^ugY_d;?E2d<$B{^(( zADJc}9SwQSy^ldt5GnY&v?H^2_p*v72w4X2=*#n?ghG2oUAkjw@BR;@)p;zN&%M#4 zYO#Fct*}@{_A8(hm;*UVciAy^vFy*OJ#Mg2^vj?lIh>VJ#xdjTuC#7(T|1h*FZWz? z`$S=oQ;e55gFvCLzIU$PEt7G0=Dm8MHsjSg`{@-6?}L%W53Gw*)@ z9>Zm+`I0wB?l{@L?c$#N+Yz0MM$Q#(MA9wHPg?83GX)iu%B^1|%euo;_qyK_Jg5A> z&YL{}D58toV$KL;h;S}zN>T)~pxfCoEpgP0c(E-o)xJU@WBJK6WCV77Gv0I8B9>GU z_AWUgef#>JOoaOJ(q{IgU|@NSmDjFCPahXWt@~}a6oRVTjhwS@15HtnJ;ENm0cB=x}^C@AE42Rz&kffuDDL+}yKNb{4e6 zk_S9LMozxg2r82I?a03I|n3Kd)5AUC4Q_bZU)b zTi(>7_g5sXz1R^caqtb?PJtU(YVwj^ZBoL0EqQrKMte;$nj~&V)|uMXkulVy4}puA zx!*#a8FWa*^Xy;i3)k!mSMLi~NmWVjvm5bQ$-o5vETu|Y0ba(l19viSaGb%9=-;US z3m=&-O{Wz>PejnylW77`;O%w^Rv;?Q8&tH7(hgZ9l5B zSL)~afifGp2p4>aYdWv?a78!Lg|8d1V8sp+K6LuaocOsB=27u408i>`XG9+#nVck^ z?E!n~gBpsC-xaLZ=>8OnleiGKZ2J=|fYg9N^Jpw7n&o>(s_h8%0(2hd&-0`5HQ;D# z3-nU+E%lRi7?`iN-pL`bX|-$?WuLNMnh~PiQn?Wp=<@+4kNcM3ttKPuZsHr_Fckor zr8?;|JWS$dl~K?l_1v0o5ESjUyT0$-bEvylv51Y0;r9Brdygsi>Q*ZZe9}XEO~Od> zhtBxuNJJT{8m6lwtrSHReNh>5G`b<^r{l}>Wxq)cE4((U0r($aT-oVJg;o^ zO4+Z`1&5KDyOyx<-udSbsLo0*Aau?;%u`%Ts6j9P^|9 zb(G0v-kTpS?QND_p0VNfm03&z8hj5!Y{n_#4~NQg{IFq6o35tVS>j@hB3T_BmFE>KW{#s>fBT}4P>~V3;mn&Pk*I!a}9f<7I7v%|R z4jy4$Aj_#?XHJpC#R95A(#I)GsZcHWz@pw8!nE0~qx&#s%~=*h*w<61H3X0${w%3I z+q#@5vU)k$m!YL2*n;L;XaIj!*^{uebg23W^vYq$y4mM+>PS`iB*sYxe!IkBLNe3p z;9ZEvQyWO_ubrDV<|c&sN4F@|iK)*5UzR19X-3yHe7i25YLYLEK!Td;FWUx8HQ;0M z_a);dhEcMk^_ihZjv>+9J?Vf9x$Nyc>nHrU=bE@Dux~KIe|Hni!eA)+RfTUJEe{9c zTb>@A!A}JV4mGq6#ay76l?l-4yd9H}e>Cy7kF$dTg8#<2Nu5S%g2=sw;Z=OVW8@;A3Kab=_{mM{{L; zDv=*@hgXljfr3zuk!cxeR;3?*18|2aLej44ilv((6LI|#J3r2;n95w=N}B}NuW_}T z?!F*T{{+5K4!)lPEE0?Y@p)ms4qSPe`jvcJ319p<`swQ&K3~8aCA#j%=3N7`B zzFu8FBcWpuW=hC?AaCZAnuGCtzS7$U`j|Kf-H&koS;m0`Z(1GdQ|H%|sXW^|3Byi^ zuuZ|g&W@G54(aUe2t?4-@<>%a(lVs@q>mOLBFyTb{O1;%QWe5e%q;f+Bk=QNaFNIge$hw9ywwlk8b5 zt4|`Vnk^LRnd$gS_ogsm$gxL)d)xEX@wH?JHBx+-7(P;LE4-VmOnJ)ePE4qdsdmLT zJe$43c(VH*u$#xa?%(z9w`;_^*-zl>j0djx=-pSv2a-E)4BeR`WqMN+QeGs7kHO|i zq{LxRYCyb}VmzwiGetHn0{tJE_WYyxY$5U101Ce*;&Xq4$;yh^*&-kd0}}pLpnjss zTk$K*v4MroxmOn?&uDYmUIjH=uD1^b^v$)tXC*4|uB*f_UsKLZk98ofkVc<9T4>%w z?;!`-P98<`Y}VvkP21Gm-Bhm^r5~g)Ra4v>xz{*P1MiRu?D%^f9l@|km%Ol>Wx~!amvB?4KUR5n!B zm|txZ?AF{o^%rS!D^^A7J8;L{rpk}BZs_e60j{;PyM-g{*3xr&PWrDj`1a1O{Uhnw z{c1HyaF4#>VaWJ^pb=@DN=4|#@00-{QK~ZdfmTHRouFC|5G;iXBFEF z-n+6_!6lfO!AyVUQx56_Vdmj{4E_Fhs$!24RtI9*tWH{-gXMNiVeltrBbBMsvni+? zw&qiE!Y%XT^b4IVU9pD)vTtsO*M~T?1j}d2RoK6tySP3S0BRLb2B#SU!{#WT4O^0$ zJ)Ew#$y}~)Q@79n>@NaE#J^+jg>DhWOCW0Vw`ynNCd zoPj-KYCDoCA*z^x<=Ck*j!P<&m-RiXt$A6Ye7SHGlc^{?xIiy%OzJXio=EDl&{I8> zKTzzV(l9@dXA&|`*7<{PZ`nZDaeEK^AxcsEF@aefMo}vKXweq47;_&5C_rEp^r zg+qJN{mzqY{!q0-({}b$U9-BGUaT(U(~6rO&LhUMsmL9+8V8v;#N+C{)=$BGp&uDK z=tr~Jw8VDi*PIc02g6Gpn`9E7<&W!0=$P5v{9JjZP4xTZE9+UCa_@2f8DD|Bczw!^ z@>uRP1-v-Zi&9j=w;g~+ z(-M!WU3XGXBdwRfZYo&RbdR-&3bZNM_V7s@ixMmz4uV;ZUa1RfC zZ$fm-12->x_(Q*BsT?7o5W3Ru9}=CWo-~}E5-?=$To5X%r4XW&FP>8SRfD}q!Pu%< zIYt011Gnz_&y&Z8JI$Ob@b~*$Wg)93iEhflRP~wPDoED@2#_L_@!L8h@cUkQtp-Zq z59I;?#hCF+?V7kk^-GWK^U*=7e2(I5eVdiTM{Qpvnl3Ict-fEa`80gRnGn#fx>5US z+w0#xA2M^L{sgj`INB+gOmltBoA3U9wegkhhcP8P=#HP?#RkA&Zv6?aG5NUSw-=!eDU`(rc|Rs-QlZV9j= z!*5**#$k__umA8qsljG@;{baFElt*etIwl$4zfcW{ttbn>TvG1qtF$+VU-l~9 zYd(9Fwb;IAA2xh*Xt0Yhm1`JWIb`VjtD# zhIZJ18wJ0#H4Tu<9;o=o_0oR^#M&_|$PP|&FB3t~(RMw}ROMx3q@_J0u2cOBwhzh<8&D zHI_or8X7(t9-u`^G%d4g_k1h!Q?zW}sna$f7*7o7RX-T0p7Wm_RQ64PEnqHSW<)7i&Dbc`Euy(;pRomQ;SRupzkfUUCQ1>$enH zajixdAWRu~vaQq19PSg{!mLjtLmivNDg`JHoUR8IH%sbAS=3kuLtZ- z>UwDEkjcZX@_!nVTdN>YBd#VD6;#annx#JAX6~a;J0l(+=B=L7x4OD)WpR(ZVjOyjJagoLFRFg#2h$x-vlapvATo6nROm^2NCG#jIR1 zQXl3mF;80=7>^Y7t21c+>!j7Uc8Tr*4UDCK3Lhk$_7W=sUn3aU^Zq?FM6{aXIR?@O zBC7>`>|2Bh@9h5wGtjn`h5I4v)USdy=`AbtagrQ~o96gb{d4KXa=e#*_Fk2x6>?78~d!JEJ(?R2$&i#J`t=Xk9%BfrA< z%B54BE9r8LyEx4g^cu)-Sreavu=Hm2Dfq-U@bdNNY8|W&n zGE1$u`l(z?M}E{SE!UUE&rsc5vU}B!er;9IAs6wfh-}!oY5W(xunrPv!4@#{NTxOMN<(HKBr^0gYVGBKpbl0gY;nqg&h! zqtDm{zVA*QCoT1fg=JnI)27V=DdEe|+<0+ZY{tgmUi?;gtk897>c<$p#}1}iDgO+LebRT`;#`aT zi4o=A?&;G#wVQNELcKf76vOniVvVFpl4H$Wj1#Bq7MBY%iWBYM(>aiyMFT@kkCCmF zW0@M$$@SuXN7K4qk>&}xD z!x!c8;THhpJhC|!U#s4HrDA61ix8G!Y9586m85|SR#|PAeGoRF#Zj>jMC3=j`%_69 zHIms>J``QJF>&D+$_&BD10;C_fNL`VqtulNY}wYxgyGRyFcUrPSGu(o@Uzd=)uXvT z{yqIQ(Ji$}tZbIZIc$}nK+276dZFIPTf_!`GzEKI6pKJEQj_%#UWRi5!+)%~j;Uy- zBY@p~M&AD1TV?*dQEA0eLOD2)Rk;-h{3RFBVnYd5^DGRID@|44O~>zoSs?CH=~ zyt#L|(bK{Iw-Q3KJLf+?38_j;*sQV0xy-`v(`V}g2CKo8AiMcX!B&QCY%)bU}q%?m?n{X}`X`8km z=4zX6q|G#7|Be~k`5N$Y9eyQ5`5Cgh?reA!Anv6g7q{wC2U@LuTF4AcR0+7NTCY@^ z#r<4X%ntReMQ7zZhHE2b;XawS4v!{zh9fN`HnW5hFnSq)KDKzRb;#%!{qvR8mfU_q zFBI46bwPx@lLC)C9FIdPx5Pj`1nYz=hOngJDXE14+oaVpd&#v9&^I1~Es?)I56-Y} z3(Tdomz{ng_SgJ`&N<7N1reOyIaLndkH+S&8OdMq?9prodAE7lLQczHXj6Edb5_;o zui{tA&HE>vI__ZZ)3sVF?Em|FZ5*9VS5x~|+;@8V&vbtO*>9-I-4Dv}>1T)0qvq@jGBK*V}^2sw4{WAGZ*UeFz3BuMK&m)f7 z7|LpsugxknM5U^{qo1#s>s~-iOSLo+4b)|>slF00ROh+&(;nz^5mkxmK8tz0!-nXj zazO<^k5|NuNx~;vFNC};GJFSrGbr{ZumBE`i!ctYR{~g%d^jqG7!|mWW!Z<7gMJqA zRnms?G;H@kLV$$YL*YDmr1TDP#G~L^SI z9}$Wd%?qSY-lG3$I@tF+dd}$`I5+a@=<)9R83}vQLZvJ+H%iW&^br}^#JF8u z@JO20wjX`Hi$7ecbN2O-ab3+m##kGIswHT)Wc9-*Tt$7P&cDrT2ovRNYp zY~XJ9$euJX+ynCpUBTCmx%dy6yf61ZTDy5a?8QZ`dAg~r(H1c16HVtyS3 z;77@gwc|c34oz(*0i9ZZx7a$iXdWW-1H$ zof+F$>A*|+aRtMXJF!Ro;LBQCAL&2iL41uf%&2*YIprsz@l%j+?j_n$PGuxCk#_3o zKHEj!-?p!)H+;+=YhJ@rMj3}it&*(p6`ClXpn2!@0nJBp7G}=$>w+YBP?9ovU^d6 zyEU-%c!>1Uk|QQC&Mn+N++G1Y`cnHAFe5d`tuZ|{0os_Z1x&9EO|_?#f}SNgBh6VR{4q)g2dY^Z4%rXwZ*rntNTaF!_h3A~Gmz_jOF~Ozk_L8XC~(AUoK! zIIVQc7ec;F5#NPiKhU|!f~3W;!JqVpj$&_{*uw8LaAJJm00CsjM)s%ecY6CBe&(M0 zryBT@DKUuOv!~bCD*$tIJg-#dW;ky}sP23`=J34j}S zbtlDgRen%y5@hqhmqNtXlkRtn+7}lH)PpZE{HCI6R314blVS&Xi&fa*H!8;R9}+~;^mJIBScuSE|AJ=%4DA02 zuZCLgo0r;Fh4t3nhU#IB`-Xl1!&UW6rlk<61@f$bT~`Hx5X?OXq3JS?PHKwC>afHN z*QFYrJ+Ifi+!!86{M;jFva15K{m2wM`j`S+2)ahHF;z;J**G* zR3J6JX}k|k9WgvdogFEo4=;xQ$=uduSX-t&gFu@HXB_L8OPj&_MwYD!Fg9N~RX$bF zf@l`!*F#}7;VLCZH;ori_D%JlmNBzTfJJ_?-rCF*e_>JA4n1RJo>4nt0SFdA|RCDk2gc}S=(X!Xb~fb#F;=I`(8_vHIB_ts6uKhs2)sAFnbB43~e+e!_P)*okD z-G>KA2Cj^B5zPIa@T?BU-RM~!q%y?qOTJ@tQ0KSI{OYs?DQ=iz|Df;NHg=)FLp{FQ z!_VTav)!zni4aZrA@&xkY5{b&73v>ZTQCL(1RHqi8`^Ke7SYVtA+i^FoxF}L^1e5_ z4P!GS*&as42(H>>OmGcQGwfrC%dYZ;XM%?f@~y&FTJmQrUrh(QZ-GQn$2c_1JHkEO ztXZlg_AtVg;uE^vD}MdVE2cEj?1e&cESphO;W+lfhMhrhAw7@axov+F@L1}v`$mXL z$t;iMg`L388+Df}4r$H_HG1nlC~t<)2Gn@gH;ZCPQJLy~)oQ@H$`Dw~uC>smNvu3< zK8!-^l#HDS9?3ki%2Aqv-Dl?4QNC*bDxeo_0NokfyFzmx;K(!Z59Ll9pBIqca%+Ak zAsIFHRe@}jys1CTv%xLPj@#j_KF#%ps2!0$n1mg#H|(x-;lJ&>q4blK=VI)8p`s%@ zuy4bevz%93v2dX|9+`ciD($?4NGH*|_>+AIDEBwisp7_G<+5x<={`6g=*I@@T%8v@ z{yu8nIkPKN(%$R*wA-DC@22_plvCPOz@Eb$x$aPJ2NCSp52cLLHdwf2in~C-Z79iR z#3Ro2>Ux;>e_nAcC+hWd?)2WbVIG$F^d;)`&)kBC)shV|^jJCuu>g)jxJ>oW_&sb^ zsSu=`o#tF~`Ay9^%)EQZp~$Y@C;CFJuw!vQLK{R3Bx}S%8K|uToPsr$z`4-aY*K~~ zK+zYf47?cWv&?VdS{td`8QTQ-UD?wko}E{o7d{kN_j?o=Uf67pgLGbaCFZ*eA(d0J zh2m5n>1Utj{(+fpGc>J`2b%wTowCYk8m%f^2#0NFh+-->=}m@~vRtG< zFABR475w*N-!i<2$ovFKxNM)@w^KAE@_8(&H&M?m-V~a=ts?Cfcc_nf8`te37PJ<} zzvX1;@@I6(xBQ8D_ost-YyC@AUVDd@3A8K&-CEs|Ti+FR)09a%X)pa`aObgdN|B=? z$lwRZAFBLGz<V`ZRu)6|}?jRtLQ#B&m}8Mk)n>$zqvg~aO)8J$jNd;OE%fzcsb2(np`hagS!s3bp1dnAOQSGF>z)4!=XWa+heS4{4PC3c7xo@&wflFhYhrN_FrNy$LfcJ@5S++vIx zS2B`9w0~H1f%5q4kI}an6W*LZo`oY#xIO5dqif5#j)?K3b~WiTpEtC=s_-!kLRZSz zB<05xe~*`q3(^HPRYrMS+_Jqr1(2YPB0DG=8f6Y|YA1?1POV%y|LlRE-iwpOH?<~v zH8~fZI;5q{X7%X^5vbv1dhxi8?90 z=eU;kT13hsHb33S;n6T&cWGNTuheJQ%+P zbfG3@Y!9(eLvL>0JUCKsVGn^%9H(a<p$8-Yt4PBmafrDx`q6i%aSHjrM#%;rLv)$sQT$URQ^ z6!G`zZ!wBWdXt$R@`k@|>ts*|#GWxHZjO$*-a4}nnbyy!Lx;GuXD@WYr$Yk|% zIb=>8CPpG`NR~~l5*Pq|ZTz3e(Lpb=qL!2=xURK(6kXzAf@eSYWYh$QMPzoM7#hvL z0*4L)S&%Qk$)i<_4OeLoSnx)0=j15l_(#{c<<)@mQQw~QF;>R4A21~omknIkJVy#| zFzgm3q_!tS4xCV_13m7ty=-Xy?CrQbk@-IJge056$*@yOkV}eN6fZ)|HDUPz8$e3hOmm69Y(iu-z13{V zI{7?F9nBq1@V3QxPEx%i3%N%`jbc1fbJ0$%?+#B-IBu!8unFl$zdI zzl87`C2?_e_(Kd`)i6@3oLQ@>{@-AsFK5ffsDFE068%tlzwAgK6&|xc0d>MlUmGS- z(OZy?-t?=z-QRg3JnE4x31On7kIq|sFC^wRLuOU`#rs@78>y5v+sY!I5ixUTts+D= z->f8b9LT|&!bNIUt1NcogST`CK=J)dT6S68g-RwbCuAZow8W zP-~@Cv!{hNK2FHX&XQAVnn#-XtSe_;P9D37WO*~+e@_RxPZ1Naxs$Z3$9 zTU>dv$^Nb0gWe4Y3?yaDHGFIVWwYd6lQHY_L0~w`Q$rWV6(r(YS-x%1Hv`?3>6>x> zRS8HOw}Bq_QB-QXH7v%V{c`x}3D{kknH#hea^sV*bv>oju1lO;dte@$q+IsUs1 z9I{z^(_q=fE6hy@#(rn}O`}APQucDaMnv0Tv)Oe1U;FOyx6KkWmE(m=nnn`cV5UZm z4p{SZI)bg#Qzr!6Zxo>$~gDWj#?qm|bPO#rP8n*otRi04R zy4sq+QY(fed}FASd9!21z{S`Mh^ULgjk+A=ipsZy|7wEZjDTSZ+_2R;5>zA^R) zy@lb8193uNksRJN&wiJ;_2VIap|A*&O7Fg=LgRgA_q1kL`Biy({ekPU#hu&4Ey?v~ z4*SEX%Gv0Qz*pG%7J_JBznXb=c8E(BOiE#K^M11CcVLYF+55!*22{x#{i>>CFY2Tq zSxdPcl~1T|-C+=u=>Uw@Tr7M1uYspmd(Ffa1j;57fU>{MHplfEtG_I^N4QvdBQO!) zlP{jKZ(^K(y~q=;E`RZBNBIyQK4~*OY7WyYm#(UQQd>uNB>?@GuD#~f0f+!&c0QV6 zEoWouKQ|?ys+Q1JIUBhLDvVBemdYjn7C@QO0Zn@Qey;YHnR@K8NgsbBmrWj@{clk_#c8cRW=#`e( zET7S&Nleq)KV`}U$*sUYG_JPc72)N`$O15|B)Qr$zr5G+)Lm&;?%kzO2K;K_faOip zxw5P;uL#mV01~Okhpg&gS(j0ic+C5B*TLYnoKJ$LBmNeP}ffC_+ zz|3(8XUA}$n>4SrK0G3K(!6iBI;QQZlhgL3J6~^=;|}lp~*_6saS36i;JkbwyCiZzO}<}h(dy{R8xJdmTg2bP5^Fsic*nPSH+)J^!tV{ z7VHKp74JuKWU%?W4Dk~}zS!46*KGEmcGc(oYIjkc+(lI5-_JI6-YW7Lq-eJahRgED zeXWQL>C6{JeILco0=H&d&F@go5C|a|)4WQn>mr6p>IPv7C1XY+vMDg;h|y%>Anx@s z)Gs$QU)kg5jx?)yC4i>>a`Cj z^2G38Nh7TX1WarL{vJ|-ziZq0@SRdP-*Mhz(ZGJumX2+w@RnOSMLJT7YoMGiPl{Bz zUk|v-?spn8Ug{&{cWU5gz18kgnX5P0Ul}BJWoJLb#V`T9^FMa4&vVo{f3^GAE-3*t zn7~{@JKD$f^XzP|;C~;DOKET(9@2X#5-Mk}zQ+tX2q6Hha)*zt0+#29DNPAbpxS!O zmkD6{H=vSQHx3_-e+zIG-k07J9r>Fiej3H5dS+$uuk_~LDFCr&I}3zvrn%I8xwv)1 z=}B*Xr$XqFXRebDMVR;*oWonMp+!tRF;QH*;eJ zaU;+;wb(FwD5I-V6`HAQR{WtxQfjr8}g$|cN6 z%@RiB@_8BqzC2kXYz3W(+-HaI4o*1l#-~RWR>sr;e!($-uj$?Wj*E4fz)wke#1x2D z5=ZxXj*y1$J+aBYw`c5Ks>!Bje$qZk5Cw>-X=95nw2>@F9LWSLMWXl(ej^&mEDGi< zWQ7P0RAylqT^$9;7rh z9)U8JiIX3E2O{imGOcQIo~C1!7pE9qGQBxF0jGiS70bP$jnD658p!Id7mcU=w%5vd zz2a@4NVk+qJCFOzqkmIS8OFb_{+aU#BZW$D2(uR9nwzyiXUt4_Nrt7Wyjb zlKAP_sD6H4iPHq|*IAhbGiTD9~Azhbooqk)a{c&zQGo6H_GR)9@1;~5_;tGF)s!(MyrLZz{6TpXG! z2Djy-Ua87yN#_JD`O(=!ss1pE0(dYQ=9qW@g?V=bB zmDRQDqZf|Z#G*K0-Je|Y_GoqLeo&b+&si*GqJfADq)&4%APED~U!scXCb1rQ!TlG+ zo=@K@p4RKF)*YejU`Tjz3>lUyOXE~I4c!i0wml}eR|@&cB=K3oG&KFWv98OPCSiVD?i*6#2I@j^5f2*S+!@c2;kGckP!GXtI)Tw(pA9S4Mbe~l0D zGzaBvWZ%4%{kCL`VI$FqC&Hhk-(s})5fL*Q-2_hx%+|hY>#!ZxvvrFc`ytI(`Q&D$ zsm3E+BIk7PBAI9C|42Ibf2QC6k5`JysZykv%3D$nA?J-sB{_w1iitvw!@>@RQk2sw zha+{_=YL1J`xkuIur9JRbMH5v(A8cbRG?Q_64- zVBvq^-#JfRf^m?Zm8i&F-Pf8zh8k=6y^as~n)b#tUvUu+?!u375A?!LQm3=1ssBlXcf6Oo6&gn4y=r zv0NiZcyfm|IDhUw-yAm*Lzv~hWAu*>y?MCh=T24H5p$<5MyM~}G>8)q=1E9P@E=z2 zF}p3v>0>4(mU;8)6R6#x$0)VYKR`X{cyLc#_$Wp!Uw8niq&Ylkzb65BA9+)QieK~k zWlCNaiZE+tRqifd)PjDY?Z!s3>g)U+Twy)?As{!KdZ*=H0tBAo-@YHxXt{MSm#ShD zmE!&IE}CAVh^-8ERJ(SkneN-5UwE_I?vff`u0uvhEL=FAos$(DHxh*7+%C+m*ETTY zC|N61Ujn{OxC#RJ1>ETgf6^8`KN5&d8M6O#?}-$+8uOs1x-yaHm4GMd9W`PrqNYzR zUDav-G|l+x#7-e`l460>mZ=;`w#rai-I~ zH_Yc+wO%Y6tKTtIe2VaJm3bukh4cs;+F_){heKBkeD*I$848i682#F9d#upV56O-? zzn>$J$d|5Kxw!LhcJ*U;gyu!iEicYFn_y^fzy;KG#QnKf!>~R10TGl4Qib6TFYRQ=Y&Zum8FgT_rKWJMtiF0ILJ)~V_pW9gNi%?WRsNCy@bw$wTdz?`P)iWGH`I>4uE zASaZ*9$e#&X}ukSmxH^-ZI5$<2Bq9w-SGzxIQMP%ahh>OH57OmeV4qC zk=!{xyERj3X{5bxIOWnX*g$0LpKTO4Rxl60h`@n0#XwpzlYceEdlxi(O}4~ecu2_K z^**~`n&y9Es;~hkO@0hMUX7a)%8sKSKVOerMaVeG(A@ijzBTle{nF`wdwvUcX z`7Hi}khosI9+>hgk%K3Th%Is87$lDQv(~CXS_7U4Yis9S(jPDs8S*6*ntf*zRsy^Ru z$UU;I)xZZ#QcV;k(*_+$P8{V+9*LufpjjFO9eg_^p-LmWy$NNg&$fy@BIW6?#F_58 zc6-%e7vY8&bNjI6PkWOCrddEAvtHK{lw;U%425sdGbv`c1@m3i%CH%WPuqOaFYtpp z>-EBP`~`aXk;iR#P6dCNzoch!6T2q<`ik>^36hQDuQxBebedVzi%H*Z`d%nhEb^GZ zd{*5Eb}UlK(&qi~*MDWoAA;4!s+43g?@y$aD))>mlzK3Qs^rAIwI(T#3>RDOGuOQb51E!BX|{SaK6&T>Q;1dM;? ziYkwnT7$dFY^G!ZD=J)3nv}dx0~X;b(!omO4l(oba_I>}>MoaH%~r2@l>&Z;dkUH= zMYq4iMZKnF>;Uk%i?1XzF-i0dYsciwBCw1 zmpC%1aJf>2P51w~%2O00lSj7A;L#EX$qtP3ijX5F!ow;T3S6%fVNg zZog=XRgX2PNRJcG6D~oAO{zfOvfow7SGxbCw)a6g_soQ*+j$N&^`N=s>-WWei z@ftZAM@1ZvNMYipU7Ewy-%`$emkfqHgAzNVkKZiSw(vW%3^oYu78zvx`O91sV1=0QLzXhYyxZ17k6}qS7lkNPGCZXLx-^Aeu**c}IZ-d&$Z_M1q409F< zPDwANKA=TdS%0pZ5HvT|{|pdr5c-;KfjRDqx#H2Nm}J?z`CPU!_g=RYdZ2ukKbx#g zbX?K|@;&I?0e;dbA}N`MDL(%zgFN&xyYqv>jg>37&ui2`$5KSU{k=oiEUIx>ik17q zyvU3lQ^Gq11-Q1{=oBxr>V{qsL-*KS+>M06-aZ>BoaBPx>c38QDy7gqUgiWDgZSHD zy2zU^auuj@vXqJ=fTNS z9$!MQYucV8CD_@>0O|F-%R6=`HSCdajs#m2@+|w=<#Pv;K~3t?Sf>ysza8qNpMyqi z+}A8qNjIe??JYN#LMz~^#-%-zgN14_=s&B)9vd;Fe8%~CfV7=6P|hcyaC_$l(lY_C z!yg*Mx(v@ue(>G&nP!f+&X7O|{B?GU3*Jt({glx~rHlUU-aZ$pCbI#zY0*9}JqmhP zgfxcRAgx$lsC!SDeD~2UQKcE6UO&GwO6jx@h#Zd#{6hS7uYI29YO)2Pc$McWajxC& zz;3j+RKEKtQtqnARs{aERR$Re?tGFv`+o^uJqs}rpPf%rJq1_&a3fbgmqRVkm;FSn zW-lFjj6>h?C%*@dihW|nuhT$sShSG_c(vkaM@bi3zx(@@B0D1%zQfYiaixiIA5!#d{o-#~yi1Z~n%Q+}K@5Li&+K`KQjYAmeQ4ZRn6oFtI&cib-15jCc2YMQFN zJi!GFTj;H>w*Z$^lnY-&t~!ZW4l-n_@2S_Ax-Sd3gHnc0Eo1Hb*R^h&m( zPeu!9)td&?^kb1?xQoB&!Kn`;Ni^|ZEEx^p zR=K(Z;FdZN!wUw0ZZ99x0u>CL2vEWn8mJpgNeI{qofhMK$1&adYqtT^G+Mg){^S2o z`pz!niUI~QQ5>#iK$2qL?x}hI6QQt^xunWAaZlsG1_%6cA?B%;l{ebECrAu`DL0wV z53m`Mzg92l1q5{?j?V*+eaMmWXw?4#y@6?iPeK#n=xZxT!lH*-;!u)h&p*QKlQ)5! ztYL~>X~lIYBz#0xffCUF))$jm>bB{rT7Q$|!U|3?o`jAf=`ea}Rx8WkV39~c(#s+M zl`|iD_xBhd=By!onDpRDxwEV-%SVMs5S%VG$wjwVXSmygaD ztYAv{@#T+?T~A=<#K6A<9jGm2PDwr!r zO|EIr?X_0A`at%5nYAFNL8rDcNZhE(%*B>l9`b8WoDEW6jMrby`}sFkHA|X|svKZg zzIDzBw`9#l`LAe$d9Tv}1C0LXE5|2kb#@b3tI!!4^QFUHpm|sozWuw^CAp>Obc`!L z8|;2Q|4Q+J$WcM-Mc9vO{Tpch#{{3qC%hI~JI}*C)0EvcWCypt^ZP7k4l4nIP{N0J z$q;6kZkEG-@1KDjydswPiJ4Qe_meMb(l{#mtme|4o?3CSOSCV^gJ?9;@b-2#QUWul zM^W^L`$rXq6E|nJR~jBma53q2D0y#V+I2rgYM`3IoqBh2udN8P^!`N$T&jvNk#OR} z0|3>30G9;kqP&{J(z126Xw53ftv_qB0iHM{`|c`jtoddg2~}^L>h(~h!!hO8mg$6s zq7g7fW2q`d5~X=&Sz#QiY#VJUQAK%x6Ho+;2F3_pgrU0VaJ;#YxCC6 zoj&0BVS&pjuW7+C6#&+UxP&i>dn*;G(Irpd4p_ZG3L3-}DByZ-5CBmr4?P47UJG>$ z;8;Gl&$-2){diC#SgE6bARy}N&Xx7sM%{%qU|PY-niSbMeB7vd+zVZ5Q&Z?0{}cX6$6@W((86fX`I(}@S-|L4jWwf}Nc0coYC$A| z6kLl_J5@RnR&;%k(X#{P4{-dQfb>zxF*VJlO0E_ZW)ECF%yw6JRo{>P8&Jwf{co5+ z@=hLV4i6%8^xXeAe9WJ7>dvuLuiew{$mb6t8k~Pa&ch_9;u_MwkdHA=JU`G-uNIJK!-EkkAMVpTpum38L&q=A98VNYPu4=yxb%ZYvoGA7 zaxP3m+uQCXnub&04ZX9c5akJ@K@Js&+*U3&EC$?lZZxi0_j-N!Vx|*GW zTPs=O69Pn15jOU#iBkfk(j7A<6@3FjK6_Uw+|VV||W z1IzgC^f=&>+RMb0Uz*JAx&>WgPdKZcrjPow-g`V)1HVX5X6&VNou~uZdoM|r^C&x2 zm#6ml2fKb4T4d?s46CbLzYdSk-hcU7ELbbNn z<*d>KxdRlJpfcWme6a6~g6WBLT>A@5@w{QSAMcvPZvS09IUh;9w3i&-bS-02a_uH! zVBz4=TAt(b_1Xt?W)s_`P2;pbsXmr&1Pb2vS1KIs&fUU&@;sc|WqgL0_NyRZ{W#rhRT5%y({2e=gt_*Qg`v7JH;S+)VC9?-$PD z1n?smTceClK1qshSaOEKo=H_U+e=D~|3}q3mV6S=yPJ5?@WglpN#*Sqtp~RU)aP`5 zS$vrh14I5?ewwS2%lt2U(&8Ote9_?7k|#q*cl#(aR#8=Cvu+Ie_Afy(rCMpvEG1wp!|<}YlN%@K`gq7QE$-@)pK|aFM z&rjrV@GGf_I0Pc~GG-O|q%>J^fl(&u4_-N*!|rmRW+-s$9bks$vFi2Dk!o9>tR%*W z-)wvz?GUSLoBj*%K<1%L#BiZ}`rTL~n9x8T-nuT|+P1<)ETMd<-i`Kh%rF0kc$#WQ z2}%$t%`m6&E-S7|6LyL-t3OU=U9sOhh4uJ)nJT8!%brTcIO>9&OyX~D=)0hw!RE(q zie{bxn&SOsJaVelT4asf=?VR0kn!fLo;h{tBmsBO4=nL_oB-&ac+Kp0GfMH(!!U1i z6Tjw?9X{XsVaG=h@|IdZEw+aHaDR02LOn@69I!97a)oKkD_Hc@=FjDF(gW;=n7uEU zx|*9<^|s>AkFv{;d~;_C2O)^f&txP4#|-URgO>viqg=t(r|5JnTRr6`P+! z_$GDKpfke@4Y_z&s}WHvokwX7WS_EgS4v?$y-C{Za5SqAx7z!ZN!QoRPiWU7#H#QA zhOtWlto3tim+Tt>f)8ySlB#nrUsA${MGo5E2p?oY!plzggVCVwaHiN(7xYRaweTGNFn&QMY4@=e^v3b%(iXN|FYlbM02Lyc@Udjr}fw zc|E0izvGKmfeM8aJxhN!c!APu>^D7RbJ~OO*dO#_X)4x?g*}=goX;6yW7b9jxDt6= zA=hJhkMrfv#bIMBFV%e(A*s*4y1XPAPj{@?FO8Y6ezI)+7m}f1l|R386oXrxCeE-< zfaiTTQ$z*{j+P*+r9Z>UHa{=9X3!JixJ`8vjZ(gnoeS0KN3;}0$%5%Y5V)7lc$nY3 zdoxb0agtguR^G(<{}Rv zl4z59?czSuLu(1Rg#j-4Z-GHe_#>sraBmf_gr(_k)*)3qG-!BdQ-XM706cZ%j&DJg z@wl9o9_<{9P>{Z3T^6iE7nZ0A95h;tGf~N-8T9J+A6-Y$@d;+58Y^r!+0GltPEs9s zB2Td;$!SXj6k9~I*!!!a>(psQZ_?3qwRxN5FNw`hj7m)dI5AW>JiHyFSt{akT{994 zUrBIDHc?pqf{UlLP!}CoY5|X8m7NCsMqoxo&hDlS+0VALgGBzLhWeUVOunNcA+XtE z_LIzeyAT8=jXzWiP?|A<9uVM1sWB${u9*ZrMgKPwW)^r_He>0>Mb{Uy87|Q;kf(9R z<Ttv`L zz`}Mf{muJEn;X=nnQzZJ(=505oC`ajlN-Rre2)B8VRdq`N=HJ&pQ;npSA>}PyG}HP zf&sHXyEWEES$Jz+qt`Ee@<`YWfNY$@w@ut@cb7dC;@$}*=Nw2n-$VGx(L!h6MrFJD z6;YC|7y3GpbA%RM<+N~cKMHs^i2ej6_P+5QM(1PQ5jaHdSK)}p8pq0U#yz8O+F=T1 zC*VMRBW0c(TbIS(!g_B!pW+5wMSLEtef_h=cwh${gjd)NDm2_1k9BV@)sV&-Rp5R5 zC*P3w zx;8tCoRY}Hpbfq)LhJAwP23_BCgL7%(OU8rG=40NfW$7W8BHWDU3VuIT0;yAFHDpU zUd!CifL6O#u+7}|N8 zt=I~pUvtV~VfxxyTgrhsr66S28CN=NfT%k`a~mI$;#FUAAq+rdd>A(NcpmrN)?(Qw zpdPFEn)kg=?Kw>fd>{8dGvq2;$kt2KVY`hHQ-hFf*OeV0KOY0en7VJPGML&yNdNcB z?&f1RYx;nmVosh49dM52R)ZDsCYaXo`)|F9nBTkXeMLI8G3nrU+yj0N!{=em4SsPm z^2rw_bLQl~HGi9lN>9z14jVgo-o2MI9WlIlS=wAgxP?+dM;>$9*R?~bWA#3~#Bw7G zbdg_=OuW!@lM_9g=h(Fg)Ei{~d&%Ljk#|1UkJit1KEMr^QLfkdFG0`OYp(8mxmqqN zxk36K2V-^Uos`_*cUL=t{kcyxGaC9zLqtx2T;p5YO6PMDO6P5705jxiICQVdXd!qb zzJj&g2ahAvToj`h6^qDn>i5jpfo=0Z`CBC`cK!7~5oVqa4d@MAL4elUuGy-od*tWC z7fkGmp0W7|_+t4br12Iev`n;sYKK=Pt^$*tsy7co`a=6JUVDd8XOhkNPI;;vK?CNW z@GafmzLb3ZRlsL^hgqw0Nd}8DLETopr)WP`xmlgVJ)noy`m_r)fZ9E1+`@!cv=@kr zusKN(CDJEWK*b%R8+9$H1{$IR^sR=&^YF+a&rVXM9lS1`M#j$I913 z?cM7P&;q<~^4FvzN!TfB;Z3%-PW{KjM@WIKfd#$6tB?0wcKcl>r+3z75qw)$lIMzv zebVoC|NE_TKQO2+Nb@ULs{tH@I<^!zxE&w3utMoKXjg3F`9xO$X%XG`;lkV3;y(;} zhs6s#*Dyuu0)t~;#dJgEyuwXJ))0=bG>v6>Wt0twLU)g(aM(xXIN-j`Om^or##RF6j!-XlcM!t^QEdH zW~8Lfm#VYtuX|G~0y-Ov28G~puuO{r4ljXjuU~O|2e_cr7#J^Orzzo&=9a+NY(^}~A>HY2)KaPFD0LS(+a}jDMU`EA&78hELX_e@(t-yZ_)xDc4Tz+cLaOQk+ zxFz^5;`5ugg-f^-+W^$8uLx7qn=OUxMYP$``utw9zAl~h(QG(q0(lQN)ji#jNYPq6 zm|sw2RStlg+x}_4Uf}7JhbiewhO$?K@%gLHT5?LfE(zf`gzQo? zsXGekTr8TfH#?x96N-2_rXPvk)mY~_LRpqWN3fg1r-DqFf+Q*sljd7oH2 zUo)GJZ(a5YNR{JWrlu2E^hSy-Yq7{HZttWPQ$t_9CrI;`d6I8c<6aiCx)tmwKX2A^ z>8XQIeY!63IestN`e9F_S*#6ismETfP&MwQJ|uS(H}iLYX!A_y+Y5u0WQocw9KmkMUGIS^ zX=af`8q#+Fx{vicy=V2#VPx3FsMZvx_fibH{_f{MJNc#Jg^|)xHx2K-1**a)BI|<} zTBd}{L3ZLfz&7%yMC;txJN52qdKGeZMLwF3^?Rv)ko%M7wJLt2ugQ3$DGlzpkt1pF zJg>WCk;}YdpY8V@Ze`jKLi(dh>?ZssA8%C5X4%tsI&V(U3VfX)GGx8k1E&nKz~-Ww zsv(HX6#?RTj2OwUBd-_!6LzSL9k5IvlQsaEGxQ&%c$E7B1T&dC&1apSTEcNkUGL(yaKcER-hi1B%`bwWXko#)m;2274V zHj|QEgN7<{6AH|g@){11&Fj)zuJda9@y0(Y(IbBd5{*M#Mq>1iiA2NFz+hvl_SpjI zD(kVf(qs(HJe;cdp5D`YjCK~DGVE%W70R}o)(5{3fZ*kz)Y74r0TqWYvH`Kcb%%Q! zO4KsuQ_B}ujckBO+wL<@_We+@>9LN(Cg5n$z^ z`Vz=v;@Knm*PF>)HUADymUrU_za{M&QM-8ddQ#VM^T6v&-XbQ?L5PXkR1S~lplZVv zv>4Gbh`N-_(kSP<+nzj(?QORY%z)-viClnUf3WPkRIbHo7nYw_Y*eFn59tgV*sZmX z!x{D+$&pa&iv3;eBzYbqx_$#`#UCXkWnxp1yR(Qh&( zX~*)G_c6dQV`sz&;^;$B9%!rGb1A1(*BUV?N(m!fdhMd|O^8dBm-*W^$r^o2&90s{ zy;8!t!eU_?nr@x7#ec5aDrHSH!$9~d@>f9n1f_D=(P2zOfR)j^b$u((8aAFGA$r!o z1s8F0^F|J+ct>JN_XPTc+ri;O(_Po6i~|%M!Rc3O+jjax7h- z{eurWlmzX4)feo2zTc9~(tTsZc_;}LyO+LPwq1KK`m>r%Vy`{u??0k^_a|Zgsi1(a zos_}Bf7XHr28nWXym6sGO)m`0{mHHPg>R2k^v+6=QRgJ}-A?Tk?KSG;5bdDHk!=3l z>;C-R%=fL`yLD9HWI4Z#m#kO2>=Gy&@GV`V7T*4S^ za$OV6zbs_uuDn#H(ASwPaCo3$P>blw%P^_hI0qSaIIuP?zqFj2n3#JLsXgf~1)+@h z?1@+zop*gY^rAk>Cw8f)u%w_lSpHQFY9$@;mr=*)m~ZeuLuO>P)h$IhY9?{ubmdYx z@I&7s_`LtqQxi?ra2AX0Uc4_B+qI;MTA})274kR^ejsoqLMMj#56)2ymGd}IXhehe zd`$Q1eKd!b#Z)6B7*x5sB>&B(C9g_>??R6OhlW0T5mt{2mcipc#_5fJS_(>)?Ct&@wONImst2Q?|auM zmZ@X9)^;uhfAp^pKxj3Jt?qyE>A%`LA8JZv^tQ3qpE_{I`2#IjrOE{7bPGT4C!m%| zZN(PjBony|`nfGGZ0a=a10vSy_Fz7S7@Ad>yRrLE5g5kRPCiY)%DCVcK@?Eh&PZRY z9#_%~7HC2&S9L!R1w^2D*=vJ?6k2KCeo9nSpxB^3D`vEV^a!)nkr!;iDk2qM8A92E zwB0w_ibJZq--78Z*&L%6%haLx+NTpA5Zc98W)xK`hno3Sd3@N*1NCxqT;CEONH0(5 z1O}xYqKH>*)qwjzp3JsVB4!Q)5hKz?0F;!)YP!ERa(lztoL{T(=ScUUETg=vygPR@ z@L73S9NV-@=VsyH7^Qmjt#K;J!8-_tXtDny*0Dmp0C}H{Mo%QEiKV7l-mf10=yPf_ z$k5H{0JSbQ^$`Gw?^ZH&$V5~n*aszAVlP|XuKm}{4l=r438n^ieE=5bB(a?#E=*KT zwOF;@iqUE^8lU4B220drWG@)y-0A4uy-R(AK5HNbI=3j`p*Yl+H7cA+VEs!~(Egh= zltc7$HfvW)VQw1s&f}^@?%MR$sZOqRqV!XcUGGS#hRc@u4yJ1LiLH^Sy{!%DpQ{f0 zRx7OHXv>0(%EEGwR0I((j^jT2i_vufPQ4zWok29Maf9_8UGs8m!`Sbu z($0DJIMI_3(Dq_uG?fxf(4b{b*RzA1scC*@7DwB8` zVa)5GbPIt@@A@HuE_(28*FWMOX(S^`(BmCfZ##@(@3&F zcnchzd~6|LQYEon0xE)bZ*z`nH*KRQmYB9{2EHWjUvV>m=Z6`u-DVx59a~owobzz| zQ6&)c=A#y*e5{Ld4SjgV+q`d*W7lauU`&Iu34KTmrJ=qu&zmd#}K4_ivMh zmh8fiAIFhMzPbrDE-#~71rPWRwe%6TCj7NiBg9fqHcYVYK0!N~75FsyD|sI(RxMC3 zMAPm$qvE>;B)zMyo~LlL5>&e1=C`TcR2(I`Il^<}g?r(OL>TAx;nzF8-iWze0E@wBQ$2aEB6{_`vnz` zFiswRZ^a|0a@Nm0(rEJCU)M+|2E|Cb7Rpc61gU#gh!Z)jrhdqsqJI4u_tjcwK_6V( zVgpVTUlJwky>j}xGR!#KDoJsntxyE~w$#4XTG5=z{lkgdB0Z03RBKt&wE$A>NNrqy zyFAha{}LX_+@Cyf5&3sEb+K2R0mZbO-t<=4m_CH|@QR9e}Iu@3xeX41J&qk9dmu)XQ9mPFzwyX3SixT~aJ*y%R$=4Z7{ z@EFu8G<*ykWufY`SU&y;vA%SmBRgYzRyM#8$HMtv9X}8GzK8ZZNjNt@>gKT)Iu?Zx zRXnJ|`Fk+Q_D@vk)s@0oiu()Y8BQqKNzQ67#;8fEOs~X9Ycd@JnQ@&N$=E^8My`uA z3lk%*_KwoMJqGXG@u{0$actwgTh02!Y}NIP>Mx`=SqyLbj|2=7|Ea!8cj7O;fME5d z66l|2lBb+0Wh{X@==RQQz9RY`>b0T@p#NX`ryZHhqI0`j>#t^1nrWvd+qfxyL69kf zjjL7n)_hjHwq-0KKax~5A&+S%h8#0?zlsDpWJ~HU%C5h+@=qm;7DY&Y#jsZlwg}^= zmJCc1#qyhy{%bX-+I*YIMOJfkk8|bC)?%CD{)=VDsqcYghP_s?w2pF$N~z6}eqzNN z+Ult_*fUdA0r+e6bmzDgt&dvpi8j;ocO>vE)gh!6-(DJDh_O{$VT}?3^r!1Fj!pIr z8Bp5qPOOj!0q6u^Yb-mzt|5b!?6_Ng{`b~MXVC*zJloEnGpwL&}hY= zi|DsnPDG#*CS0ld0yy*2k{?d`OzYchv4YI_4ex(nec%|y+`NL$*Td3Xy@_SEQut@kx)rs;zgOh8234{HR@S8- zlb*<$bBx|wcb*OnSBn`EXuKMSsp>mpX=^>i&Sq(r4JJP@rUeUMgqO!ZZWItw7k&F~ zzj(E5krD9WFp4avq-iacQsant2(8Q(iW@S|0NM_sxf9ii%byW>4*pP0hZ22^;)w`D~->jpK|j%i!*vqW&xU5y-F?iCj4f)^>7p^~nOqS&Y*GSOzs0?LXa`DPEx}D6 ze0IA53&@#_bA&e2T=%xEJpqAcnAV#cupy4|dupYJyhaXeR51xVda9#emcsy$h@ixz^Xq1!r zCoAgsWx%?SB=wb(@cx~@6Q=}Ja>FTr5ke3TG!^4`Qx#c|$ z!g9H#CplNWJYTSbB52`BvQ*^LHkIv;#6<(J0-dBZu-7;}N$>I{b@MxwsYoI|(cQnx z1<~~I`_HHdL%N$92ZMfRad9QNBLJ$6+F!BmSV~FYU3K^r2aF8a-G&36sIhO*_UH_6 zU%|4T%BtI1TEzNWRGUBJ$L2U@=O3DhwI`gbZ>=rIx2GcZ+{YLRD8{7+DW5S_v!4?; zGYRlC((YeriskBD```M+a!40Cv#l8sdzj5ACOC8{7@4{DKi*x4DJ*CH+1NK3F;CAT z<5q!&3#qjS?BDn|dl}=0#{7&?JUyxFpmg{n@|_F|y_Rwv8Dhc1;yv))5L;hnxvAT{ z>{N{0eM(eu&)UF3t^jL*`Tx=1Ldnh;$Q31ruI4QEamHJQP^qR%pWt)7GcqH4qc|?T zNMezB)~_cyi(0Pxp{C?!ZI{>>RuL0y{l_Dx{~`Z|IQX+ralWm~($DK6`&Ov>RSD%6 zyl>Q^Bs{{mE=hsAqr5_};8vY>0Of}TasSV1nZ){Qb=|sC%gEzA^X6WuZ_vTeAb^lX zI6@&Ub&fn+*P{97W~$OX$4}VQRL3peyZk+;`i*|az2zk9mWY*k$lzTG+#`3uwqyw( zAyx!!@fKWC(f~>Lj?hMxq_8w?hpJl+zd8+ssK#yO0h7$UWJfU1Sgj$d>A<7P<*@ul zb*z?KZ?)23(rg#P*9%kErMN=&j_MCyy0F;o8D$KKN?h2_9I$y9!MB=`rD4vNa*-W^ zo@M?(7(57sQ;Q25AN9^I_Ajcms&@^%RS-Hesy}GbmphQB>yv0@p5tKy%EZ=RREW(U z3H23F5-dv^bpPG9n^%!{hejQYrK-?1E3)Ns7WZ|r^6Z7d1O|2VC_-jq02h6?s+-F3 zDlikr=O%L&mCd9TX>!2=um+aVh`Ku1lR?r{GFLtSxaNW*V0ar~(%1b>s7qsTV4Lnn zCaHy+lAr(Nck!ZJVQ2l|3R-%}z?ET+sfxeMvRCs@B(A|veB-Xc%9(berCKM&#comY zvl}cz%F(#a%lCVh-#ZKTWm0!D;-!Tx1Q`fK%Ff-JHsfDQG1zbOb~1e?W+?VO#xR-s z4f7y_E_p;T^7X%?h6F}_%ukEY$W_F|I6iRAF8?btyXoLk#;wF0yshxkRPnmp7>*Pi zc!?PkYExLlzWV#ktME~&tT#n!_p}*2)ow$1kI6+OLrlf=Q2*`rAhX4ht2F-DW;q2_DRPfV@Vy>}Iy4u&+-zcUNoOkxN@~$2;M4wZA4I z`wh;wI08mBt5vp3vA z@VktLSMeY*g&V8V$MG11td)_*u550cO+jUPJ$iM{O^k~;XJzzcb?$lSjY1PAUIunR zI2b{iklBwB3uUISCvBY#tmkOn6+K*=CLXN*p&0HTlu`77clKG8ZZ-Obo||juNF%-e z%91cd2NkG@&5PViCQ#8DYvj7;T-LHDZQu&qqbV*FV=YY0DXjuYC^mk^LqC^ z-1Dg-G3}+niSh^9@KRFg$=#DKpPtoR^R@?}&N0tU)!m@nKWM>%I}kcnz26X_--pjm zUS7KN&FdJya5vQE^UtT%*j&@6J9-H4e|@FmLLUO``egQxvqs;O%~LQ-5?Vg5EtgU| zj=1YbSqDM1UH$1)-Q{S^=F?&D#)>YjaKTQ75mN@cbkMpUNxl~l&Cnbco=_`x)K1Ct z`L|Wf`KEVnL#yBDJ3V2OXKOoILN?1=S;p2>i9PE$ubLp`?!(*eji$f(s6*A|Mdp`tVxbKq4kEL zW=^nyU#joXV^MM9+CHApQrC{*TRvk4lE-F+Fe8ehc$Y2hM1@G)i6v2?>ZkJNq!wXh zm9=tU_Y$lw+)p520MuNQox)6+l-Ro6a;85zJI*k&66n*AvBQgkn$0Cp=5{j>{Hq!r z8ohAJbMa()+q6o>7cE0W?G@&g7V)@MMZrb+4&BF)2RMZcB~Ce|IkaNSRW7HKNM7a_ ztt^A#FJKg`;mXh#z-!C$*qUBgEtJ~e>F``XStm;=`ja*HGNb*TP; zOT<9tBXPn>|F7hPzD8H#88J85M-t4*3Q}gmPWbCq!k+;ip!dENOdXCJ1!^9Z!vZy3 zl=3UDKzCjwC2Tg3>Mrdfn51$05@pSQ|>W;j@WaUJ6P0obBty+IT4R-}H^0xmh6& zF~+8HEL#U7HZQtR%nJAW>b-vtx`EkP3A6?-_g{tj20?dQH*l?%d&i-eOa%r*!~wz$ z25iH!T1)kJU|OzDps+=Q2)fR}S1-cgwm_bwe0^t(%0@yonY~`$Z1?aIuPj$#&rs|duz>KpvRPE;17dWm+w&}+~=d-zvB=)BWV8U6kT zR>m$kcTxfcora#Fjfb|=M-kRDJ!bqiyGhw#%{hoA#dY&LYzsmDIoOuhJ{PqS!NkEA zy>uZ0yOx{QFC&5xeywVQlO=7HES&*9uGy5_%d+@$Cfloggo7N*2Z?sBAdGZ~D~k8x zkuS;dn3)qd}Q8z%X4ee9xdtqzX%Wdw6~vD8_Q^{NXw6$A?v)y${+*&77~yiRIk zd5KTz$%|~_o@shmwU>qvz3|TqUGt-^h%(`4(TxdMfP2eplktKW$t2Q1JOv>v_p`fI zeH2mT%$t&%V(L>$ZOv5tC!+Z2Aj_sC*N5U-+r2jc(!>u zpR00Co@en66!Y=tKv~$K$;Q^pt@v4x+ZXV_vZ86?SQ>a-nFv8qB6EXnUru(8G%Mh< zmLUCs9_07Xk2cNfcd_YB>NSf_95kNg2?H(hpwNB^y92+0-ZN9fp_U&ITV5W=Xm#m2 zi#x~h4GX9j8hTH6KF&FwkxSj8e=_>a%qXe@kocQxp1}B!pURz*<$xI}sr;D99;lw$ zowEJE!x6h9cDF+m>{+0xxAa}&f#=xj@T(T;Cu(}> za%#jb&w5^qQ}I|;s_;J`WG~4Nx~1ogUeQ{m=dZI&$=6F`T@+$9hhm_yB=gL_CGclz za`9<96>^DxP_`jGjiKXS;f_*6u%~zRIuVt?hcI}L;SF$D9bdA@G9<1m(17_qXWx7( zZJ~H%5|4V3dfav6`PR(BP9H79f9W5qKb;SiI*c3}*IGUBO=TRU?*9FrXBrIsEhz`@ z@81(TsB&z~+<(+BUZV9WYoXdbH1wMvbA21U$i*$a?_khgUIdihBjg*cJR$7W8`$yR z(hRV$*ZK!e-40(n6CMkNWEZBzop)n(d`&>vvpVE%vvgeoZ`~{)e=d)Qxul+JpANWw z^YFCOWMBB!zn;T1$^4^5^|!8g(Lbf&DrU!~xyIZgE0ighPMBPTrV}V<$cF3hYu=B~ zbpx~(fhkjSj|^`pI*N@Gp!5BKOCkdUVp+KUe@>*`^o%uCBYxA(CrEa-?ru7~nzka8 z9|aTEy0(5N+%)!QVv#WP)wim;J`-_Si~f^!J%_UAOLqUzcD zJZ>H#uUbdJ$qHOcrBZYCS1A$sR-@TNw&~O@1imkKbMJ6)`)kecu#nFAE3UmO;yPaJ z4$~J=$F`48oeJW*&v(p`9nXXkh}D{oyYX{JZdAvu95p;)K`nI{?(ZxZSb9{_W@>>x zv;3*Z@XWH39bO6xZQeB-V4jID1wYRDn9&oTfo*^U((`GS9WycqNeBQaLd#uH$gAm*T6}hJkId{_~2{r$kEL+ z=!W)-&==U^Pl}}15rcBDf4}XYNCAz#H3>K8&p^Bw5d#&dw! zJwUc`Bb9>|ldS3eEN$*sCA!+6LRq37U9)LxD9&}HI{@PyOw7ZsL?&Vqts)gQNG@?v zj?^}(`rpZZSC%UM)RaV)RG5+`6FKIE8S_8M`u~zRTzh>(RrfLg%r8aca{;46I-&u4 z&bYC4*4{`}^SkNtz=3rq;596lfrgtPcrTT2*>^e>9W z2t>eDxmQr#d4(?C8UCM83h=2n4Hu!}eeG+0pe)Hw-Sxpi#<+q#W^1xAuW$Tm*P&5uEDD=F=}MVZRjMu|A4L_sUfQc`vX5=}4H zomSPx<#|_gV*-bEx5|sjspJWl19y`;Wu-$5D;!2hYC=FYXzr!22H?@-*T@D{&|iuN z#~#!@x+OJ0Qk4%jOP?`3lX-NTrj z!aoLrtbWIMEi76d-SkYmoY(KkktEv(YTqey6e-UuK8GgKM|F7L$NG}qOwLbwlHU^D zK!wo5-a_yL^lzuF8`UaZV!Nh={f`ruc_-)K}S-l;aY!4v>j_Nw< zD|0+6bxn&b^)G}b8OrHci&;HduZyA2Iagx=Ry$y+0wLt1?ww~Fhq!U8Ilu25Ujpv5 zAD9}(xugaH4e7NnxZL4jz>y;q=!x@AD9afMn(w=v+Pzuz%jt-7@IFlQYxzT zk;%c9nb z8qekevz-q6g#Qv}0pzB2AVGI9LtSIk^~31BR(S8U8^)Y*d-)};*3y=l>+&> z)vL;%xZgtgHXwt!SbRXaqQ%}s%qDdE0EStAr3#hKceTCX`f69Fw4hoK>pXQmzs^Z6 z4w3mO(p^50^^Gw1oR9f^Yd>92EoLd}iGR;-Zic4Bf_*>ou~ zfg>j6d`rn!Aj++sgXxh0f5W01^W66;x_{#!u=e&)W_XdZ_T&XjyHT^c^j4sgbN%%%pLjMcU^V54@yg3Myt)!uTYIYFKJ^?w2N4rwvBaox``+)-FyC z8E*yvCnf4a)7$7cea-H@HtPHTQFQMAO#c5LuN0L~Dd$7-Mtzh+b7pjsq>SWz3bArN zv(2dFkYlBs%_(OZIUnYb!{o3z&Dll{W5ck`%pAVH|G@Rrb-S+D>-l&-?)ROa#5V}x zHV$7!KWw`%fNAX^iWL$AxQ(r+%2s;n4RVV~xCS5%!>sm*81Cb&xx&`AAN343#^H~_ zi5QkSs=_X1vVQ4yut>xRtZcjCVIsR%nf?sRI_ui_I|sOAxqT$PxsKXAoxgS%*NPi} z3Y|{ucrGMxiTPt3`;<7^ZvMXF6lkGbV;TH2)EAL`1w2KwU^hJDDMzGlS^uKn%>#)* zzV{JLvxHG;xca9=f)pi>?GFGBwZM8dH||w*R)j)OhzgA7e;PfJC+-{UTs6sz56^(uPTd6Xxb{7x zw&_B9W&YdU{xTJ{qx1|0y@GJOJD$ga67{E|in`FnS+EePYJ&MfV^aSmlf!|bG!8|+ z4B4)3y8Q^}U#&iG*#uhU3WEGr2g% zkjm(k&3m>*_(#gB)C;HzhVpMe-x9Maw=Ep(I!jGw+$Sil0M`W=io08jJ=MV+yX7ld zHg{OB4p+VI!bUK7C8QNqyn*-~z(RG2vDHUqig|_I{bo#mtZuFh+kCO|yg){4mu*M# z59nC@_}=_L``>cA4|th$wn!-SwSQ5m&IB%S>50*5Thf32(4$zA4JFul$}e2YGp+3;u{C@qAX{)H+eP_x6h+GS~eNqomFW5H@wi_(|fP(xNa!W0?t3|Ou`6b8&`rMI^+5eRS zcn5r1{NN)$;84m)wT%11v?NpkcGREx$Z_Q@G5&V8^ML3Au!MO(Rkbwn6Gl+c`e)cZ zlFMp>g{JsrZ1wk}-V9aG(LllbK^~gOuVoPopT$D5so0#qb~(}mDsR@JN|T)*`le&B zslfVP{SP)&&JRWYws$}}!p*13a#T+z?n$l)y79&N#!ThUw@ffMGGwa#tbL=8`?S!_ z{eG|HNsk8nx@OJPd$x>Fd@fXn!vKqu+XZkVB+Lixk{o7D0{Dsvl#!JEwVz}!n=*vm zaBe^MFU1n956`<#Aoo)C`kfagh+mxO6_~G|4BZgTG^9M_a_`l8;?PfP3?rPdB&^Pw4T+ zC}r=LWmScY8#Uuw;%_`fw9fwxd!Ni9a{evE-7}Le`o=R8w^bY%)g+2ZL_toB;`Uy} zhX?Dp@Z)7CgXto{`_98;8D2hJvz1V5gY1XO1*Bz)_`hg_c_C+G> zA$gC1p1vBOi3u571SNtS#;291L&wIq-ZWpPv~JX}C~sF94OfkEu)_;gM|oHcs3G+4 zd$-48U+J&(#374+LnwPW790*(J-2o4=68CpB_QVKQj;|w$ctAVjIa0 zsV?k){!08!@Qu#nR6KeMXx8QwENpG>v?mY-on;AhHmA0BZ{2B~sG-W5h5eJSrH0yW zJ*mVqmx?6Z&RT#lC&*Q2!&WZE$nR|*xI(B76(E8v+cyK0R5qGPM%auV@mQ z)~x~52_2s-+mh4mtnZ|@&g1=yA+Do)vy9J&Ly{SIO?pH;B_oCyW(?@h zXbTMCS^YXDi)-a~ycsOhUg9^)tR7$ui#;VIZG^poWuNAsqHSGdSyT{Ix|oNAe$s1|D~wH>~Xcr!QEDUm){3*urh2wXtDvEEz@=<~1)biv5#P(*c5P_8%?18U%~-r; zfkzhQ%5N`CJe#ltU~Rv%+59{H-|+`OJJ$`K7%D#H@xac?JCa8^v^50}D52=!!rxLJ$Sco;jm@g3Gk^56exoAs$|a3O8+TK;3#PBl7)*M3 z6!SU%0G_2}zaq@jFW7%ZozV)F^Mqz+^Xg@(Y955jJvlr*~QilnzZlfiWqIgzz1=)Tu;!pj=%oNn0A zLi&a-Qrw_epDlI8QFXO(_S=a*v(%;FQZKblA{ebHIE&FlYhGu(jp5X7+v2IK7E#%n zK+FyJPV`!L5`L&ZJBh99b58`7xYW-}+c-=@+o&&2t@GJ7&i$}+ijV1f`^O zUI257{7fkex%=(8)AzBR7PTLwY;#B%Zpt5r*dmXeR*t6f*P#CWxuMR0{_OGQTiw}s zkMAo+;wg+Id)Knph4QQ)EA}v57w-Iu7n`d0CqBqz-=oZ;F0bD` zDM?mT-H`8}2_ptmt~+MA=rXzyhAC4fjfLKt7zp8`8;-lz&&GkLrcU0EmpiL>)S~ zmpqk>-@1Gy_tVFJ;X{v3UAUorSvD_v|FQ0ut3O^7ZbWMfUC~|tR{Zw*I^hajg^YwqqiaWI-UKZk<XcB7xooSLU`KBgMMF*^*&P9S#JyzgS&3v!eaeUU zw9$Pb1v6-)!Gz^&kH(9Oj^g6sqAw92XW_camR`d;;sMD;=e9_9R@Uv{x?ynd#HQfu zvw%4I_3ZKZw z{(fc`pZ3iKiqWL2$_&_2Na>44;v2O2A)KCO2Q`) zITu3uwiZOZ__{d{mI~^e{v9J0Bww1>HfK<3p}A?Qo2|JwgOV*bFo}N((p*w*E6GEe z{zY&0e#Z`FbJvQ&*PdznBRE7|QJOFDzoC7gF%#|p;gCP&QL|_h&xc_R#MFcQL0&^L zfc}Zqp|=0W!&pzgK|bx^i=CG&8`+i0zHS8;m&UR_Ism|BvYYhzN>P?$mTjMeL9Oym z?+f&y5AvmUN#l~3!^RKazAdDtUv)*`kE7G2w##rAVhX861`jTMM{k%`e^G|3c0JCK zk7^z)l2v7WXUrBwR>Qc|Lsel~F&=KAHvb^7y*gESQzkxLlbkumoQIbk{h+>5CP+Ww z-sIfzqp-w9hVfks4upU0e&4jWADAYbHtM&#a1=Nkw4&B7 zg(})Fs9AV`zf&zt5|DX8w`m5?1oNS|2H#y)f>bF`q^`(y3_UuZzUDgOr zwa_E9c3ig6?&&YPD)^O*34{_!wfUy%Xu4t(r?@dp8XTXof0o{>9}r%UZKb>~NLPaN zb~WAQs0J^{J+F$ba#f%mKIInLZ}lGc`ZQZqVd9<>IKCe>*;q}p{+BlQ6jK_c5+oi( zJ=X}#lFv*jiU3y`TvvX&dgY)r2Kc^W1dwarzj52;5v;1ea+q~lwO)~{K{a4w-lT&} zR$$&~e{wR(-K-218}A9Al@#dRJ$M&@IJ=iIe+jP;lL?_0*xpOP*VxGylo4F?;99E{7S5#o`O9=8iID+lMPkGvbH4bP z%hipZ0QQK-m(#z7<+zeO#-@uoiT}jycM{OE&f~;!O|e=>f)$W>oq&_XlhM#xAPH@= z5e>B`KiPG`aUxm?9<@5;(7mux>fxtK&-3Jd`npBlc6e1j66A%EEGPpZca6mFbY0X8 z1FFJKJ_3Bi`eny+EjL+vE>pn{C`lr~koNhC0s81pfbTn$!0-wxcJKZmarBMO#vR6p zgp1eRvYm8HE3Bd-g+NRo^#NqiEp9w**eqNvSZ8p~nMKC~S0GPPT;=I18>1;;xM|;F zxw#xlX=YQ!`cdtims^xxi=_qfJQ?R8Dm3okASyO~#X(efeA^+*NY^V_$|38@xL#RL z(ely%_G<<{0c%J%_G`-p$5p~tCw{{$IJok>IJKhT~|A62S`9!U3>)~Q=e1{W{q6L??3w;3O z@tKN8tMHuf=JN3Rk9eFTCaNAP>)jzD{q&B};l}iVsF7Y??4O8O`FgY_#DhY)E#~^ht$sjmf5ER6q`nP^<)bnha`?!^Bbtj7>8OS^ z5}Qx0Ft`JmzuXUwSMrE+-uNlhm0f^xt&N&jnUmV)3%Tm8|9&P(#}|V0e(G5gav;XM zqIi&L=DGd)o8DND>}hSC_-su@rbt+q<7Qx>l26cEp3n7!)Sa74kGe3|RXOaq%WVQ! zTQUFHpGAC4-G1)Tf!r4RDfQ>1Y8}vN`Rz@{S&#|b(44&}v9&P8`~|@P z8@*eFNdh8|3e8w0t%Z~QV-=!lyQ|Q5_P)n+N}dPLN+u%Mci)$mPjycfdh&;G(w2FikNhAhI-$GB)(A4+$PDjt2w<1k`=HUr=Ciz1y7u>84`XdG zaN)~|xYZFg5u4>EZU%I0VTvw&4_a&UR7zj(m_8!kuC) zYBLQW&#$2@x6(*J9BZ+KhLBHg)(VPBfo>H94Z>tpWpesij6GB6aY}F}i(B{@Jja8d z`!OR-)7t2hGUJh0wh%?^ekNU+L{ggVpM^f0t|sD*2kYlFQ{utY&t8VG)v`r>o!kI& zBv>moE%P%b-C4E7ny;{FxuLYA8=2Ik>m!84+QCzdVQzCHX4SXQ`atYt(VEqXr2d?; z66yK?sgLQp(qgLC859hO_D=|*NF{78+nk?%6{gy5zzk;Ta1m8 z%K3K3liUdQf^@3$xr8qE@WMh<#96hgw2{-GkHP23&zALuV^BJ=`iPFIP3lMfBfb7T4^B*(|u3Zj850l*| zpd)NzKq)NsFyVX9*9lJ)ddmJ`t22s_C^Rr`OUjg50U?f?D>iEDKg~wdL;kYaotQov zho_zL@J16!WNy$X^(MQG`dK_?VsA^wzdCjFzhxsM8;+YFb< zuQIJswh_N4n0-X$FTuw;GW#w!z_pMy!_L&i9lTBVa z;CR&FVKEc$c(hk)Ov}?!Ea&_KW!xf}{YBY(%r1otZ108i_CL9;PTP z4{9=^^OgE@HTOKF>l;W4&bwfIfzrjJZ&MlidCMRloQFxtv(JM{LA5`-6*YVyE)vMXfCgL#aM0l0C_i$H|AmuqNIGYow1`MAw}j*{<9F zT2!<>k{pLMP1YS|CG_q)@hm3vWI9WqsAxr|uiD$)(-l7sgYF&I0uSC1mx3X-*@#xx zH+Q2_T5NGXR#|s^CbbW2LButm;F1c{_@bSlY-M*ELS0NA^{~ksl6zVzn_q1!zjL$2 z4(oKG1t0qa;wc}D;)!$dgBi5ouNF-}<-X2v&I&f!y@f8I&i<13NA)oxdiX=^vxYmM z(-{SOO{Fu>xtBw~hootN3T-^HGLWem-NQW+h`MCY#qjs`&kduC68{daZ1izS5F29U zO%h=46uZQzbtd42i8zQns>G!|9u)Y(I;%XBw`*edj^0>rYHd}Gr;EoeLRSXY`&N5T z>EtX=@=sYfgx7i3+16?{-_VHSuv%3m0^qC`$an_u z8^Gpn`*u6*^)c!88NyoFcMUEJPI$4p&V91-*c(JPYX4O=pOeSmw9~9hPK0DXwt3sE zvVIjYz}&G5|;`% zhrd@rYgC!k!+RFe>1KlW_OjraNSbX7FN@>}0x7FU;dbN8LCN&qhgp9$?kj><_PY`x z42iZ0NO?=YQZ~^x0_KYI*HhFXxY=O-8feL{Osh4gG9O@r_qy5SPwP429Kw@!)<4Lf zMx2E&(#xl+BdGr5w<9tL;$tiariVDn47R&Yzi3eUBIRPiR~?Sj&j3WtrEbzyX8_6u`qMH^thrWOHFHZJ z@>_(vw(4b{`zKm(z+Ee&o>a=q*5g1`nHTLJZGH8kWc;U!{bI`mF(F-$r+4`M=Ns~m z+CD-A0$!7X+GjZ-e2=fwlwNywx5PB^`bjV8c?lA!H^zS;R>%uJ16~Ow!gn5TTX#Ra z!~WZ(@4_ASyh)Bt-+$r_XyEL1hLYkJ_$aE6r4&^)t(p%#MUF0;>;B4=J|RW2#?9J`2i4aUg#*e*s7!vb}R= z<6LOk8|5z&(dBy}>*IT>L5*W%y}37VgC>k+RdAxJUiU6{h5Sw8E!kZsqD(zbrS8|Z zi~BS)of^*^5`F{h2)Raa5QEtFN!1Gs^sxM@zeUrDF491_TAKM8A_s#9rGwG?&HRaZ z{4y?^nV%K$27G4P@@d`4d_R0ro6-sXN&R3aQ58DC9rCEOqnDy9ei?W_EAch6g;xAh zTi#8I#wsUVgTgXdvC!^aa*H~Kbof2v!SHx}KSVVbT<=Mm@G$GP=^%-)-inylwN@eP zt~5Q#@be#5v9r|kko2xRXS!sn=yK2Qf1y99i8#SsTYqu%v-N}hb~M&+d#ndj&ADyT zC}c|$wN(@7OZ%;8%2Ky&aZ#7~t(0%!^frNaIErV%C)X}{4Qk2X_IdqJN@R9CgOn7XK02Ne1Ix#+o47gtii&c|Us-z2GnQ4BD5L1^owt$s zzP0w28)z&GRx`$EB!cX*a&%z!VGZ{1W3D|BI{`pfyJYEPV;c{2ld0Q+Dz(cm^E-sucbLN zaip;lJ~rF9)e>cZc!Gz#K28EVpo|;q-}#^u`;Wl0V1n;7;59>s(K^Slby0Ql1b|hX z*dsB?%UYMR=Hh~CKlq_LRrlNdvG$~v=BQB=hLIfmE)EP2$@fDHlXPnLZ|QK}B1#T2 zihys)Qos$(G1mTMWvHEM`maWefm_C$lDOaKcWduS?{5a)Pc-z@b>1!W4xpcZo7Jkd zezHN;59Tvy9f7NzD_~0TN75}sneW+O|i)L+*+ zDg6b#Og?Oco`d=|5_-W$X7wC8eR}b^e@PLpQ$Rq9rhy3Ceh!@J6gcYE;OErt$&9bf ziSco9P!`7S`g`^{AEae+#J+=3L;SO4#YXvK*FJW>4zqfm!P;t!K&gqRUfRu$>MwWN zwek!LSRf9hcn)^HyO!_p3;w#V-mA0^9=3E%gWB0Y^9AI+nV-`3WB<5UI9a_hGX=f8 z7g3;oLtSjahT8B_4ucO1zLw~Aanjl%>@w@(>TMBVG(4i7Tj=hGUA`gEefbaYvu`=t zjfb_*K;4EobF$xSe{}skq;uasUMO!OF)uV4$W|ul*mfUwu2H*#(qjzzWY@7fBCDoQ z9#$&T_{~=B8x&nXShB8>pR@ECw-V({*%-+hgnXZ=pT*fWB#?@xom=0dVhySieXw}6 z?dF0eCu1uFn^_ZT0jJ>Q7;+Sw#ImRH@<3~QI}ir(2_)IVId6IDa5lm;R67n6Q3b%{ zTlw{E)OG|73~S=!N%I$X%e zHXwwhL!$eHM?;Dz>yIdR@{7Fad#gmRldEOJ`(|FfH2{yWy3sp1b5Fj&nK(IeM<^u+ zF0E1p>$dzNpY_$Zux`pWrpl<^_>X1NK(9*zE+E;QU^z9qs-~bLJjv-V(r&M&2cv$f zjdyY64t(9${OhUugv_5Nsb#y~lOikalS3yLUanj;>g%X~+>*_b68w$Y-{BiPY|LxQ z$-M>cV)Vhv(5vI5+lkW}A$63ffctxb2Axf6<)VMuj=e2TU}yb{Q`-Gfn;+1cr-<)E z!X??YCvxJ!_Y*p7_Rh^;cTa6NOQ=|6wB9(thy79K3=0$Q%BE#O%>1;?!{V5IT`DMb z8eHKytZqfaUOgxQBM8F$8TE5y$*sTm4MGCMn=!C4%d4XU(6J-0Y{5i0bJoL86s2KU zWuU3qlQFn5=$xPUj4&;&3ada82b+yXT2PJ6sq8x_CwJ; zUhStUh^x^s;~0sCxR*urW@J;Kx1#g{m!Ac0YYnHrm0mWF0X}S|kA4TEEF61d7hg%Z|(<&TXxU=4s7t=@qkdbBOwNn7JLnl@94Gk-M{GYNx~Lg9L#L{K{KVcQe4o zvi%~QaJfCzZ*C6|=R#QR)mt@?Vt5B~i)2rqKZkut%*q=t;0A#+ez)qr;Kb^V{Vuj& z%jM9H@V)(A&jCD{Yc@D2SPLMxSiK?}k?_y7E{vtm6RiA93!Gk4OB) zzAQ8~O11yY$^P!|b~`IIL}tMGJBy!u1kv-XoUlAh_91_`^S0<^EGfo$m@@`zuUIY; zx(v~pAre$lCOeD9$d) zNH$#CksdhRq*|__MPx(AAZQC~rJ5Po2CRjXvI~6wex^N8T=}JNROh%J{yFJa?N@XD7c;CifOQjy z0-kOKeGXHcNE;4YM}Vx69AJOv{dL00d>l7C!3}gE<>1ist^Xs*56fU6f|XvEUKF~H zl#6jeNtwjc(7(2Dp5ld{_CN*D>Uc4_PV9W?#-~Ew$y?sWWg94L-N08}^G%mDi7e~UXJj&?zR7!XqnFJ1gSUb^T|;GGwY)2`>cj%S`E zX(W`rm?ga2k8A&CcjX9^mrFQ-`o93P`Se@_yBM<-R97lEMQK}T2=`*ny&QN!G+(p1 z5t~+0c)t3hLQ!eO?rD{?Ay+KERD{2mSkZb~D|Qp%M7|l~X)Wx)J?6UTQh3hozmU-J zrw82SNh&;H6gQ`^1kVXEV@do3(GXkX9fWFTd3Y+sbb9#djsBrqxT^tOLDytsJzRNi2#naPN zE0m|DF{sj@QuFYvdx%WUH0p8x_TZdY0_lJJh z2X5Vvy34$2scs%Sub#mxeC*l_NOIwYRf>eja?uRlw_3FXa<-Oe(NCLhP)x}ReQO)Qiv0Qu{ zWNC8f_Yi+gtlV!N`CBRGl8K)ExoN7_zjzoxhNhpxV0a@TJ9W1)D+--y25~2dsoK8g zX+o5ia0t@j-_(5>MZ{mM(Qm&bHNaM8XFK~|*;sxnj$XV1zy$lne^v&7BvLHE zpDeXs@V|J#ZzI_7K1#~X3k}_yn(D+4xMwlg;!>M5m*j@d9>|5@DG4%egErdt2+Cmd zF#SOt3-F5X7t@!m#!-OA={QP%$V;5J)^v!{YHmlU7PNh}EuY&&E^7Z)ncag!h4q$4 z^iHEA_WcIiVBsaVFGabv(om~D^nFT5*{ADl$m8dxz=Z35t-Oj3^BrCA4+dw5%a;mm zMa#FBzhw_ew!WHK+Tk5niMy*JW2wDb^zMHPF*HxFPQJ9V%b@=vpB{KqjKu(BHp@|0 zCGR|0UHE8s=(`yli2AoIQ4UIta_2>m^D6Pz#&;?`%l_HSiEf9$<2M%DVUjCbTg1#|ol%zp*ugV#l|zm{zlE{SJi zDuHcF7F#0r!U$?2sW(WX{v0+jG*Y8;z}K|*I_Zz`?ZKdBk?I#C4D;+qzDB;)z5dB+ z#DovlzSYduOhBZaw0!pVQ$2P+Q z2Tzk-aYEk@E;Y{^nUA+lT=spl(zWzkQpQDlefiH)5i!K+3$LU&X8BLDSAxi1@pq0H z=1p5)6MFyFO8n}9;jH9DLGvekT{-OU|JSw2wwOlxivI{l=l;wHC2x@UCqwihC^l2| zeuAWNdhQy;?7RQ(zAG{upMx`2D($#l=*+hIow(pn(b*GE`1Z2@V;thvexXI%7S(p1K_YL2h+*>VL+96!KYX*P9PuFP$oIBNFAnXyF8$R(dTkGAn_r?jU}EZjLFliY zeK2?Q07}1Li@wL(9a#>l+c2r?&B%D-|pe0QO`-zR7v3g6bQ z(Y_fhE{2Q>wdX^-Eha+Mzhb6utu3qZ{Qj9n{v$d#xO{5?I-bEvjIIT4I%M#Eoyl*! z!s2PkFgTIye%dtGYrDJe-odt16tL)AWP#x}W#CONj8>sYbw?xVomzpT<56`MgI4(= zJXXYI%YmL5-el0g)SWJ`J^bCsB=Un9#J5%xo;ErZ6MHo_7dRGxs0(k=2& zaBfR_CL$*``Bb(}k<|QbIx<0s;Nb1jo>FD?5gdW_hF*uw&5!3I&w2fxh@wY4ppC!h z78L^Ry}6IRkQn4W+wK+4=uWxJ`h#UH2iM6s6A)i;K|zgf!L6v~!ZGo_Ie^FE4cJ7m z)y7NrdH9HF)N8Ptt4?t&#)?|fyb>H%a{}b|-b|DW(wjkbq;~G_FRvWsB#sr0hU$h1 zs3`- zJy$DadmlWr1q4>xArqvLw}YQ z@t}piqT6}2GE24&Rk_@X{zr;5uC^B(uYHFsFV2&`4t@PLEO(>Zmeb~_ z!J&Lq(fSA8xpYbwmCzJo*`((amd8$<%^6*q`}H|fqCEW1zQr8o_E2;FtLmjntHhMC z!Z6|Mq#wW$+kZG&>cVj@AnPtRhrGTr0v~ZU^W(R*=0-{}-doJu(A-Fa!^BWXv3LEJ zcB4J%a+%ECwU5OYp#ImCAIwLXaMGAt)qxdx?0wLrhsWV!s#Of5m~ zwOy(PnX}LT-CTTqn|(9oUrLsB-yZGRXLTvtB$Bi0Ko_5UvIQos8;J}*}ytt@(SmMD63m!m;#$osV+7hNb-_t9*M71!*K)N>|L_B&k~Bt^A+O=1SF z@N|8q1|EWkM>S)p`y2aqpd{AcR(lFJ)VaxzmcTSyfzJf3Z?7}NK|9WL&q}-JR`$g* zqk+Eir)DN~%KerP7ZdiHujwR|3uSvjWhHbTdp_0qFIZwYDIVZ8Ngfwep;8)(ZP*XE$?k3LXRw#7j8(lH^S?(wi9BsS(dbo8{grCd>=E1Z)+xCjLoY2eKo&g;^&j3X|qUPvhq~0 z!lq7oMb#&cNuf2lp0u90n>#Q0egdC)-hUVN?+^;3y#i;~Z5Z$N*Yb^Nc0`S0z4}W{QkYt!XdYUfhsWPFziU*neJBk`_87>uJ+{ zK$E)7=a%ryJ*lzOoDx(*wt^ zJ7n>WUw1&sms1uDd9oMt9S+Yk&td}ifHG0^nk|66=(q_<-{z;R*hg4@=fi14Y*bh< zHcE}7ii>A-tMXE;(}7-}^MnF6QGH`=P5n<&ur-9efO32cfc**eDw{fkp$^k*PlE7R z0y>mG)RUD~|5UvJ$m%bRG%2he@er{HQ(NOTZ9;yBjq>sG{S=66ICbX=K&*ma=Wy}8 z@ELh6KsN`?MJ>$dT@07hu?pSo(PnJuIkhw~xo9!Yz0`J&i5E;Z#|tNz3z1>#$-8r9 z_=Ba4vcKv;k}OEOD|9Dw3M)UDtmD5*UE8v2k1|S>!)B0$Aoirr2$-px@IM(1!1@=O zwjTmmz*Jr;gBPX)^>h_4LTB4!&#FbJ@_e|sNY*y7Cc@eQ@$AM&7D zIIvy$x7h{sg;|PD+J%O}YcDP7czxLwV>HbT{90@=LgsU7 zN{d6Wys#UsqvppcqelVT$LzaFoq+dYFe{xNbDrbvm2)eVJg*=0diC56ol}oy_V4De zs&4(OL6dzk{Jlbqv))%5>O38Qv+L|>{%{b6Ec_FSo4!zW z#m}+h9Gl9iw!X!e++EO%ieF*80XP{GE+zZvw(KH-hn~ayfA$^b1|BW1o9w-5mt+GN zF2rxIYhX0a1%3!g5pmy_moreD!u%$dC*I{qn$1}bXICW-aJ#Or#z&`;w+9VHUqHq0 z;jKjeVrK8R%zwkn9HlNe-6KQ3uKR;rb~NJ?IaMOAC~! zc(1=dN+RFaEyvzAzkzr!_U+$!;RKh(H}yppgUr8L!kM{o)Lc6mF~qK_!`w@Bnq!%n zanzF*1j2bM+GHKS-|15*5Ynt7_Jl|6?YNzWyElA4ZM$`BiVuIOijcud0(5B|E8~l> z`a%C|saMmyr0w2TDs0o}=5B6VA2l;0VE2cTD8LlZUWD+A5L|1sP0>urwsJW343GV3 zZrAM})@3aOi;P{YE5J;e`%!|!-_D=$@=ruigTl`hP5U=0TNJx=mJ@0hG@{{rGr24L z!JH%&Oq+Di%k;0sO*b7C`u@&qUE1|hW6S?3-#6HB-<|^{GDazXVA+;m&(?Np?~zZd z>SpWl-R)hI+xSw7iqoA&esE!GObcHor)!WPsANG`OE*i`Lj*i0iNLG5s|hUi>qso4 zXm6#(*#x+VlV#?0s#H#&9{~cWDvG{Oz~J{VEmKEX((j^I%ry0ZFxJ4|)2;Cb{3qe` zw^Z`IlPYH{uAr(dNMH?R|D`?@RkH zbgs=VtH`nKC~j4tEc^P(Fx_g2=*oL8(y-Rg>PH`!yGtfagjEe;#$z%!0!QSBq0+M& z2pZ(!B^>_yE^kY!Rl03l^?R0xjbWGeKwEpLGg6;ove7Zr{2eY3{LNK9vbDxeXC=~j zQ8N~Q38-5ygxwsi41JHr<8KG%xtu?y^7wI2t9g*}+ctC3&V8Q6E!LNCtE-l8!%gIk<#%WqQ%ep_T9wki)Ax$l%(V(3wVa)eSa_jmfMd=%Q66PQ3 zDSWpXMM3f*FkE#?C~9jh;z;pezw?x;4FG35LxU$-+tuhoH5soA$6jwPNQt(q*nfXu|-uBR)R#nsKhUxF&CxA$=q z)15CCIDd&CuZG504Pn03Dc?|k9h0w(^sHnHl@BQrWFDwf{K*-f@UkEQCB$44B}^7( zkHQ+X5)nbQ^a^R6L26lKy*jWAhn=NXnCVcd=p;LwI@Y%z_sHXzMmxg1pbYr#9F-<$ z{SrxdCdco(rozqN=usZw!4+59D~s0`(vUpJvD z2&}-7y$?`IS&$`HkUk`N-^9}P*!F7uUHMDYUZeQspFoDlRaDJSG_26PF07C$5TpUP8;<7;^P zUNq&(#1r`E=_{!s&`BD5p3LRL_w#IT4#VaT*-&Uqz#_A=`hnrzJ#~(y0530J{y9ol za;?gP!z$L>k;Flg5{~UO3|>(~_>t6*xbB0K(XrfBhS(Xr!Fp(?+RE#E+fCaq{QaaA zc8#z+zG9mv#{s%@ps;uGpzW2+7eZbrnFfUCxGD%be(G}eFT6o=-8tiLc3aK6>rHz^ zzI_u(qD8%+o!i)pP4n@_nOP$;e3xW?YE{GsmVI*)E`I|dy}d9 z;CXWBn@q(H&2tU+HZArtAOO8BmRB+XYbn`a4?%i<-85O^&ZJk~n)^(77Ibq*4-e50 zX8NrGENy}TpQUI*51*@7^}$X&wKYr!Y;9D@cJHntkmA;p_pS9Rd{GXNQC#JofjevS z9MxMXZEVR%NT97w2~X`U_aX_w~zx#R)w(VZH!=9Yuv zg{7pR3deB9ekyd62u zZSlodX5O2lK&_qHYHoqfcahC}06TgH24UwUR@N24U@o-Zp09drqhAAfT6RX0)tzC0 z23DI+t2P2VMKGAG8SIn}ZDVq5ookbA^K{FxvI)K?2&t{j3YugyuUQYDC_;Ig&&}s1 zlRcHYkYzgntTz#66Xf+vu+nI}%a(2v198UqxuW* z_+6_(f7!%QJ-T87I@?Nq=s^GZc(tQLMZLCgK<;2`+_&wkIO`OxNe7Uo9_lgGK?cZ^6y}MJg z6TQ|eYJ*FFaFb6>Y$7g+ZO=uQ`F^MUF^7Fib|LaMV)bw)5IR+^FBQJpAaA}qVH^%~T8r~p-xfcU(wXPjSXk>4$d0@f&ONAg{j|xTH;8)p_~g@K+o0#Q|8spcR><)wEa;6z ze`QeTzN;vC%xk>LY_8nZ6}p6XD{QfH3tAeqs+N?ggD7Bh_Oq?BCB<@{L>&*Rd1e+9XE z?Dn!dSg{I84jvIHZ_@j_?`6;a;;GSbFC)*H8EDITTda8Z&KaqSrXsi+ zN{&UwPX=N(bF}1P^OZ)3frh5cr+`Nqj|lqU^BO!Tao?avn|8r|7Dc#tvxkhnB4?xp zazbSMQ`1G5jJZMWhQ!-zhNVLAKX*;Xq(!bC&AkYieWVsPZ_B0wf%d&iO}WvQgV-6W zkx%Z$H>hnS_K!4ph(JV`fMDhAE$~Q4;>f= zvI=TIRY-!CyFUhM{=WV-W54QM+YIYt(7DaCOC8olnH{mj{q>_<4DnQ4ReRg}ppL|2 z%>^(%k@epJ^#pr11Fe&n1}b*E$l;MbknfPnK}#LpUGuo0gX&!NgGmEN z?Xj@(1mMSwm-yJs-3E=7in4*{h$x-{zz4(-4=L5l?N>Y_P9q(#GgKC)w|AT}mLz2H zI}4%cIsS3dOPkBDHq830VcvL z2MZP)_v~|ZsB^dFU&o{OJ-?7tm%Oot0TcBv;Q>!eAr^!oG^T$_0PC%dEzYpW+1{`X z;9XQgzZRu&O;IT)hT=#@WbhL)jYXk`UtZsJf?W0i^XF{rR2gDQb>V3iydLJopH=j0 zzl=X9DwyU7!1}BOZBr)+QP@CKNUQ0~(0GRL^U_D{wvO;4{_rQ7zZCbpULt}wgYWIS zu2*i4H$U=K&s^yja=IKl-0viwD5SV!|JO&ijeFLVwp=V9oH0c%$^d#|r9;tdrd{p1 z*Q>+AELY0sGv6kjMP2C$IRXN~Dugks>x^5J{#jVJCdv$dyFt1bdqpz1l^#^8cfM78 z8~&DgG{8(~Ont9*3Mjr~+~S5PnzekWUBZYPT%&94{2yWO9n{no^$nw39OH@jvM!K{F2q8cSA%qapU+(+O z^T+qhH}kET{m(jU&R%oQ?7dgnYyDVop(#tX+v|bTO`iiBo0U{dGEwJI87xeN|889b zr@tnJ+=b&J)7A8a-|<~5x8s%7S3i9olBINTSXKG5n^z-%{?N9FxXpH4%l-BP+G;~VL-ZUECJ)L5mFZC0$m0lJ22$#NX?Imk;I;GBkH|80 zTT7sa8wueC7L7 zoc^>QBENSwVXSu3n+xggn6Bn!#Cm!`qsqih|G_?78($8|dtpeXv&!?%?x*ik&Fl6K zbl){`TOyV(#O_CmsDDmqUvEDwIqGlGSGxsusHxAXbPxH^MkbliSe_{AZc{h;9<$|` zjYrhSnAxRW^T!>`uN-%Kjg+5D`#$GuLt#`dH2&Z{!Fkn4)r_HK2c4L({@Sg&3q!ly zGby~$HRX-n-$6Yfxz>Y&Ymn@?iG1h&2iLpn^Lm8?^Ym8tHPO@c%{*36xBEQ^Fz~}5 zfx+BCk*T>RvZTf(oPqpw{Spbh)w?~9Hv@Z$eK}BMY!j$g3)win;W_K_GOV!4Vx{}{ zz_#+Ye_AhdF9j*hFn(9rrSDbIWQQ!(Lkh*am$hNUDP`%Cxg0k?4q4!G81&3W*-V)) zsDj=d|5{ejAn#!idd%AE?1sq$^uOrJvs-F+X1lHE46B;?ZVx|-Ejv{PX%~o(#&$~ay2ZyP zps)vLG$&6*`8r)__Dhv}qk3&zg*RzP4*DKqZ+E?So8V=H{MqUXO!Ni%3x?<8K0DDr zKi@nDN}1=%k9PFbRk=Jd97t-o=kahfxqDjL8V4&2L_BB{1az#z7AjB5KVhwv{PELT zhz%kUO%xDkBaaMrtvr|yszVr$cJc7iJ{5uPg-v(CflP}BNcMDOSkjIQ(@QQxA=mWL)*8{)_ks?;tA;+h`jmgV%TO9I+%@l zLD_BmWG$OPw|Amz{zGn=jPNaRp92BsyJC6SX_%ZxhJ&fJxr!#!UXx{Y_|GpjRT@|? z#OO#T?hfjH&g+jKY=(j)9T%*HX*a0S#6kLx}_ z(eo7&;l3Wi%L~)kJ#CNBxxzkUtpiH%k)K%qE7PtB2~Z9I*8XHgwPU!)0yTPySAqcM zV4nF;$QC`CSUv4!%yqO?`_0ufMJeI^;sJk1Altb@Xkr3Y*T=miMLJXWKYVOoQCoPcam`hxqqYv<(Mqz7Ku0+wyd~ zuh!Q~Qz?VdO(niFyA3XHl&GS&!?wW(o#DUhBBq$nn^X`xTXzhaFm@|pXY>jmLj?G++WT7kh-hE%Z+`Gmw1TptCg$gn!>FIaL;MYwp1o;0<)SrJp?9 zXnpL3!l){_@yl5}9M2vGs#YYs?8}~hsc50K-A&m|&Fk!=Eot#F%1J?*&$Re=hWuCG zLFNK)&m|5%3hOAyG)fa8na?(ev%Aw)pLX&8#TzH%BeH*9q~Tv1RC)*k*jIEbIMExutiLt=Yt7v zKDZA~Pla6^x$5E?6pJc#Q)S;#5i7K-oj`AQ1w0HGN69n5 zi+|jspQ?_9&rj!u#Pq7buX_x<5atrCu)5<1qZ2EFr84n_q5CpsbY|&d=OO<}`@ABa zaiGrx=Lq36>Ys>{kV}=aTAQg1g={66-Mh)i>}@gt6aRo0o-d+qIDf;L4V$1x%%*kP zF1RGPx`@0O6?BZd67ddq#PUmQ+co(enx_MF{e28tB2wX(6sSB~8%@r)p#!MMyN6-N zzaR>S`7X(N^i`XQ)Xxh31fZ4s0J-Atr0>A#h1a~O@C4z^>uKm7_eWsnI_83TGHxn8B$r`Eq5$s|2iZiBlGv>I3dI1Z|k1i zB_q?MC?m5|=8%l1?_>YqKrLT?*cO?6|ChTdCi9=ff0Q#4Lm)j7%B(O><)+-N>2^cE zOa15eu-$2m^M?9QLr>@wFCBQ_c`onqp#z^vQcjznPTjZX*f(<+hR1?#G4RX6ZjWJsL~iVi_POZPr6k?2oaaebcBPk0=2>(EzP1cy8coG*DMA>otNcF0 zlp-k?x$GC7NbuJPFT!PT`WO3ZuY$mWuEVN#H>AQ4w~dC(@pGmy)u-ngM@h2gH@q>A z4it}e6E52*^u6*Fne!PZz4~9C*iAn=x3bwc{xqU>?|COc0C*sisB1FgVDUJ1zW07v z(g*EFCaA#omb1Nw?ncRvc0BMR@9V>Ei~R0$FC3t0PP<}zNvP~I+R1V%ShL=b4tP^J zcqAjO zM|mfG!v6b1*oPfWT~};(Zle5T-aq$9VdR&?ee0Ez;NQi!TQBGaAnsC=I+c`9_*t}4 zm!{+`4#j(r4g?-TdzkvI`@c{=^R~xq?fg;gjEZhIl?V^#^N%k31w7wbP@3cPrcN;` zgM@sTnsEOnH}H`u>tS=>(wpyup{uHH_FJy=zG^gSW1ejxpdaz4Z-=wEr>2peTz zr}5}7VC8$XM@TJy@Gx+(@^wyRkL$PL`1#v~g8d<|>{FMUi4*F7t|o50-OIZ*LB9A% z@qyCKluuVo4SLnj>m|;29vH5Ck>K$a z+R+_)Oxx0&Mx&dLm!F~8TP7QCh)gRlbBQ~Tg(WE7tnuAHlyURL^6D?%EF(AOZV`^o zkKfz8wpI5XFCbv%bQ?S(aeoPYYx>b)vz(PP^@3x`_4#YbVV!r zsoX*3&*@*22WBIG{#h#ckqrOK+gv5T_m8#U2gdCf@ax-@69Re7h3*@PcSP#!$ES!M z^LIM+tdu=}I31~Y^`iyUrHyo5|2xEb?*00u$p>90IunyjEx?-PR!=qM>O-m~F1cIX zh_0CX8L;?P$6-Ag5!tG8`O$>|{Z{REklXTOnC_$P{ImC!4w{4eoIEde9R1;`W$XLR zr2VL_)q7!);DpAX4)Zrw@pb8;!PC}B^w|vOS(wSEeTe(528)mpDL{o|17P{F|M3_Lls%}jxJ_14@H_gzjI4;xlxurq$BI1UF%R9vp=ZqsY|OS zr}RbngZl4BH$Hp_cV@YXu7}jz7W;qDjc49O7K|PcCTeseTf!Ebj|QX^STgnAG(UNE zNi#WYrJU^;Cs&Uy}J>7% ze6;F`NwU)g;BUe6p!6TmaR)it%Y<2eJHI`&LA5`5^ao7q?z$tErl{Z{%n@$c#ecm- z={oq2JYzKbI4veN^RdAv?4_KOW(i$+#MZ8pNB2)k7+bqgmpuM+ipWbyzbk)iMm^RY9{v2W9x zv!a?{H@l@ZsLT$}2Is>%zG5@KIloY8G12)TRr*D#-T(EA_pSTx*&3VA?D92Fzp-UG zcQC}fWa^mRfZBTwob>ID-Q|vVtr5XJ88_yvOj-_qpuB6%t7___+_{i@d+xOAznA=y zg@vaL`_MX0pEi2TCpw7sJxQ&)UhdW>ooh~a20c$3O;)tZa!jhTnsZFf=%LScyE>iM zNgoe9`~IgzKi%Tfwt^rvws8XIbO1Q&Xa~vuS61t;C7R6Z)uUNAn|`?XcC&H@=Ui_2 zr~1^MVFRD+JYA6iK4*6^?m<2eb>NR?Qst^?G+B#MYqoLHi`*feCdtL@z~p z%e<=KHuW?)*Y}|IN9%#pM=ni+&6V?iBLa!r@C{HKe$|5m)>*|*o>snN4z8FXG;e*4 z*_Gn`zLhr{ai{cB&R)&7H@c;RhrLHzYa=1t;r4?2N)hSXeUk5gf3OOA46&*{_?Vk|SA9X&1p73sp%_&5J2%GUYAMDmW3{N449_Q|YBVGsl!w1{*yZ$bPRHy0&83fjD&Z=WzJQnT3=fsZ*vF`EZ)BbFgpY;M zjc17p+mQdZ}2;M+Z3dw+ZyN7jXk^Gp|pp(lu?}PqSsMpDt!PopiDK zbSxyJ*Y!xR#i!Y&)6qIF;x4EWEoVCJ*d)6qBEIWo=k|21|Et2Jl^F~qU^kNEA3GPD+|E7p?41Y;Dx8gH4V6uDaJhiSu~u zqpo@GNgu`b8&5B$rz)SJ3kB!@@#vJf_3imTC9bC$M~QmMGZ&8Pe(^qb@!j;>f4ADf z-t(I}p6(5NdoX*p!Fttf$A{lZiTV8rxo@X6*YB>#^0wT%^j_iqPpvmdk3IHzD&mX| zqQQ;SSN%Xy$vFpS=VcJ)r5{c6h}w}*J#J~h^dy;efMm|l-8jyPHW z%XH`AqF#jYbs$_PHo(etKda)kO61SGHXNXyuie4Hk4M7#Qsfpor856-#~`mIIQ{0P z$se^j{&NhT`UU#92LF#^@E`X-`cL9N%34xbRNB|^>rVTfQcTjr>`fee$V(V^e`34_E&u|1U>=f&b|LbAI1s&rkn<^WQT3e>9NMGSS_>ec$%~rG&wa zhY=jUjLg@Adp8}XLo$zDgI)hehyQs0(SOhXqorn2Z-`mi99NrmzQo>qThfzyXHN|s zjV|VH2c(VsRV&4d*UvOO+`TX1%l?C}Qh47J0(S)Ku}u15GsY|y?B3GIgKgR7^Ghm5 zk8ZZUIm|$buu?8c2o>u|CsFGQDB&7G$|dlLC@EMp0F^Gs6C`4SxCe^jw?U=jC>})0 z0E=6sB8s#TCD4;@0I(=YjC6q_rJ}+mEGZGS0hO@9A}LTZL=dd|;U#lj($y}Y1Py#j zkctLS0x?u703Sw4L@iP~$+J#C4~GP9lDJ5~f&iwS)Du?Cp#~AKpT*pkAduJ(-$({` zdS{PBickBC7MOerL93jHmB>hYf_#Y81g^WqF06)1_85z0RHsVH6`QGZtXHPtzr%mh z!%}MyQ7k3Q&xja08o?F|x&DOL!mmE%WwrUw#;;K7`3VaUe$G$!T?RJIf>(|hnThHj zQiOO;FzQR#Ly_t0(sKlPA(TZRa-BwlCIE#wYuT*<3(^zF1E5S~ZO<-d*a$ch!kEY7 zkg?C9%%gXj?F2M(d2$?bg76_xuiiC_hy8hA@gPB&1o*WD%_4L!CQrDH zv`m@zj6_9V4UA*@>7y>op9ar_+X;*&2xut)reN9v%r4@zrgw;-ALT9WU9@YrIIG`VVv2i zVCpVnPOp1|DsGjLC(2X*$5+UxkC^!OFc|JLAg`|_|dSi;Z?6jAs10>QWkR0C8ut62Z?-ur(YkMxIAg&a%jBh%fG+*-t`N zthq;+#`Cc0jptR9p@>o!zh=qa5ld3QfK8+-xtc;C;VZ+pAq{cioP}avCoI0J{rz%s zG3XOSUHn6Ef)K;m9;h|5>#bimWa5UI9`gEZnmyqlBjjPT^%vqr+IIPc=IGUQhWnRv z3!1I6+yZlr???7?;2z2%yEY?T(!KoZ#2{~Rd87PD5C1-%`6M)XxEIT%Qch(HV4<5Xc>4$TO``FoF9Q~`X zgKH_=g|8InpQ&cF?+_Z2NN07+^ya|V5);Lfb_D+wJ&eEy z$2me|+0eG0QWHH`IN0}twA##;D_VGy3-krFuSJRW*hvqXg|!46RJqxJ={NO2Ey^Fl zo@%8Vf^)vTw}~tL9p{-CK!}ExqO6+i}ch0kHfaqJu-6g!UmGA^(s<3 zE>Tkb^8*nLd$q}i`a6&#CVw6blCOuRr>LF+qzk-j>Fw5{N9W}u?NR6u9XfCn;Em!B z!0LoTbxaCUO09dX4&DC~E&1VKj-0X~WzR9U7(}lP=9jB>?JsJUS-PT+8Fcy6o7b7+ zX%w^C=b7#2MTw)4?DPl115$~>fFh?t)+C3K9kRx)R7a)3OPw~1_|g^*s*Cg*x!Id^ ztcX0cidd7J<)2U?E;QxL4S4Yfjz~U8!Oy^{gok3DEJZzW*_~s&7UM73jYb?3SWXEt z1O7;Iu-i6Lt*A4)f|G(bJtWp@*Bw|(hbBRf+p=1LO@%ayhQ6!%D}a#~09NB!?FrQZ zA5Rb>yc8-(*OrTz_(fhd83Ppn=n#B2=Uy2WX$IeK3e^-#m+Mt?+-vz7yZK4PAi4t~ z--=YhHJKPkbqaCfF4VW$LXpgjyS|N{$wSr=(m;4Dm3Lk*23b@DRS>vdix@|xf`9d< zq|IfA(qUuEQS!DhVzu3BBbp85nB&rNv!JoLmK?_- zLQEBAwNb15^n7M%Bz0rKc3HnVnYemuCpwWgNcuG)cQLK;SMR${w?cq-ZPu8(Egx8g zI!=C1iR+9x4){p!SUFma32cr&WkOJqs&qbBFvu2&9Zeo>F)jcstu?g-WwvjldXpCJ zqZ+3`Q>L1%Ol~>HD{WP=gBmOS?pG~x+sQfthH-h#q7k0M!hC6KM?ata5e>C$WWw$W z?DmPqVR^{Tk9rBD5Dx%9dOb3T3MAbWkl9Z{5q7Y;iSz~7r6Iyc>Cc|^rJ~5YM2MmE z z6i=>$CfH=wXn77w6dn0hT&l7>NFuB^?&9vCO0GA$=csLKcc-4~7ZrXlPoTkc$}ZdF z4twV_4M1H*Q*k&H$1LMfaP(71_j#vIbd5hQyslL5 zcV&tyZY_2rMbIQy=Z5U;ah&JH2|X<2kK4Kv#Tv6pxcaO($4)xa!bHXy*eVUPII(Gv z0wTQws^MFHbuIA5H?lz$O|`t18J`@mkujJV)kX)_AWt}$fE%*2jr=7?Gsv1D4T7iAzr?I1t`JKbXUw~xjp0WS ztQCc|Zfzt^>M5<_%(PGAN8p{(#RhBE!fsB7Uw9t!SgG>wnsKrYT;Ocj8ycC5Z(B5* z0I3uxXLe?$0U*TvH8CPLbH73Xn8>T48s^BA5T3473Gm0tvWDZwyrmRpo9WC>qngvm zg-;4|k{uf8&)}16S2*^@Jo-L|AB^3@CgJ$GQQfe$YotHo}t)}@4~TzO{JICh~DD!0|8a* zkj~0~m#II5>Cb&}!Le(%HkVM3QD3_zj$%#!k+@4_z*cQlC*#=GC&=T*v%$-KVtB6V z!iH$i2rCqHc%nRp7qtUmM%U|?osml+XLsgVT7==xLuHU=hsv`$M}n7efsWk{%AL?( zxZlkg4*|nGEyUGsVm>dqkmKI&He$EvCzb`raq3(~z^SbFEDV``lTnP+_QYh{fo^uW zd*F*HmIl?ku4b#UuV@p7F~H+eL&+r*NJ~I1zhx|7B+~Odv1S$ai!uf~&sqP&KgSHy zBycw*j^PlozND2McS{n-D|#QX$m-;Ja{7qOv`B3>y3}N$LgI^xlI+LE1Yc-(_<+^b z+PQk~E$e1rK!T^T$`HY*ck>!-K0M8?d2cE4GT@egJl(O4G&0Sb4!4VJ5$bkFf@X9k z1yup%EgO^ajbRCE%K4NFT-^_ZZLHD&zU_oz^oBPb(VSYOnmO~uX~J0@^3dRC>jMM6 zB9qRj$EO~2M*enHowG+@TTcp(EX9^4OMfNyC4!fMZM%jPu1PCQIuBf68&8Kkd*`FZ)fytuWps)$A&du_7Mp_!NIZWsja94vWAo6_!rWah2xPwx z6}e_pLs>(EkgVm3z>TW{;McIvpkrK*0?9Y7Ue=QyL38 zIT}=UG$V>(7XD-=p@#E&f}+pFw!fh2vJJHf?~p8u-wc!0#Ks7-b0o@dc7$EtJLx=>h#LIBlmO{Bg4tH))xL_VBs2;QOK@G;bP9>;T`B=kNxMW~E?%b0+!wJyh2(YrR+vIS4)^ zC>n&dI!coUp^uKq^$WyR$gSTY!xBv&iT7AKBLq|nd`2!Cba}ko^Np9?AXb`#H7s>9 z@ixpgNAVf%fRNhVa38!!zx=n&HeOZx+F+llgDQp>ntcXEodbg$V!@UyY8l^p6y=yn z?h5mQKInG|hp?SPRVKlY!e!q`U`M&WrXz}$3k|;jE?B)UH0HMW(sGMXRmqfoP8TSQ{*o=NqHmkc ztf3pRhZyxPR@keMH<2iP)Uw1`33{GsOg3tmoxPhB0gQ-eODQuaAByC8um-HOpVvFg zzYx28V4{?~b;(1K{t5oKyWe5FYdknKw6AgXbdBV%6M#y-QMLc8wKD~JcKpajXxl=- zn$Zd!&^5i4Wp86a0F0lw$0IMMn^-`#Qz{E#YPJ!(X6kDy$>r#P3Tq1PmhMXz+*N<=;?W4tQHrF&|?pT{(uui3gBc+?!N!1`@Xt z2NmMZkfi6N<-8I8-MP{~f;`dL_VPEPl7}c}0Mcj?$khwt6h2u5vR3D+VK=(rR^rhT z3N{PU8DWYPYXL2f*7nUcUIXKUoeU{4?A?x_xJ?j)yQ!OHBd0>ZH2j;91d;j+iVgLT zp5(zY*qH11R;Woce_I^Nm+h{zlRC_ z?PP1(rEz{Qqt#W1H|&>_4?!iiPu?h8CfG#+SS&75YS*l=82@W;$PcP0YAvKvLMz+_Je~N1?Dj`cYk+Db zk!X2P1(tv@!h9C$17}S&O>Inaq@^maTwt&f;_$Un#4PoAS<}`DnLb7i}xkH zJ)(sHom*n&k?8F|Uy*>iZWVDH@+=8n(z7;6tD3Aa?tzAep_ma7lDdfFrhNXgabUFz zHP{&(OBi3@Lkrq3d|FHba2l@?m;2E;!Fn3DmR|22wrNv|61=6QWOWmwhzoSC5mlwV zqpxbgSMSTWZGOft;f3%EKTZ3~`=pk@V1CGXr%%{Z`c>pYk~i1&6V8(REfze*iss+3 zj1*bYjE2}JmCIZru;d<5n=}1FzxHZWlLWe2KLJP+fD0tCRjT9SC(-2TrkB~2G05-o z%h{-cbS1>yhQ%ehs{~~bc@q{pvEtPIh7x9-#KO>%dE6887o63rUO?JO_?g@e0kpp- z@~yfydK#XMTT-8}mK9cyDid4eImg~oSH}YmpjHZCHo=eu|8RUh@>i+2VOWa6l>dth zqDPdd#Y8;y2G4n7BB%tD>Vx^~ZyPsmRF&^$pB|zPBF`|9x9J@;1^s-eqe(`K@%aFx zMVu5O{_C0K{bfiGq@2-cJh=@15hAQl!{D^A+czM8xH~wA8eTT?5aj@M|6AGV5YGI{ z9riHI)r-GpHCde#oX>3Cu#CK051k1yC*GtOHR-bY2(fE89Nfn}>;fxca&6g0x-`{< zsSSnp2W1HYK+pARf)qD!p>~4=qq-qXKn7HnuF~FDCZ8xbr!=j_y;`aon4ElF!pd8V zsc5<$ko%F4{v~N7savP&i2}gU#`9mF3XI3RxNtJ%IQp(r5vDx zE%|v1`aeUx!yW6o;Ge-7e1fOs*S)gIH5c{ChNXRpHZh^wKe1|D!h1hrqL%* zm{+cZy~s6vQGiLCYBfhQr97W#ygXL>!Y`L)ts7MBTGQu?JonA|lo{ zO34fGa{k{|&5a9Nmj?zyCpJu*nti067`KXQF6hA0ns*ZjTpnj?q^d@LOk5au`n)Q}^I<|dhKMw(Il zD6g4|oJKwv2eo0U)=aYUQxG|0Cw^hB26!;}X{#_Bi4`qPm$j!fUBy8k%&z}1hEm1` zoZ}Tl?#^jcrx?+qZuf?YY9~o^sBUOA@*3RN13!8PJtUYUlFY$7Bo-Bb!ON80qqyT` zzKhsfk=G*0$<901+BGqtITGI)DlO-<@Ij5@DN(@!A;LynAa?z;Htkd8?Qsn zH9m|B&svTd^zG|B5yoij=ia8h3I*iigFnlC83BxWj}sBw;Wt|2p9lg-mhf-|^0U>H ztXh`Kh|v(lFWhrAD7PuC-%YRg3~RDE&R4vJe2g$zXN9^=_$glf3LU6ZG0}+J-YdY< zxsotPBRFs+WO=^Xmf3vq75!*qME!k~fBmZk_QGA(60bP=Y* z91W4xM=k_Ll3(*a=H)yErGKui8#^NoP;@Zn)`Et!?92n&pI{m;lMT3>Y60qZ3uSvu zT+j)W?@Oay`lRt1K5Iwrv3kh9b+Y@?CnGD zpleU!Ny5Nh`Pw&R-~V@3Eck`pB7x4nJKku6ec;*djP4=eZow&KaeFPN5#p#JA}8 z{Q+yxynmFBt|}O!pq)ZHyp|ZBLq8#{?nxJ-xN3Y`H;JLLd?%$eN4>OMC>e|SA_rl` zUlPC^A!I*pKikuY5aEm(px7CeN&bNdUxe1e@&pSA7j-D|AgETj^oepPy>FQij5Xkg z!zqX&GbM)RTV<=Y@xG!8r}4zve3G{F*~q_%jtu8uXg*B8)0S2KmKuBx9UCXCfDy+; z=MLarF>NvEi+*oZ05tFHxfhn4l&fTdp`XhKdiiVgYQ-zD4*O`G}4TparCQjhM?$FN`gyU>A(>}SUC9T z;?djCDwmOBF60S5(HgRr%D0(zd>||${8MW73TDHZ-KX-EQy5NN3->2HYJf1|^9>6@ zdaSdkt8dctYg@Pn5Y;*IV%8jFI-eB{*L4KU+O6sBVp)eG-HWg}$qth|wB)T5YgV+! zg@c`?Sj)?-B1%xt1BQ8?>5PwoktG0Ynx-2`&1ji<4Np}BnzK@Vx~}cZa+z#S56Kod-f!I218qc^6GCI<*kNZ> z`iw7(*S~Aml!~1E;#)$cMU#V&u{G*qE|zx%c#QRFHF&3cCvP_`&l-~)d3&C6jb$gP zn3b&!NQYg!>uW{B*9NpnYTF5O!IL;7lNx&DyE@Z~4r4$BZ7>IYU|lvqX5m2<=-t4h zrpsb$v1YJwR?}|I5eCr8$*jT8v1wwlv2OoY@4~eAI=(hE|%hyyfOc3MlQuJBve!6y=wKuD)! z^K*z&0=&5PTR#moD9vE?Apm3h`^D4qr8MX-)WDwvRpIZpkr+B=k}|OIi|a%hdvRR? z%L0DdY-`gJt{d?qSAtmru@PQc(m-{GTybl*yq(eOwvjcFEOJT<-`NX+k+!TVXcAmE zswyJ2XQea39cd{v!>ZupTEOr)43v8udf#~Of&#m?;8Y8{Hqd(~Z<3l-7GXh|Ko)Yn z*>x%o778m(4P+x&ctd$v=rw#`@N^l{Q#>!}3^#6pW%bvml>da8lMEIegx=x@*LwfV ze)U8OzDjWAv5J~p80sJJUtzTmQ%oJx*LyWbS3UK13v{@$c(V?shmkbJNIuvwza`LwZBVj7Tr zq_xPJgU)V`8H2kvc`V9%h+BV#QwKz+!{`nH0dEF2bVOyTVd*Kj)zcGtf5xOt*dQ5F z*-57vTa!B0T{892J34A*D4v;QApA?rH9nM53wQVixlOQvYR^R!@}3yLn!2MRgo1Pu zF?@d+Yax7jIc6+g{KkT?8HaD7R|hW2k~%l0?wdU0H&!)YLLTO~=q6tFaaT-NB5lI? zBmw=Vufm86Mz#hw2m}qv+lj8>)delg$sUHz9MBo_#?z(i66I!A*jE}07YXjX=8Csc zV<~s3yW;}+DG%MQRYHVtbuXyh$AHmXjHlvuC6cuip!gpJC3ii(;p=A;jz^?cmV z0o$lqiC`x^Ik_>0C5*~;_Lg^CCFeM7hU}K#5i*ew_|}8+uLbh2HM((8$UGVF>cLzj z5F}+s{yknvPv{+=7GL`8z8L}ma8gq!M%^Q?*iPSxjoh=+0!2R+V@*KPD&G}#1;v>1 zU)}rty?|sOQbSf&J61~+eNe4(xV1$^^stcR&9A4~jRctsi`yvrs%_uFZA5kFw+riV ztpF*MTNqH3JCVq^Y>XWX(FbAwCO591>DWPy{G99bQDQj^dAA`cQhC`f{vo+`Ph#76 z-M}U(Y&@iT{mo!vf9FV1&fcZ-kFoX6fU6t2DI?l`OOqVRvYIpQGU}dp*vF=Y6v~d( zu!8k372@qnZgxhy=9&Z78;d5Fmis5v4xnqD%U4X^+X=HZhn7Fg6f_d@mE9Lv`ZZlI z!DpF45iZrp{kf_`iwWv3Qz?5P<6D}2l<6Poo7wPPnt+=Uz!>So1fUmkKvloa4!?B@ z>_W+!yn?$o;R`g6_x6n@8T%6y;o5?e< z2^i4Aw9ot`kb;o*MYL$R-Z zwN*=t2OtJ0Pn?`g^NVC^2S9OLmN0)`atc*!t9lBYsY!VrAFsvDtsq}{RV(RjQbUBQ zgiUlNOgD{kk9Fp1i#~$U;;mZ>y}-0-3*yu3xK!qTgCd-LRGTc zD~mCbRR@Hp~rN*GxZ~`sR!~@fB8NzpxS-5Sse`R0(c&s zqbPDVLA9K~O+%?@78W$wIF-Kb_%CpTmxdUdh~9&_Ksz!~}wQp@6^{`fW|x*2W__Wx2I$D5o+ zq-aX&I3+ohfuK*A+B2j*PB{3>1w*&SPFy@#b+s*pRJx`SAaMFdu;Pjjp=fJ;b`gii zDv#Sl&mR3i8K^US3l0q3t%thoqCj+C*(EMnGy3M1!Sldf48IZ1-Ee6KCF(_Od<)0c z!`@%2HXWwshSvpRCL)Ov#9nY&hz6CZ8b}aW0*i1k^+{w{aQy_NCYV`F;BIq257y_~ ziJw9^fxDcFvUPjHUPoH0s}+i(X|=IhwvizX&>obf^aQ}iTdpvuncMYwAvG5Ypb=go zH5v#x(3Kv3oSuF{_KQH}KT3`UgB>6{K6=Ww`TBY3Sa=J!0eOK~Mh`4;_#OE*iFb$w z{V=lT()Df`LW+lDqE~&2iG|#^T%}-${yg6f266$bKwyN+In+`YFGpgF{!|L)58Ikq zk>1=p35I*oo5v~ZQR1J(Q2J#Fc@*mGl$7Q=F*-+Cp2;f{j1l79by(_U=L^ea=ylEq zft77ODahdk#ARP>J?mm`?9xh1fRZle*Mu-0xo6hFb*_TzcVc54VZpwcfqL66%NS_a z8|C)H&VrBY0ouWFGqp>b4WyyPG1fG*ez$k{{WRs@2YpuzK)b&~B-FqCKCrqvUgwfW z5*t&qOvMjW^e#HPdzsOm9jncv_`B$a+Zp8>*9OLtrQh=HO`Is@be<>1ze{QaKG`|G z2d>-YmyW!$5blH*jO{Jq#a9LToX1@ko=bP;A)w5?w1WFCdugyR^cHeU-p24;mrFUL zmU0T8&f9Ix+UAcu@>$_FHf!A<)4!hfHfkk#)}9t)<#LSn)3@`@y9*uQp-)?Vor#GFL; zWJe3|J6(X4cTA8_jyZoBm>vnE8QC_E+{8}63RV|L<uSnpapzs&IZ0rZEInU-SDj=1?ElEYEVLsxtOW z2!0yQh_VF{)?J$c*doc#nUPBcQSP8xN||>tDx=I4j!s4pT)@~Gr7k{0EGUA;wD3Yz zog$b>h*+tHXKI+Wtg#p~WyuC{Ar%`}IWn-hC3wjQ%dP1)H>0ac*yhMj<@f$D4WAdt ziUZ-x1@t|ATgIdTjbG3^#TTd>@;K~C;kgdRqMUqys0b8Tq7cCUbzJK60n)V~?f{o!k*haYA@$LsdU0oxwr!wJDIIQRnvX_#0l zi5X#4H<5vSsUyFf`I+(+$=ckbzV`<$8P*OMv!oN2bRG^iJuO@?sidO~9QqN4!=^@t z*1<1&UBgtPg@=WfoIZwVFZC*ft}+4`)+p$*2hT;>8tuqN<2W!wh&FblCcQl<2V)j& zQs4iZL2Q^8ADScY%+Kn__()$0zd?S8RmPiD3b9O=-E!zW>RuNgdRRF{BRd=rYlNO? zKDmrPuz^$>Q3p(kDqD>GhcQ>G%YTU5JV=*#xAVY2e#F6vi_juUZ6($)(k${e+iv5` zQ6)(=mKcDx;0N>GlDgo$%UQv#tEfrJZ%Od%3rz}hnthd=|btugNW( zUzuDN>dKe#P159BSW6pSh@54=M!)N@;Lh4M!Sf_7V@$2UrO&KbmCL5I;)VoIhj=d~ zhsW!u>_82Rna)k|!(q~D6&xT;@NCFvGi=L*Fi-8H6H<3!fSw8>G@yWe>J}lqEq{Msg#?Qxm=<3cM*28X){Ai>vjz1|%%W|D*!;`X1qL*(X z5*qN5aari&0~i~*EIm{1L=dVIXgV7!CzvXcn5#os)cyM4m4ZBv0V%sVTfWfHalA_!)&q|A*vVS*_$m5*9K!UH)3R7y;~P`;Hq&ndh=ps~V0B#) zuUG6BiG2zzi5Z~aa5%brC>Sx~c zI_@gPP4UMrP(EfsYKQ(pz zwdqBGPpRSe8J!L7L^rblvtDs*JMb!VQRH6^d56#yEYS+lVwy^*YLp>rDl=mz{x-gVtb>c}Y4ewLzb z!}M8s+kKc7Uovkh3p>&alZ(!azMs*c_+q{c2KSvRkO}MAaegyrk)}i2VsjDK7S8vVsFRCXiNMXKGWlb35{%5Z@`RVE1QZ8bJQMZ@%4HG$H8gSl zY3f7pZBegtWTU~SHMQWT1?AZ4FL~4_E9hw#Imi#y9B2M)SR%f;u!a}S?eorHNqv}! zPi10ioLCtzs=Mi4(WXXunAQoGUslI6YDWpGze}x8^yF|$!2LB&mY?6e8Wy?2Xu53U z&S6oV<$!L@agztzqwpJl^TP0M6C#wMinmmzVS=#L4QhcxHG^pLsjtU(+7%AMmER(j zKcQg5ve>4%*o?_nOl2Sc;1u&2|7_3MjXw3*MnLx57J_caCRU`}%hs~0WBU+Vm@!|0 z$>@Q1l7dVle>2?ttJowS>!+?AN2-<4&KH?M&naQgOX4%X*`Gh(0DGin?+|Vc8O5mB{12-sKZP7RId->MN zBlHl7izP+Aq|b4N5P|>TRK@F8L@sS;k~MXc`)1lgd^RYD13OP;|9}Lh{xn7i$2Jal z=aR|>0cuVG1x7JsY8HLolFBGI$0hUo6xj)rdXqrcR%r#diTWTb@iU+bwLWD!!9+ayP?!lH=S1DewjeegvAP zaU7o=VtUie6JyG-rIzR|yOU&dgp)(}0{{CR!hN;QF_+{l?3z)Y6OXvzPaszWv(gIy z?~S|xS(t*4MdX6r9BhfAU!o>owd%GQu6O$%mzPZK{#wRSJgg$pc0+Y`PM~v*=bKV! zH4Hf`!?0S+Tq+r@a|yIZTl}jB4}A+6%LC@pC0lxh+AC zpoumq1`{`SoHHsPDz}MO;_wC|wL5grfL~?W+BY;fpAw^fRSWMkKri6fn3cQn3=1H?rD)GN9#^;wy4V9nnu?FB@=7gkY_pzZ|GRV#$~qPNR25h-swFWGCQM5vyu5e`EZkZDuA5A(6B;Wj}BKm7X;|%vR&G^>c}l=V@goYxG{k zJD|Q5&*}cM#F>#ZlMXd)4PpE{KzhmRpAL1FH4g}yHS@G?pvQnyOH!x725YIKB23L` zf#b>iAm#b;w0(JS-T9f`rXgY(Tnd?=W{O0g2q)-;S^N{YeUp-_jB#W>|0KA6N)T&Y zsW~J#glj4x=MmWEU}f_S<1J&8*k8k3@Wmi@HS}WA4Z@S4f$+GrE0kVM9>ZneeH8x~ z-II*}6Pd%p)i= z8y?O63(K2-Es~9*zGF!}-n~U{PJu7%q=fq}EzYbOp_VKPu2~Ou@})3E{DHYYHLgvE zbjCc=Hj`hce=ze&jfTPt5Xx%Oj_N}D@XOV=*`|kz{9y}V%1R+^Y|N1bYsf`~IgBj% zCVi6eS=rVz_~R*C+B4V60>DYFq~Y*j6dG^s|m&_A14PbY_Gz8cqTJG8HWpTc!>IDoriM z&IHhMY65u1_)T)%#HC!@8Mf0{YO(v`&DPZjsCA4>Grg!?8^}~S7%+{AB#`N{ebG{|n%oE-=JCWE0B|xvvgS;&w+H_v^Opatvm! z>XXYcYaG*YYn*-d7Iy&Z3IancvPZ5=X)iTzzRncHi8mVE@P{RQ0q>RNmVtuFn~Bcn zhv-bAS&N`)#N~o|&5B?F=cZ0k;rj?L@E}~Ghg10EYfo}r-oV}y&&=(a-&`#AIBdER zhqw1XBDu)*Purar%}i|$0sN5|%Dbe&?c`S|5U&~E_RSWmBI+1Fy>^iXdIL*=0ap>r z)q`(InQRvCIzL!z5{&etuWo}ojTV+;T&9$H#!Ck&c9V++L~h{8mh0eF<28UsjpZq6 z`QV=nijo_N>8!Acf(zRb7BosA0t^OK)Fy3bt~-9W^lTZLj$g zZsp8LR_GY%&}sbT2%^Dy>+7Gl|D?HCoEI7PG{di@?=jfVaoPR22jXKJY&{1=UM=Y* z3^(W<_N|76p*H37N|!3qY`5IKM^?AMIE9CE@DgB{UeY9Q)?P0rR{?qbaLNG3UN0`u z@1CGup^gVBU+Rt>q|wZU*K*M%X40g8Gz|GUv&Z$YX&N7me=Ds}_ocSUxu_b*CWKcH zU}3l=5JE5~m-Fe|rg{7Z89TdSu1fZAN3Ih z+wHuBzJ2~^@$^|h;~h0esq80khK47SE;HDsIvyVHMX4K2J?1&uP*YN5Jq`qlO>d6G z1VH3ZT}OAZ>QNs~;rAA_u`x$%BX3fLNtRM`SqWv3m{^Dds57dfmP}O)@&W1?f$XqW z>`i9eH{H!J4WQ(0xH+6zD^nyLa@_6Cf6E&kEy{0oN_`b?$-!mMoqncHQGNTkZcg;r zvE1;iA5o8@#Yf2&0xOL*T-akvybDwi!fxz30m|?2j*h77SX8L*^q8v~Fw`mYd!XLg zr&t9Y>|o`}m5$}8{!}nqeU+0xSpC4l=Q^vGyd~b6{B&uCeC_ZO(OJT8iS?#Z>UP#K z(EN>1Q|o2kIQ@;-r%vfUnEa4>RsRk=5-M9?N~;1_2jni)w&hyRxyQ>@jTafDE9Mmm@6q&&e`>g<=8 z$k#S>A`%=Tu>($dxchayo5u8zw64*Nkk0D|co7vX7OpQqD9~cXPBEH$%EBbWdsXgR zL#E1hvSD_vWUu~S6@5#kpOt;CFH($m^+R>Ic$>NKAgQYz>6y-R6d3|up>2H|Ig4^N zZx&PQjc?-p=NNUE;s<3V{*$0`Y=-S+K2%z?g;c0;ucCcxU8HG+&lhNHo*Vy?06R~f z!7bMLGc^rJe`&?FWxeb2ib9(9=e1nHf$LZS5UDReHF6C1wHx-+9Az7|&&W!}wi<_G zBe-t-Ff=T_86Hae(F|7>kvcPK>07XO#q|?6vpL;Nl!@0J?u)-<*h@;J+2Ca!e8v~M z2$A!EEXdw-O1eue&I>!xy$m-i8sV;+TWAB<74>3{OSg;l!P0+1y))V0|gUDJkQ%g!%c{D_pU73O{_EJ zDMnL41-a25;mO7D?8%9XJ`2Cgnvy=^UPz4`&p#K?8b}574xvHw@0zI9_@7j+^_;^z z8QL`O@zOVpAOKpvdy4+yEo{HI$ewn^ zRC)-uUeX&tn{%cSvc5tFj!IkvOcFQ~ozUe`|KFT`>py$Q7dW9}Z0L3ab=- zI0!ri=x;>g2Jlf}Nf`HsnK!rP7y5F6;x5Yx?c58B!Mml9>oPku4F#rg zRJwU(FklVeTRJNur(T`55&!FK`^>?Pd!0{^YBt+8qkH8o>wyl%?z)CLvwWdN-OPJy zv5uAM$EzUZu8f64xZ7$ZN1_>|Y@}~Ro`=M9U=sdD=CUj3G5-B|QVlz3yPgnAe{VgG zmWW`2^jMI&=C;-5s>21iAG9Q?Th=w7)P%L(!wpN+2scs^-S=gTKFEZFlOQkgA7iF5 z5>i&umbK7?!NPfJwto`qVnjVr;S$VnGIZ4YAxll-WPOj=6l|~zVW6lpSD?cc*Ao*P zlPV24s+uaEZ(X=|lZ|%X19-wk=Hcm<4+t?0M)Dg3cVt(vrx(X$+WbSMY`G7=7Y{AVFNfD<}?rpc_Qh@MO^-m=c)_wJNEC%lwx@y)cuzvMJ6H1y=Q z9>S%NmCsZ{rybqCquP?Zdb|i>vtWv+hZ>W#0g{>dy!p0|B{OT<7{UavL?4y82?WbUMU`X+n8&#cH$59vYTp#|(q`0qYTV|~}k)HM%9a(7Fa(nZI0h99!a zAkFOF3tK(y9!;Bx)EuZ!TO2{eXc%8*WAg=y-I!DKGB3u*QOPRq?s{E;QisHGG5g!7 zDe?5h8h$s&U+~3Woy$F4iM)QF#kTrFN8=5`ofMOz0nT@Z7|N3fA6flow3FU=D=x^}vMDc=iKgssczj7h#h(D{lk*`V4Bhl&7D&BDW z`+QB#_p~z*pA6tq@-4e5K8+x4xj}e4dRo)IlJZS<};I7r|fPb^)} z3Srm?SQkv-5PZfU;b&vwgf74oG?=VAMto(NBQw%m@RkUfJcS_PjgNID^ZRlOY$=mNZwfo&jzUs7$mCH_sV^%cjQgCbka zNLy+!NA2o8KPmt!tG0~Z71}p`%Yb%zdsDc4US$K08?rWYunK429eor2M3Gc!B*a zbpFBH9GOP1bx}1E)4Q_wnr;KZR+Kk=%*t7R9pmwk&HWOXMd6R2}$qr!QbW7lE&$`_L_c` zz;Th2lu0e*U33O+d8fY@+S(;qVZ8gE5Ul$srR6TVJX-8HGPLGs&tH^|siAU@>2P(DHs672<7&~8eF{H4O&y(V{`EIJ*L+76 zyGvGa?R~4x<%YM~WiN7%+9}TTTfFxjUS%42B|ZdSmS_sWZ&hTDOlvBj&8oj7Qp;(W zRRW4TCfpw;BXcaH$^}EUTiTxDsG0>>66(-Epf~O66<`YL2%BIER?I!J4iVTr0M%P= z;zq@9dvRCw{@6X*flnyZQ_`U8Np9p(pu@1k*JMTReJT4CU+xzyGg-UQ6O6wtOU8!X zV3J>$sK_W>Pk_HW#F}xZ=DB)7Vx%&~`&88#rFI8+M96SgS&oWd+{&Hpsq@X4sDdhQ zZ9tC+(Csmg5J!1z?D*v4IA9~@LI>%y&bP^QnUOqSf9>yPOQ=3aU+V0TjDrWyr3fm` z7m3vvNS@L??E;cFfctlnbqyKZpq^VO+{hlyp=<{Kq-$?8#-D|BQ^JTY|A+`Ia+}B* z7mnNvav&lu`n1MtH+1^RJ>z>sDjx_B+QUCF_wa4v%aOkXJLTYo+?(on`R_ysS=O)V z!idb2i@t^KU$Ia@szhka9@Z|IAe8!+hq((Ey<%mwEX3Sp$(y88$vZ$5iM7Vhk8zHD z!f4b=oTgX1p$&nchCYS=H2XnCl{(d-Uqv>iw_*A9@bNO4iuMXP6^HsDQ!Lx)t?vhP z*MO|WI0;|>u+*~p^%}iPIxaRfirzI-q0O6cuHKpzeTz9RKt=Maq0PKn$mOV6WY9)I zQx`jb&eomi?66CGaTpq86D-EqIgL#Ma{&+`^E~}ZC+UHjS_Qn*xu^!$33<)oS7^!z zvL!ds-y@B(Xwx_@3(7y*GDo+2xK)kb#2n22(4eNM{}-&Wat(`Ri-xAuc2zc#rfP^f zP+gPi0EV&fE;0w+B8dE1Tn{vyvEOo!?}yr^nud;Nwg!U>udfIH_xKUl0v- zvWb~kdlwfdi&h}}9sFPn+3%wKn~U?aEYt!!DH|FX?{E&_ig~!fiK;RF2zx4fG&uaB zMQx+O_2ITRUh#T3kulrH)}_5$kUgZS2HUC%_L8nB%J~&xTj}*vg&F4fY|3}- zp)q9+VJp2-i0%(wr3lT&U&lNXhN;2LVxNdaXIM?*ZOz#UOm#z_9D5(KZ0wAum%g>+C6+VB_n>)p0-qZWvU+Wwf0ef8{IDK<*l0#8hS^QXKKt)>Z z3iX{q=z1e5j!`j~P14hDr+tbrq|z=&lw$L=Cze5ktClL?eQOMFL3ueobyD%q!MBU~ z=Y99JFZZJNbiY@|ZMQ`EGLK_!#20FRv{njD5`%=H#Gj!*;kuiAxxMBP+=wo-LEb3! zxnuHVEHHf>J6nKrkbjJOA4V03*7dc#i^7`oQ8|yn!6c|My~*-M=H9$N%aF=%(Z~AS zC=08mHB>V?HE>Hbtx;Z7B^*qRkRpX?FUiLZ*A;sP{LvvpjQAL9jjA)!3I+^o+B}eU z4D&6F`&X#HdbJz?Ftsaj~< zu09RDme0+ur=V7%YNRdKFROnbtBdUgD(k^42NfStkI{U7^@n^q-XNxW(vI#R4>iVe za64tIiaRX@qNxt2P2hF9rCu@z*k5UcPUAT@z+>j|@jcfTez)Wn*BHoo3~|da&)W!_ z$L@p|r5@BpP8$1z@0dP%6x+kPzDSyU=rE7l-u22#1@f-VWgCRvJ_g|1ANZ{qsyiw{+I^th-<)Kgf$q-D?$3GVH(eyG2%=F&iF&s zHSrcZV!kp76I`2^A!Qo8tXu7kX0hi3eunDSf*%T8KcB;RRuRWR=ORWXoZFA(CH+sV z7o;U(Wm9G4@NCli#0_3nEzt_c=KzT*=_2jY4F5>R6|d;ADxJd>e%>dWFQ);=ni_S&4oerVCc{4n zOqUpEXqT?ADs?+V641Y_JFtwQr>0(6ES}|WQY~6jme4AfImc?{jz-~*Qu?Gp&x+Q88}e+ zGcb!~>$uFaU^$Jc;Rr$LtPe0t#XRyosV+56A1uKh+&;tx=|X3pBsnH7qR`;uF-qP? ziJq;miQGS@H7|B&S&Q&8F+>~l2`TJc3uUJcYvL`>C?EQS!;Rd%^4=Ft;lF@2KKA}E z0f%Wd5`W>|W|^mY(RD@Z0mg6q62;Ib7Sq_=V$=QQ^fq!NzerB(+~`atD6f3M?>
  • *$fPOSUJzTY+j^D^; zf51KID@8_~;tS@xgkP4)SLoJw6{;~wn3D0ZS>HpS5F5atY*xB_CBB8`1a>R&7q&Rq zt7tQ_Q$R^iS|@)^HhW?;osz?14n66eo79pn1-!Wxzo71i0%pkbp$2BE*M8VA?b#>T zjAq<4Mo81|vuAA3+IhNoHInyl~KB?2`JLC&cVnPk;596qVQL_ulSIE3s z^%PNGThh#OV2Z1*geB-Hj9P2qV~Em(d! zA?kAEVV~zon}NS1t!E?eueZceuZRM5!wT;}MjohKdo~8}3tHb#_@G;}nOX(AsSMS2 zORmLfCZaD!{cTVNng4eg2sbSW@jW!{^sVD&G=!{-)WaHa2WhuIQn-X0s@L0jE!Ybp zfEL`hf9q&uC8>@2$PN;yk9V*UYbql10t@nfz;}uo`0)8fKnIP@HT2Ggifdmt01q46 za2$gmR+(BqPnrY$Cs~^vL|LPUj+}OE0CLsQ5J0ng42qJibCeD2)%1K{*i2~b3N&Aa z$&&WzzDa#OI2QOF6fE7N!h@9CS~7+I54RV$?Q2)RKBgX5lRDbDFw^*SA3s%MQ`8YO zWSX2m#EDc=DolA8xShLjrbT^!yoGE1JNkV3t^L2;Kw-o86;+zW4Vp7gWR5Z+y}FA2 zapu4gP#^!bAGS=HOVoSif92>E&3TA9&sJ*kh)e-em(G4HFwe zg5jp*__W?20YNX8ktFe``jybpW`g?kCJ}Ka*WNdlV>|o!JN^YQP`tb+dIz~a^@qHo zVd|>t2ufm}1~FtD)!VDKo|)UPs7f!Kc<3B_ZkTQOeu|#J0jk`t;}TkJ1}Mwb+sb3! zY6dG%O{e(&(ul;{l4wC;s4ohQtJAArfxk&=?U^l4WW&f36?O(Ns4pGBM0Vjf&~D_n z3?@I(-^`&;Hcluo(du@94!0~m7ZTX_E2jl_orI;%7d$nJn zI*ahGmfW7|&V*H@zJ5dqp+g3j1q+>RK*G$cy3`rXu&9qqZlyy1aepQJ)@Ls0jN46X zPagl4kVDLP$r$R9d}HJrFU2wXQhmrK*DA)H#Ad_(@evA>%+Ju>{t|_1fbG(2no|c2 zeBb$Xv$u!Aix~6VY$v%jSei^VUG;7=T@r4&5@xYB&np~9wxq8u9j&xBxhHoZ2%f;TKL0t{?aI8`uw>C>hH8HZ?YfqD)y-; z55JtVj7e5PF?VZnb0Ih39lC4kV#s27cP-DBZam*{X{tdO?r!#;EEHV#8GuwGLpqnG znl6uqn}^5Y&8$!(Li?|%c;rc^=RUxhn)2i$qW7aBj>T<-n=hp;S#{8i zyzJj#T>3pX+QMJIV&c|4yys-NTFlppTKu<{6q;T*EOWQz8XK{R_Z&^ zTeeG!68Rap#Zt9!i5XL?%9;Gt$z}Q;**{Nma#-4nU#FShU-hR+(W&8(`~IZUR*3X1qqXM zpW;u=E?lA(WEEXe|F;>Q6Sc+^)a}JIBC9g8WP)Hdvx=8|-@l?{G3V#O(otKf_HWbm zrw!A;d%EATo4jBT(k`k%g0lhplIR zu|zW-WIY;hHHyadwYs17@F49HHa558l~>aXA?Y3-A+<~EnfWmy;?4N5JAFvE%Eeeg zi^=T@_XW^MGs&U4xR=l*YXF>H!E?R?swN|I6rOF$r!xd~b3E^lF~nn9LfO^1nJ|r*7t22UMaNzzIe~OBKXsoFZszm92!{ z0|gByFYKRy6<}=i;TnF25{q+<_bKaj$)(p08gRJCW}hFNeb%qoUFzG9ZZSvV6MVdY zIe>rOFS=Y@`z0(+WH-3o5%{$wJ9C~r(fbJH%98x2akk1lFLvb4g&KH6Tgn-mdQ|xD zQ7FsHy09iwwG=odE7jj7{gd&I6=YUCCaxLzLv$B6DEhJY<0T@n0Dv26Uk*y6$U9si z3QdS?XB{LSD#{IdjsDxiZU|Hxm>_=g8KX5X?K3&XJ#c&@8B&E7tNIHFAP zJt{lP-F}n=?%waw?KiF1{I}zIfhT(_$cnerpJTWM7^?i+=Q_ZgcsnCk| z@$${JYqsK#DrJ)^9~EWc{azC)am5Vem5h@uK@w2Tf@nThrP>n7o#K2D7|rew?J0Vp z9G=AKiOS_(JPzprA2IK(1zn+?c&Rzgh%;F-RGSB(D;?K3BqM_sy5l149oLvvdmN-9 zw1-9Q^Ti%S#1*G_RtYG@wGLTKY+I2%?g5U}XLNxs`4$3jQ8T{Ujaak0!UUa%>1lKX zt7?>~Bnh!E$WAT&{k$SDpVwx3`hw*?U%61%onVtV-UJmKT=*@HytU;~y_2RgN&|03 zR`&NCPV{Q|eJC^>{i1!*2XqYnX-7qG^Kj$d48&SM{c-Ww`5{Y6WG_!T#j0CoI4y7C zX~vs9jibi3n;G8Ru#2P$)xoH+M)L0!(FI^$%8IL(UfKMQmLi4u&?rHHg{0jkqGFXD zw%lX}RMNkMzr`X-T-+Svp#}$DLyTJC3SF#K4{FC|m+k}VO9o_D92*XT-wosfr<8u! zYcJeWmF$_8YHZn8{5f_;pX2+X$OvBRxW3HSW;Wt2_D|t+H->ZjNlYA(P3LH89NuOA zVNASl1sBpM{tcww9S;~kDt##hCvXrn6I7UUDmYFC+Ss}E6SEYuYt(64+bOYxU8IHd zkaZqYi@MSpP%HkknQJ;G!g{T60oeH-*-1{2x;WM%BRlt&!&*(dEYa3{$NKQAOHCvz z2<(<}#hE9dU&fcpF1||%9ZQi$AvH*f=A`Mc{)L&7AEf^-bvadZ+a>9roWto z1+y&9B?&qgf6HPJTYX$}u9-4({VZ#H$#7~!M{p8~wS$d zUaqyP5$P9-4FP6=&N>_4r;Zmdk|U&^u!gwl7vZ20;7>4v?+`4)=W=IZC59U5e&6c* z&=&Z(i*|OxYZ-@eWp$+A@KsW%Xf#VSUkq0tf6)+Z8axJYX#Eq_dIm(CdBsMq@vb?= zKdy%%09!uoE>{le`RrznS>@jDdt6OMFoQ5|#@d?r;!k2j5|Mq)lw^*K`UsrF76Df) zJg$sPtaleHzE!0C6jZ#u{pU*gf393J%6F{8~KZXFQI-dj^Rk+a3jnRsZ2@B&suZ7Ya72Hr*^v|d!4I14@{J-`Lz ze_J81IzyS6+}h1wH8XI+vRHD#bois={hJsnrJuo2dsf|Z|HG8n3Bznjyg~ZQi*SS!3+ynM zPyo!`;X>!pBh6>~E_xAW(@IUA=KDR}-}u`H;jYf&hz#Xq9_+8?m&!%PQ$c_fU?Ju> z#xL2ULE;PM$9+Nwq{NI4Wd?&@0pC5>HwW&tsnUWV8YOq4UL8 z#JfHl_D1&WbNGui%AI_QrmK6!Dell+SF(PSF@a|0&+)5$`$(4=NBGkbQBw>Rdk$@g zY{avbYviYyHio48fK-09XthRu8Z@vq!$=vHIb*BE=6zNV>bP!|;N(WgJ&9-fGP*x{ zM|)j5W_km!50K^!%%Tcso8_pJJUb4wv#>^FsXv9z@np{W{wcvCgS)a4Zp(M1^kfNZ zFX_qZFpm(gzh~FWJ$*+svDos^z%j->+)GRaD<025PYjn3D=MJN#+I;x3>aFCs7EYD zLHSTYZ?Js3udLT~W||?mCA4g1eNy{GF!G4)s`tW0ZK-`4U^;#pcBV)0*fC#uUseM9 zyoF({v-t?4ftRju0!=SNy(6YA!3(e$Mm=tZ@nz;A5Zu1CT=P@bcnZ!8PyJ+rO;!fU zS6l#XH|+lyXkMI95?}1sm|Zp6L}yezFkF_hsv;XY8V;JDJt(?tI`(Gt4T-w~JjrHL z14H%HFU$|N?Z$*ENFb}~JDxuu9$S^x8j?ns%9{F9s5}LJthl+2vXZ~I#QJ?ZF6T`l z77KsQUp8YVqpB16L2|+4gg+1CcUlH~p(^>V>Srp|v9oq^bgN3_2MvVfD6RD;JIi7JYoX)&BL zyO_!>&NQ0#A$uX=KbvvWI1d%B+iJAB7$#^b5W2ai)kQ6j0txqzVAF$Fl%}0PPJ%17 zyg&1;&AgH15>~2jB;#M|0W-UKiH`>!=*E+q69w^NhmoRhQ_93N2@_-g!B6*9;NzbS zbJGn+@=DlL7pYgdsXi^@ym_-!=EmrDhS-ZrdPQ@@FpfnXWzsVWE$1z0oy93FSkrL@ z`yaGRV&Z?cyaG1cgc0GYxhzMhEbBO{No8LrqJIlRLtYT~Px*wJSLoIy_atD_JxQZ9 zIug30gv2F`7@7y1z=w4e{ErAX*hRcE{(<%{(`x=R?I71LpylW$(rJ+-Nv7s~3pMmy zB3)4{FCbEMnL8+}xbdC(w+{Cw7DCXo7bBjP6s{)p^>x_2a+vmVDv8|p!0-uqW|PyW z?mn-^)aaJS?`6$C;jun0Mju_h#dc^lTagh#N#3Gs`OGKX%RQ+S86DOt%_Q5pHP|_ijMN)%k_T{ zrAdY<1Clz)m{Y{NlpIy+(hZOfyLW=VrXZYsDDI|qtrv@{+#$5Hhe&&lwKa5L7L|lm z!B(o#9#DkvSeD#bC3a^FeM{}R3}tXWoUCQe55p&J(-(5(`^9lX`pgQ)+`@|LrH ztliouN*D#GRd4D1YD9qMFviJpY{aQzq)z8|M^T&)8u8q!_@x)Wj9^E4xPNi5l%N89y{5e;56o6$ymDzBpQ-v$@q4YdZ=yQ|<})ZD zi>$fnOng!SZ^R{{nOaR@sXuNpBb#zO{LRZ?dl{xDMt7uxm=mZM$p?<~`T3 zai=WL7ye04UTX8DYY*?c8LFKegmW1Wid1XgGJ?|J!&42fasTw-zw4G{Xy_F9m(rClG^Tq?d12~d+WkdFpg-Q!6T zW@?!Ip&1x*1|K7-NKLPTwViTx6{g%L{plP05`fVEC`TVzpIF4YFCvY|EL5IB>b$in__Q5QxQ>;Of)WXMY62)l4havKoF%2F? z0%f~TzRw}8r;oO?2IYI5`9Vo;?(O*}&|-~TxhFg2t0hA_K9O~v?tLeYmcLf^(9G#; zuWm#p$8*}h7{xl;K-eA4;j8+vT;NZ`dv(qo3obF7UYUyC6vCgKT@&toO?C!NioF7P zYu!-_yApO(RM8#C8H}^i51P7?e0_MOZ?twED~iP>j>v&@%1igR<|YzNh9{uL){Opg=<1x$A%j?rMeok^}Ry!#dF zWO(D7vDk7(=P)i`t%cm>ZFZoKJu}9{gRi$iPY^wZgS#5i%_~xQ9E0T;POk)#x-xu~ zgee+MG&l}WjPxVaU~Fi8@;Ct_>Oc<+>jz|CF;}Jc97cx*q{up81Ki%ehFI)$J*p&y z#6JQszgFFts0(c|teso1uh5;sK3kv|R?L5yu$|@m`}m9#B>ZvG*!v1dcF#1j#bHUf>B(4=umG4mFX+5NbB2aErNt&rIF*0pEgeyIw6m#Ki|dOKDu9 zpG3Cs^S9_01_-q*O}uNQ8WMJl6WX$C;-M$=@0RtwVV_}FHK6h6CU|J`K3`Tx3cFXm zr6fS}L!f2eP*1Q+z>&@VT?{6dhA54Si@zh zs3gUu1Eg=A&fD$7eajGUlU%ACnc|C^asOn-q@e1HR6(xbf2j{t?~uHY;solJa5Zwze|yAFYl^J0 zBPT^9+I$vmQAdhdZ?H1s(hrOegCza2%eY%zbFXojZMLmG1Gtqnj8nlu!)3BNlAAh! zs!7Sr(QL3y%y%_^_}oCybLZ3%bHf{ykhq1>RL5e)+b#GLq|EHO3=!uq|9g~kiYSsF zk*_L#GascODYjUS3qL+4xl6INK1U7PdzjR($-zk z=_A@*#45t?sxP?B6{*hF6DOjZ!D5fgQ6q>IH7RtpJwhVXZfkcsZ#&BBbZ^22&zGAj zC2s9opBK%gaiNRdzg$OjtqE=N%$N1a#8EH%gSnk{!NEtRF7=T^!1|KJl4IatMzWw* z@~3RLuQmhe(0XV=b{?`P5_p!}vT$9(Un?X0Y%0=(d#`P9=(rSuUQ5uu@K4IwL%hITQm~08aE~kaC^*7_JQEDlc|cU%gIWnjBo} z2)`t&b_}5t#PLVTNkkR#7lmLklVHl=XGve~#;=_yDUogAm8-+eOdR8xeT};s@s&}b z^cn%W)ztH=Az57r-J?^i=*c}_`Qu~z6!YuxhZJS4<6}QH%85(6yN)AmP=EFT8+8OF zseFD^*pRy*({HudUv}>)E8RXh)SDU~m0t^+k-1TpPTnl_3CJeZ8YxqePZTZt;0WQL z$8@~-l8mI(4VK)F(2S&(c%P(AfUKljdG7yB;6OgNeb!u$h%R!Us8l;pEN-}j4YD&L zr^|x>IYa%L!L>YB=<-j}aioKJCniJpE!6l%igmUOQ-4d8vClwd*RLUSI!M;TtpxB6 z%i%OYh&>Z`R2JGi&@m9+>|O3pPU#*j!*0{&^x#W?&qv4yDL+;7_qclbT1L(FhJyJG z(=XbYIq)jhpSNtJI<*$A{k5F12Zn^&*OEF)NfW10^g-oPLt0zg9$>d>1Y->IJShx# zS47_aMOX?Rv!=drO4`h#l!zK%8e%|vGDK&ua(@B3LTW($+%PxiJHVDdH7+lLbsPZh zBuB9pMpu0yty*y1(3oZ%2Xa)!i$(ZmFhN zH+|#Jo+^;DC4JNJ+s|G%9lyt2xzWJpm(8x?4)tyNAL%a!&2txt&Q2s z#{1J?A+rCR+T(29f7HDUZ9b%pGGu0`*{L0>@UBw;aGv!VYgFuey6wrN}5s`7QtVl+fs>g zzA;<_pANfrJnRGDC*|gH{Up2b&i^;5tTPJN94voYaLd)l;B%9QV>F4lJw5y*%Bvq} z4F`iORj`kZ4%ii`4Q`-3ZntCVu{u!Y!fBJaOyOLS_WT2GMq$C|H|b4Z`irY(@E1uDp8L!U0BebL3Mff8cn@7aq62Tn(Vmed&o-AJ{)K}^akAVk` zY-e)?@&dyNg)M}9-7@;Zga~FWc`ow;mO}2UyN-#QyE28w+jN|%xMz_UsXmPshs|f~ zK~8G#3zX&r>NfLH`S>jj^Or(_lA2iq570YU4G3l##kNb z2!hn5oRX{?v80P)?}<*TTK@Sziq8Bmsr2u|Gt)G-OfpS1Hn~vKV%8|rs<}XAeJxEX ztt>ajY{twIO><+bv9e96)T~r!QOQcl5^)1cQ%X}zR0OolaK0uHrD9?0M3X2Bb8pfCiC&&yc73qJ7VB>{V6W-Zfn*iwYOz z|2G}7XRymzb3vK{XiQB>N8C*A3stQphUIU4>Vys=Yni8q?Fv0rHWD)Vz(K+()0@_8 zeo_`%UPwi&+&W(BOqg5BuuR@`v%*DUdq0Y}?_hsFBL1KgpXRIw7rtTF!uYu1xZ|I92w13%sk z32d&y<|iKplUY#o1eAG8-YycNU!ZmFr3N?693tSdWlN@NQq?r(E^Ul z%;VW8p4e=JN!#e`U~3E-BAK;wYt!-Xx4LLuk;=t?&E7>!50bxvAMG<_)L+O%Y*!1Ps`84Mx)6J3Q9uIHYyjHB=s39WoYS-tS zX7-;Q0j=?I#Pf=7`=mOs9v}G{de$Zp*Vax<_BIk<*z@L#i%goq5@$ z`yBT{A0eLOU>Y%fxkD&t5+wtQNC{+gwzaG++cxyB9R$yjxQ*bXK2ek9W&KgDkY~CJ z8dIul)k(=!RU{%-gDVaAv7`CDXf%))r0Sd?-crpMh=9tQh)47gl{gz>96axn%=$7{ z_7bAs_l0rHgUn*&299D1C(XHnu91TEv*!%%`cq4XuV+Yc5p_Mu!b{bX*^i%m5DX6; z!_1PR@i3c*>-HRlqcgL%01qTdQEoU^b4yneA1~x5&b)yNcaVk@1@d#gCHVK!qEO%_ z+?C);;tFtKgy2C+biksttJ}3RoV{ZVdNjCOF46 z6$*SJCf`3-3BI%^F#ppvxyZtKslUsi5XIRH$-E2cy#AS8dBhu_un5-m702~@%J(>w z(#pMP9c{VHHY7|$6sA%ybaD=3o(v}f!3U9d?|Z;Arck~7b`|cSgZ&Uhv$aw?TA!JP zqs|e!(W|q5sWgA1@mk4~+g1~DvPZ6I%ElLV1gv5$`s^SZ4{o#E9%`wf|7!3YcMO9! z>yUZkO$qv=q+7uJB(={WV6n&;87j-JuwrH~uZzRLd}ZBSMnpIdp$*D>CUK*oCF{Gx zk9Q(=z}<%bb@_r^(+_*>o4zriV0f8m*}Gd8mUVmTpnYW#)&cw)PV`M!NKLhb=S!8+ z6A1@$Q(B7g2XtQlSc80Gmf79Rex$}G%4wSYCQ@3=PSX%c9@MFA*~W>)^=|?1GQ$lc znNh3+c+x$}6v?@hy7M#iI<{65ejGR$85*;@h#m^!9~JL}MXplr)*ssiRwlT(sZ&#| zt`)-0uS9oU?$%i^!bTnB0MI4F+`5Jjqd~*u4kzwUaI&$mqRnloDuYwZVHQnPE@2uy zY9K(R#7#}jhhDZUOlVd$I+}8jiUaQCKuf*a#tb@?RcotOoypk5V+fG5z%@^smFq;$ zB_&e)Z42~ahiBCKDYrAiT};+}p97?phb{#zeqwHlRqG?pbs{e_;eW>Q2&7EUK znp=M=8p0TEvaXF#{$bXI48K18fN3$WW|(E8S|yT-n|alPaZ5B4Bt9z4_kxMqtg@&| zrxp!^`_z*gC|_`pG2Ab;&c9EMZQ#8JKGzD(u?a&NSzm^6E>#Q%OEUXHk$@hcFZ;!% z(6WD1eEa%uG#hNkJkxc~7y6rIbtYQ1WbRugDTsN-7ZQuv&U_$vZ!Nc?Z^fNa@x>)+ zdZ%lalo#|s-IXcN%zOdbJ*_ZiD|D-fn;e-1iwRj<*@_-d4p9)h! znqU3gk0H0`C2Rf=`gO!>_bg}ERYx}T)Vkyk5*|vbTz;N?It#*)I%LtD zE%`l-{GSl5S+t34D<7xf2bgE3ug|gGhOhLQI?ka{E8e!fDC-;9d~p5y#qz7^ha;e- zBrj-s*+|A|n{2M8k0dmGe}eGwXbdx`hQ&<9-+YA9_Y40hq$KY8_9-Fv!k)7Z0&Pe~ z&EGnszt4Ypaq}=V0BOQd;uq^$tF2q>yPi95qRG9$of3!1>5VTqn|1#US#IF|F$iPZReR|0(#rITJfmYI;(U($8&BEQjQ#r3FgrO&*-- zdlG+^qq=z&55*Dx#;z}C%C>EUEfT-nF>9-k{fH|cc=odrT*o;5Dj-C+vrm|lA7#v9 zVerZhrS`^iNt!5EUj=_V;O_?K;$LU){f{dRNWfK$93>u1dA0zO$9*4}@1UEmiCMs2 z4m|)Zosk#Dd^HvT&oj@m9Vh3Xl{%LNB~*!mbOY(oE7_+p_^X+>B$OKY{&1MLyeBvs z#>fpH+>$3u(#=VY4`w#op|KwIX}g>y*(1>L&BM!n(S%FkF1D()^Np$D^X6#ahX}&W zLBa+tagQIZLs^s^sm|FAu9AS_(XH9>68dss@dA@UWOfh6WWq#Mh%ycJi*bv5pMLQm z{^U__pD!Hs@HYy6r|^?Tn9vi z!Al&UJ4k*YBZdHNM}(_n` zq7+WlRf?I9S{c>1#xmlJ1_kajClEN16z(;FkUiPglP_KZz>DJAvb+Ximwo+by@A~_ z{lm^}7fFd+Y8m^epe0Zxx~=*3Ud**oH{Q3-XK9>cH_%mZShQWCoF~sdV7GwiZzN5{ z+$!ekYQ*3#!U*t47yX*mNVB-qhnX`fbZ+^gd&_pzJTFqNPO2(HEso!Xbo8xq+N+NI zny*U?7Tf;Teb&9Z&PHbN$SnblCr57pJGm)7xJwupQ!L-GiK2WoQ(P63{KB=bp{f^F zmq&gpq-C>bo&LbRep}F>x!QBYn<{k*QY@XrqK*6JK|8-svikchFa>$}{Q52E5-~-$ z(;9C4>FPW~jh3)EV<0{Db=x`38sNP8eX>(OO#Cq4xZ^US1Ea80BTly1YfxO~AI9O1 z*dP276@UMxbV4ck^LNeTQ42RuH(k;0EDEcXf2}X*Gv}dG72uITTR3mC6X7rA6DD9U zJ1;zF9Cg+CWVz6v38TRs)TQsMES~*uGfR8ub7s3NWab`2a*;pdFOx=iGy;H$^X!3` z>KO1ZPJbnoV!5BuzPaH|ciZ&%sXD?_iClq@I-|#JgN}$yk66}XS(n*h3DP}V1EPnb zPZHLVM@=Q!bHm#?N1&U5A=N=JtR+xaeIJ=>SSR{wz~nlZoV$g0kZYN9X^0BTiB5F8 z8T&rtNz0?Yl1#9M$61K(14ep;X`~w-Ax7gC5USQt%+E){GXS(c#fL>no2X+n_ZX|A zyK^kZPqqBlnZna&j8(KIPE4c|1WR}il#M>b3lAWVyCF-3nElxXF6QmE^c+6eKN2UK z{?nU$ih4lfGdNfLgpqNUcSZqM;VQUR8Y98Cs=yASi9Bb@*-A`vM#=dDs)rczcuFsJ zx65KDSbIi^<+~K$X4&FHPI?hQBeF7Uh>ClHHBtIz;4iL;VlBIlIv>^CwQOx>ZgyEO z**lMn)?LrwLO)OlYtrmglh39g;`x1eLV_vvi}NR+J&IqgBl`)VSQ9AiHQR=c#0W4W z8%}c=_1#pqiD#$S@r!FparEg!)M2K(aYK24H|H9>BgOG2AzitHQ~dST%s6?xAs=G$ zXlK<6zt(t8$m5zNv!o}5!axZP&f05BY=x6(KKzpbtJ{pAN#DRSabd^|b>KhK}wl83sjC=4Q%V2_;P#ZXx<73Vh0^nJTk z#$x{qJ15HaZ+2JLrX^jo@C%g;g{@VV%eN|0#hlg1m*-u({gcO@sdKkOKR_zgyYkld zN0zCtRtYMd*>3o{3yF{Hye0#}j4S4hCz8Ai9nM;P%3KGa4{?uVoL4cW+H@{f%_n!w zqs}d1z?R!m6)ooXT84o057SUp>eVg^!{$roY01*8og@dkiv!`Qx=54?NAQHQMrA3; zZg*((ia*cM2H7wzQC#!wbJm^X;Ro#p4Y8ikMbxgy$<8#!0!UQzkFbHoJP50K*4pQP zhdkGd=4!{IZ<+hBTR24TMEwuE8E#-l_Z)u_Qq)ImrvW5|AJ${t9}wvr8ltmaHoB$7CV>S`0uz`v@!z@{43+?>7SIZk*6AqClvQ#UzAcp zC$CWLBU_SmbD7Vit1>^}PxE?sRzai_NKMJBAs=>0QSa~OwVCTtMaoz3)lR5Q)C$c? zPS{O!EM~~JJh>=&n2Czq6~?Jv4?6o5Z)Hf%Y3#O2>co&fOih~8Bmo!lP?M|)il`)A zw$WsJUfE5;`mLpS;kWfQ$|Cea4WdayWq8(13UAw@(a6aa#^ce&)EE|&7&N*%laCoY zN7w*O@T-|GbPw~!9^#4q^&{yx=Qug{(SI~SBu{f-etL?CIL)evdOjCg;@NIHbWq!# zX?-G!ccKyuwzdE*<*z%4^hDtY`MLDyk8RBZ+c=)s#|^(NbIs8#PT5OTC0Kr?6_|31 zt!HY1HluitV4fDU7?Ay0%D0gtFTO-3HyBcCxhD`E{a~qO#diBGHiC1E4Px^INR$!I zD*`M0w&t@SqL-0~U$sJe#UrV;(wGDg!!R?;IiFB`MO-tw+n23fHf`%K><<3~c@a(N zPkP8dg4UomSQhD3(Li4&7uaohIIbDd4)k4oO4$eLw(@(bnx}%AU|qVk@Q-DV2LkvP z4~hP6#pTGQGx7@Tf|+Lj?APxPW?7@HOC9QMrD-*UTzv z2`7_KpWgM7+#eG#SJ7LuJN>74%4(k#{IHK6|8y)HdKiWIi>hdOT-GJd>P`9G@KGQ# zu`#UJ=z3kt)QaK7#r!3qBUU~r5$PHlP!mo z+{KEj>#p*PH}pR~nEO<7j<MPT+cLwQaBtn(jUayLFZfn|TBL?Kb^!*5eCjiGQUh-iDN$ zoj1<*x!(JTWg`S?RNMus7K3y{whGHB{ewPGL#imuci{|RK=NDL@eC&8F z>IVh%wSOG^we4adFDC>D2qieb@dOl+Szm;7kYF9V=;uVOZ5zW4o`(LZ%t%sPr=-_@ z1>b*T|BWBEz6LI#xc{=ZS@+kG<(jM0)T*-A?%ecJ#t-V-CmLh3g-;TZ;-^Pr+{CX> zxGZ5iB9LPLyU%34U!O#|bX|VZ@*nneCvqN{i1m5*#6lAngyBz&g)vd=a=r(-+b5*yN1}A$jOfQwY*vYW5mYI8wDd4EIB;zp3Cb zp|}SkgOs^!8sMHG{Eu)u%P|iaugSa@=ZCAxNK<7gn!aRzNcqr2*nyzkkkC@vYoZxo z&-<~NA+8}(LFh!T^zr@VlMur{0X;3F;J==sXP7p`>FGjXkts-bagBv$QB0h@%Ki9l zd3Y?Z%W(IQ2+|As6>^UuAGWR#-JN8>`;i}1-%n)x!*2(I<%jRr3quh_GV0x!Gd`=> zp*1C%#V>So{F!J;qYotvQ}2T=gMRlZdIq=2pqQtOWd!$5u{OwY@QFUc* z!Dhq@Cow#m0Qzz%3LYxtREMc{YY6fcc+@6vk=-M@A)X-6!E^s^^^*|S^pdm-q+rroTncMCQC6H(N3s|D5+$pt75L-N zkqdMV)lRoTO19uQBKrmMH(c~)kp0gK{UTj@6)E;raRP8Pl!3j;HUZh2Vb)2ynjAeC{nX}w zBT7`7e&=57{f<;=#!A!%8xB4bj$2#NareE&CWWMf5lY7dc2QCk-{bxM3-8ysPz+@= zghF0uW|Ba7z;y_-DF~t6?y-DqoppubH(zk1;XqAGEZ&LiF_}5vudAo>RzbsUvG~_c1Zn`#LL7 z%%UFhyqPCAG!MKmn3t?Gn#w!YYc@j9y;!H9=di=@b%Sdw@ORu%Tz*$5uvhLn*jgoD zXG|6u-E{vDL-o|YV?~5e;3Zd6IrsBbHm(@2)M|eUDVB8eQ?wYJ!|H}Sml6Jv&kayx zM=aj97qC1uzPp$@w_spfxdhc^caU&8_oafcrj%$UoF_SAIyZz<=DK?G-;PnV&0^S= zOzowD>G19LPhoYaSZf3(G?+3Bzoxov>j|cGhW7!hg9694(F7a7m7wv22YA*H{V|uV zjJ5vhD>o8KE)=Sm_eaW+u8nDU*qC32yr#>3mY~AYN z>&d()S$zFUS-%w8%gt#n zHV4@OXq}%7C0OQ+nGJBj=5zkTFNEGed;uQG-g?4Yaq*+P@yJc-OQjm`z z%zUX{pMkD0v8$l%{h*hYe%NTQ+c-CwRiQ@dn}@@bTwJq?Q_Xe&Svqp1Df+#bCh5bnG*hRvf!y+ zM#rkI7DMl-GWHN|Y`6FQ4E^t@QvoAj>-C6{tT+rl>$-}8~;8dp1 z=QeQT9>{EvsJfj47LQfC2W6uqL0c`iE5Hqeem6v6eL;O9N8F3}ointZbkdw+xq8Fl zSr%D+H9hJMs&0}RXl<|ICSY=Pky#!(s4H_ZPnVSCAvOmRW`-sj3yJHqXF2;xjwb1| zm{D(b5wq?>)F##*JU1;muBlu;$2l6(cJU9d$gJD4FUEbDCuF~edQZ)EwJOJx_?p+B z!+)m_k%(iKe#`xDp2WV-HdB_5A2g9Mu%BmL4Z@l-@`}}t^fZo&vu5O(t~`Cj>ki6I zYOns1YbYS4>ROKYj=I6yETYVfG2H5TpJJt^U}f>MmcUEUdU*T(Us;uD-89D@NMB?I zzTD^;>{4|;rY8Fl;qwPcrR|;p*R26VJN+EGGzk&tY+FA4Z{z89@Gt7D6u-Wyq|*nwgHf9}Sx*DZ2Ktds|rIh~$ZH%?JGot3PwkVyUh2w@(vzG$KU#6h?cq*(5* z4Hrm#+Y0^vcAoKuRBGo1^$OQSd+1kmyYlt#kcq3H`Ksvz(!6E42%XQ6x%CVKdgb^0z@e(U_O)7AzdO;OdJ3D9NF9F=W92T5i7VR#Wrfb zPToqB{GhVCgZ_cDUL$C|+rjZ}2=zlxp-w9{7t(*%UZc4bXlp}|Xm!X?>ZSp+y{=7# zU1i>{KnG=qzb;79WYXfN@nM9^5kz>Mydzun8Zk|01Ppx2u^9sXjCT1@RVtHh#-1lg zHM%BTKlPpEkPe+sS9O8 zm+A-1-x0L0Gi4^)3#SE=I2pRAwYbm7XI+^}l{z2+8z(nN-F(ovT!q}->{ks@`S+k3 z`XiB2U9_TZ?zcK}bj34IxlYqRvoDSR?c`aX;#TLrJwY~U>HFZ!TAf6F@}bIiuA+s= zYp1=mM13SSDkIDtnam~FEui-_y*-jXQz!h=bE0qAmBdb%^9)@I|06pf%TA$>Ky<%6 zYfA>c`-__0BxLLV*lpf|+lkerv)*;|Mi ztl1yQV$_p+^R2JDz(ul8xSMC_5a6d?+l*Zv0&HB4{)m2KgXKDuRta<5wj@<|V*U()D{Gyyur``~C_+!qTM_$yuV`a99-hgQ?cSu)M)>;-f8T4tWv}s~`O^guW z&s?m%@z|gR_947HWrrJU1%anNwmf`HwvlV-UXQmOVm~EhwA(i3 zkD$4e6%%wPW~Di25`e9jjJQ+lEgSO99*FJ|gS1&(uF5<>Ju>rhOLP)1jJHC2v<@vw zZIf=^sGLf6S;(oB<8Knee67!cUMpGVD0_2TvYPb^hWy$T_rS8Q7D1HL>!3-xjd1aX z=2bn^hjxzSxqZ3LWBtXp1qXm-LmdNFzlsMyn7%w7Vh2>1vkgh_P)SXZrxL!t|^b@7AK+Kx3b= z&PBkQnML_Aj~W`cpk^(~@j|}k<9G3&JsTRF2ONrkU%-QU2OsN}JW-+%Hbnlo(2J>K z0`4xgBc6v5lic0Ry;+AwYt_emKrNV!;&kSghi?PEG^UBJ8}(JtluqFxAYm_jfw7^G zu$uL75IOn!+*qDbdK{i$yf6y&G>(ovbAvygpbX(oixyo+CK-Es5ec7>6l`{vYqKh5 zkb?~rroJqzV+@JR{h1U?=+C@GW7~8|Qf`dtsQ6iBwtoFTopejiD|DCeZj4@J#w-DY z%$T`(qqJhIsRh)%vUVZWF!e}b33E5Fqk?oEId)aQNcgX0hXO4^n!^min4y-&i(cBr z4^7ei_&tx~SDJU-&%VFXo`U+p?^_jlX{)l9^^o|QOv{l|PDd;spB0&H+8u{(&fF$X*1rf!b~9gzBy%+mt#28gStq%^Z^+&R1>re)S>{f^Te4?+fv(Do zHGaK^5p~k$+b-_cB#=tnWv-a}F3ZxUm+xXkzjAe)dA7y*MCkoYpQ|qi+9=xOK+aig zQqPDxZWj$I`t%#Muf-!w22L_#eeAPXd681ed*u!=)=;00613{CaHN2{cVFKwZ(%LN zeVDSt*{LSzCaPZT=~gAsOpmu+0~%Xa?nM4nhqg%zD89{o@6C18^sU^QJfH2NTK!O^ zT&FctYTz}?$`w+bEV#ScBc)w_t<5rB%?T8|FL_WG?~NncLJpui0o_ILgcV zVtLipaLX#>$kA2gDtDbg$c@)s~jA8-uX5w>@bJe|&%r^_8D$_+o@yNnY!G z$v2G_lXwuP8Rjd-@zJC$<1hE;H*~YtSh%JqE%ooCCo$E)U-m|39(tweWFD&x_s2(l zgXL8BPDk&*l1sP_q?$^ca|k}$IJ9yG2tJ>lNH04op!6c|EDa2 zkNAZ#4pX7>Z#uA75X*WAcQIvTTNI3h_**3_FbjL=tuFz$D1J==`Fc!+dGrYi)X_*f z$1R#LrRON!o$P!+1z#Td&&Gg!WmzhC^@1`XO0$7nB;TzRnGYRcUd{v+zW!i};JMU> z8P$in>|^Yyf~{nCaS4|^cN*V+@uS>U;@DCReVAxx9lI;3znX|LR;d)|lY3UkJgU4`(@ z5Kje9mwVTNFj9$kw@gClwj&VfewAj zFz?i8$lrXSVZ*ZcG0VkBg7$pOM=0;ZfCyfZdAAUK*>s$gi%FQy@OF>`=1juFlqWHV ztZi98G5HF+&FllL9ne+dJ6w^sITiP>5pOE5Zb{>eG~ea#@`P8^u}&iP7egLdd?PW# zriiED(HHC(rM$wlhfZCJ+&jFO#Jv;p!D2VH?KL75!&Eb(gRXV0X$`Oi-YOWGy-arD4A#JxjCHSrbZQB1n85 z<-(dzsdVky3T#$1Jc+&k!nV9PleHdx({j2HmT3Tv_CUs-n$@RpC!097`$~*`k~D#& zWZ^*L4z!G_dRJxWX(8aif-KNbbzOK}6e>*Ap1;e-|iu^+a_>7Bo6LiEBKdO4~#&*Tl+mn4U#BcanZb zT*_qd%n}uIa%o$rV7sIvGUQc%-_@y1oGD>`Ovk{SyknqMGdiy? zKn5SYm?1i-pI_!mXN_dj<*V=nl_MnDX00uR9`8a08Kvdak zc#^>ESmm_Nw^rNe22Qw{bC?~1Hj zd}9YNVOS!DZD+M6v%-djTHkW{P{+fVoR_{z-W+E(s>1xa8xs_eJRB?dByD8TP~(`t zy7Y0rn1zTnzJ=_2e8OzfbJn+6E(}EuYg|Gui?Rvqw9rqub$0&^AF^)ERNm|y@_^?} z`jo|U^9SbLA>HhFCZb#%0LNV5pNap|O$b^Pcpn~XkZ0BpXCwqDHE%g?!)jqFLYOXm z(!+r=?+^LmWf}jD?#G@Q?Z7OP3U)uTjka3|q|{TE=TcS`V@v(KMm$7LOA*s>8xe4(qe!-6Mk7!#Fy^GkhM7#RnVdL zB#DrKOH$m>S))-wZ-`0hp{tN9*`+hh1~{2}ab*vkq)8{#Nnp;DM3}Q@wVX0r0l*oO z#pA~)+P|CfcUXQxxd{%~sE6XtjvNi37ogWT`?pjp#fTuPN16EnVzE+@$RjxFA2Eco zt0|8fm}7p~UYs@L0IXw&bWv1F)b>1~Vwp3~woz4u>$0LlL!^%j;-;b=S;Jc3=Mf!? zBOvOk4(+FRaYpS0G98~+KLYB9-qz4-2?N@z@&syEch2sudsPC0+0@&tP;rXdDieqt2nFJ?` zKY+j2Pbz`*nJJpTN8hFwrTMB~sfHq3{0zdU!WVO*$^GEG2G#GfCsC=tv+$Ov2dQY) zMB9X~!ui%CwTSP;#!8Ehg2z{Yk95fI z2zg(@b)U&ZD=aBm3&e3i;Sbbj+pY=zF17oBX)u2-1>3u7N=)$wd;>n){Q~Z>T+d&t zb6uV3#MofHs&4+JjHjx@ey;G{8%GdtNAyzEvO~bTAcD~wdSn&kyJWilL%Zlw!$JvF zC?E&pbzz9iQa$%#qJE@qopthLZmx%=xIfi^lAX^k%8hBvgxsXw^_4F7{7=uB7sKzl zI)l6presAbc0J4dBZ7TLP>0725%wX&no5|i#-fRR-#B5J9r*GOu$4akFX%+gokU#c zGr1*pFYA7H%-<7Ug*oNq^p=pCy^JWC9n}yF25}yaai^K-)}l_%GT}_mj?!jDR4_XL zn{*3ViACt>qYSr1#D^+)+xub?>^APHJJQtI2 zDY3}~kqx|#c>qT8+y@-M=fEK#!i%RL3ew#=&SI1wQn9mz9&@iapfMwIo*y$2=Pcta>a%Jn9v@~D+pR*Rz`>D z)^JmnfVWg1vr)eSNYScaLK^gT;(g~i1{T`T(eCg){5a#FpK!Npq<4PiajPphbRubG z-~0$L9T6<{ecIOG5{voZc}T()L{`cVYD1)dep$h_-iPK-^UBGA(yE+Dsb5~Wl=S~> zqX!+-X6pi5-}TA*DK=VMallsFH2U|ME(%t3Lw>OG=<2SorVYv?)8vAD!dbA?G#}gK zt94zCG1fvqxHnrVLphqSK3*BSf#o&3tu3o*AL>q06}g}f`IbB@NSGaJ)tP=F>(qA5-Ja1;+iFqSj&$kl ztM|_i*xiPjmxI4DYNVAN-M^0f=)Et+USzmZSC)>qmyRW{s2t%tub4`iJ?tEdavL}{$`hq#o_Xd z*wJG?oZ7P1YLLu=4}sP{b$QWG>m7eL?DXm~-+NWI7f`k0ZqGJi7xpm2Ot8ne+m;`O z08s}`z)RF4NXVP<>B&FF%|>2c`uY>Jc-R8GDiqzTWLKEZ$N0E2bGPn^4@ zoW6#bY_tf?1;m2^YVKpfPA76tvoranojJP%@)dT`_hJ)u82=?J-13@PGy~t=>XWOr zWfc3<>-(TTNbmFtkHdcFraof#&xmIz+189XdY9I(o*JUtHi<1a;cm}kl}o}chq*-w zgfO4nHrOoO3i{)hRgpWcDcfhR8kWya6W?HxuzhvpNX?8zABfm3k6)--UW;&TUt!q* zHd<*hQrPc8e?{l^o-3n$%9@x@)DPJ6okH$5Mb{wu@zv%J4)lm3wxj0+3xrlt6jDI9iJ0gov$Nz>;+NJ46q;d@ zBahH5Q0#J)^NzVt%*c_fjl6*#m{^Gfk|Vt;O{K}Gk;=6@$;%A?pw}CTx#q|18mVC^ z_(RM=_6kY!d404uXScA7iEbl;r$z@-ZC{s=Z0{UBJ;nEejPPljSn&g4#cj{|W}F;0 zYT3J!^_}|w^_>m6GEbZFA3hAs?dQw~{&)D63faYa9WaN49jbHq2`jzm*UDiqGo#M0lkG zZ|a}5agy^FDrvJz%bpLxl)K(YX161xTa}S5z8~V;@xKdM<)ZTRI=l}nnU#7=SwmfX zdl_$!hJ01xUQw(v+OwlY?sm}8W932q-q}r{$~TH1f8rW2Zn=Z69(Zd#Cn#{rc%j^+ zdr1KvL%X)m^hoW>O63!Shhc@X?&?wx{d9k};VANd8cY_UKf8~GcSjjyRI#35HjUVB z?-C^^9RQ`#%2v|#*&ti5LwBl?h#0_#0M?-qtbK z3!clDS)ZghT2AY7i=~}k?PF&NzO2_&e(GgWsR>n%Kh}Y!>s+Y{JsnuPyoWpr_QPUr z4-b%30Z8Mg8{`8DyK?h7UE405=MOArOF_|E-Zw6bf?VaZ2V(Xfqx> z&YUO<$Ty1Rnt%0&S>EcmYY_nwdH|T<2cX2YC??#;aM8kH+}sJS^D7`f>1=z%J7yZ0 z$vR|qAL~sY#}4&Cb>ApGN|CUFL|PmU2jzsBZ$&cHQ$?81a?jsoj;DMc^Bvc~87{%`hiNw@zF-bz?i?nc(;Y?v6o%Sb8s=A39nU|gtJGO4HPxopKH|_!WE^#G^S&z9 z2hH>3BC`)1A+c<+onQF(-B`4*K9>9j)qxwbqiH) z5tK|;lW(SaTD>xq#N?S-SRf?JydAx|^;yDmiZ|JFFa$y=e*x2Z)onEAiXnZ zhQ9;rCEi2N8oR+EFoOgdcl3_W6p>l3OE6;fINQ=i6YU1#00tbXV?FW9oMtDV)o-<6 zWvGIhFmLKq@jP%SJy#XB!-g(B=1-V|iaE*-+-Ib74@+E}n{7(RQjDk1nJFnt{LNS1 zLeA+{!^MMG+*z5b-k!*~C-Aql@C%)VDKJ$-)0Nf!8#rkKB zUmOb!sGOXtz<)a-16tIyKHVC~YFVEDmBZ<76I5KI}dAz$?Xz}=bZuWB61AguR>6f7;mFZX3bH?DclH^tA1+IB6f~L ztJmcvN_V%g7DZ2Of$|t}nQYJN5j)ax#2QYO^EYe!=xfv3XPi*$$NaZWxn=CG3j%jP zvwI`$+UCMG{8z7B)$DmjKXYJis1w!z=`tSBxYwy6pNc3KFq=7LGe}PstmR8b@L?yr z4;dd)&_RJuGt}g62$AYdL#@{*qf%+ zF)y&+Taz7~60C2@6-T}L%(2l6lN+*L0Ke!2i>||NDgvrYu`-x+BCsi(-R@tc9!W`< zt1{=$_TttjSPxL>NkCCF7MLY{uEyPqr=jj3>DaKF4m8$LvVA5i@77JxPmf;#?rt@2 zE7ne+BTe5r^lrs-C47&4R`NSkL)K$oLL|b&x=xGS502eaRtuN0wrDZ90dt*;q3xd+ zh(pG^L)Lgoc28Bb=n+dul)2tUelsM{e4KmCwRlS}v)4qSrS0rcR{1f?U#4L99ArC} zotN&vEjegU$^1zj_g&Lr31y>%dM!XKl$h@+V;g~hlmd=#BEJ6)$8WNPjL)^8TVP?# zVnZ-LyaJjh_V1gsXR(bRb0u9gqT%mFx%r#>!2wI!3(odTXJrXm72Krk1hW1Npk?+I zRanW>6{nRq0gv!G+&NelAEs%crQhwZndwkW{vSo>9+&j}{qeQdTx&^NZB|}bb7iiK z?p}d4Yi5>IRF*f^{FbRHnTCQ~*1Fg#rSjHAp^HjpN|uT@ptK~l#6%@UK}AGBMDBM! zKYsu4FFZaEcyr$8b$K>4K7j+POr6G#3JlaGv6cP0_5>&0&-WzwPT62yPBQRP`t3-A zK^B3H?uy)p;e8gB&T^(mzV=VB$89Y&z}~a$6(*!LGnjtO|7oVhG2o^qwdG1BaXKq1d2CF?hstFX$)1o~ytCW!N#Q#I~+Ix;0q#D7-ryrSo?LWxBrUt6{iysvrRJ|Z zs;in1>yVzb$IPE;^^5(A+K;|s9R?+<(rwkwYKL3ByoI^ialW*ROnL}sh7zJ7CCqoNsJ@^CZ_$2SCtHLak? z+REAs#Q>mh8P-@gei2_eh_qz-?V>GO8iF` zK(l_K^Qv;Z5mF~iMLoEp72uyeB*akC&GVjTS?BzzvX(}Za5f(6pJ)jH5cFWlR>+;MlA>zLY-A+^l@) zxUGA>34F1dF1`bd8uk!C@O%}b-=3Fc3a@~VjTOK6|Il(Lqi6NBp1WDnR z?jvyf>rhMD%96YX82wdkV}tfRTr|1$AU8scgvzKNNWH{m?P!srtq2##?>&t=V^sO& zgbjn0JxT|T0ZwAZ*!MomYOBay3ZV7BaO@T8)GEqbna{qU`~6i-u_}S}h*g$-*n&$T zj^O;DX#2X#?61wdJHUU|3pSa2{H~@5o*Y|GXQZ!3x#FXG5@VWag(>HV#|+d0uybW= zuXk+xiM6?Q3Uh*i5T7$`8DlFHIoDBj8UvL*L>9jp7k=wOy(rC$VH z)lGO1qMAA(cNwX1dbmR$0>!POpNlqx3$rYjlROZ~wt$^YtCfc#{fu8X354{|H~_|Jt~fVkqikmKEAv3aYw#u}&M zB#JxFlvXlj`1lz15xCO=D)u(lP18VPYZm}A2X^;-uPH;4Sa+#t`*6EWmw=_kauV3D zw{k-yiyB0v-jOlQ-=)(HS$NB(k`ML;#UfhUx`9YvpaP4DlnhjrCYiqC&QZ*%2(V9t4|p`hl;X-m5EruQj2l! z@b+_&IC8ud7H3Sg@56qpA4LXE7Q@;Kk*S1_tx+0LQd@k8>+-}zNso;1@n_XYdb#!V z`rP2=WrBr)WmEk-L1&)1{3q!>?5LIO4#=$`w#&f-y*|<#lR^H0t0&h9Ovvi6_3C9K zs}`eOxrbUokw$G2p&uB`{$rN$6X;+YCXh}yz71Of-%=s1iM@z_eS~t+FlwK8uX0tL z>K12g@@#14wjZRWo564!gkxVdqk=cH5Hj79%-mt2SlI*`c0-WjGYIt)2 z8N~OLmQ0S3fCg5XYQnBEzVl%KDx>WtuOzKNb-L~xTX=@!T7>1nV@{i zMQb5qwNs9<>PLx>vdfXg?GkFb&6hYpJlwqTElG^!U-%Us6z3BL`J(o7?PL7`nIjAv z;UIks`;WGmTdpDHF%H|E`ZV7-ad#4QgWbS>0TdGtT5DqWO7XaSgp?TviYVFCA(=nh zLZ^u{EO|@Wo|a&aRTZj0iD=E`8`UR*YZF+3HEbaX%_*DIBHQ-p)(nXY9V@|77g}a2 z6T-_mD7f%K6aRq!&-}p_1QKv%qVzv_2GnHL#G*KXg9|;VikGax(A&h#A4^K*HI?f6 z8ft{X2k>R%5>#aUr5fhYN9Snf6q@iomN64_z_A_4F6v|DVXSC%4n0Gi`~Mg1P~c3I zsv&KToy)aeYU3Ne!^h-o^_msEDa*HlP@K9q6OLSUxuu~HR@#3_ftR%K96N=Z0Yjme z=y7#Te9=sFGB`qcIw8cJMYdO7#Z<^lNL;jF0n!$t@7chn>FOKAzj=Ccvra%NpPXW= zabLuv#WyQ6#RAlBx~3xgx9chW0UKI;-WRF~e@rm<%n8B&GFS}GSn#Pi z&uZ23e{rt0I?8$@YBk+}D-LKe#b?X2?j{E0(Q@8^j1-zNeFAz)W*UPnRS{k)e2jy` z>QIu>HiUinh%n22OYhVa7Z{^0It3s1plwLr8cmekg1>RBS9Kgjf6~z2`hXd5eFwrU zI)=_%A60K*AW3iUlh4|}-(^_n5!fGj_6OZ3c!U6E*hw5cVILsRSqzKo zH;qDz;HzO0?@CEVsRoVAv@nhAnJ}zm+B31dYm!zX?Bs+q4tlZgsRz^7ReG zWYHr7$FbwEs<^C2iMZ*0B|B8WnmjrQUa@H$93b7B${tMDg@y=wBd)t_k`;WOa6Md#Q z|4Mgv{wOwTLMHKrCW@`3((yM(`r2MH;+Y4PHK5+Y*S$*&Lb*DDm`Rkw>@r{0pNpO1ZfO}13OjDKdVQlB3du0WJF@*QNy z3PmK~3+>3jS%n=D;wK>&((pToG;GimX+#~<)+8)ZHotUcT6xp7ohEvvbdH3@=s2JJ zIHbN7Z%>!(bn0)h(f^KFgVTnmc$lYNcKZBnubu{;=46Z0`L8%;ZeQ2}OLH8s zjSX#*Nk%T5S1hsm3BfW;rS+QpQ%E^*MUPV7j0CcSoR2Kcg3K$(wEB&?yx8jQZqg#xi6@xD=E(qH-IViW^e%Fb7w<5h zh=wsG858(n`-JW5qtw&7lzQNr$cM#*QjZ66BT9!&1VYGgj?8k4& z%RI*{8Jf*`VG}#VcPEXRP7AN#G`@-ML0D{E3(eW*qlVMyo5kMvgW9FbxCQxwcA9d- z7;$z{Bw9XstF&xTX7mVoofX6S)@0m1_iHEmYeYTnp{_Jl%DL$cC4a?L5adzG!Y9!ahv+kUiJa@ZVo52rCcOq z{es^a58;$7ApK9~qGagxaE|eSGCTnmF`mptJg4_#d03NAm9Y=>B+*m|?dpG12*g$r ztt2zF!R-@i3$Ri%+(hm+T@ z#mR>DXkMma5t5KT=&mNwi2h+n?Mc!F%Crjvj;*;E*}3+U+hgQzQhiwr#_6$KEcNxw zlKIz5E*+RimOw1kCS7Kw~Lk#<3*Q zzOb>TQ7lB|wD3<$N9agNi}^L%eklC@O`b`ZrtLD(MzBo zL!Hn1`zp+_v7+u(n(c8NV-osm!n{vO7w2s5(5I^GyR%E% zj#@GfIc8HFmuctGzNoy3GXO%;F@%N`LcVup)bet|d?PD#obnvlmdI(dmquUbg=M0% zKA@W%&R+6LjhhCMsWNq2d|M{1{96QAxJCs8+)HRCItLEQl;8-I@hO#7^RT{V9 zW7u*-m|@)#+Ca5>OxJ2?6*|*!eSv!?_9+v69WU*<;&A1uhBC+p9$2;f%B9(|`-zKY zsrjzU9KRF2OO3wv2hJk=fZXigIytq3YoP3+UI1=Slb-+pHUS8J7kIGk-@Kfkn*^AV zrvHBM8c@hvTsdakkX;v2%~HBm(IjY9kDsctv0x}eJ1&<6c$97jZI9OrQU#Zc&WXCm zYQ%&LklWF-+wS`d+aqtiE6>lY}n$0Tq zWxI1PwIR%E9`;Y>vYXsDp+JneQ{ss+tK-;7q(7-My2nk_A*14*^^gEd;o$`@dc_%c zDGwqH{2}U>%FO%JFltOICb+jBqB_Q=VS1~WRwTwOGu6Qx6wT_YvSpf(G0nmzCIS}k zs*O4UEGXcwr=Uv^T@y>Rwkm9-{pYkYk`vcCn(ULC{ckoKq?W7z)jPb>b6%h?2>T$3 zxcsZ8$-;Png-k(cSHdKB4vQ48I#(r8LQ@g!P~rn$wot<@2A5lT*F-!_EOgvkkdv~} zV;YNx_iR-qIRyBl-@-oxcP~^}*LFFd+&LheyZqDueSB0%-NS|zCJk->o z6m3Qkl>Wb4Tvvr{01gGq>{p+nZW#o20TIEK`e9ixw+{6auebH0UZj%z$i4|lFAqBg zaciU&`$G3h>nalU#BBhAOZou5C0t=JSAw=H^VLuLZWppzuoLdbiO>SR z7B7{G#yl38uK&xKo1ygje4?%Z|K+tiNi=DUu<0`avT>Igs4q|d%Z?ap-S!bQFFR~6 zbrsa9)tkHuC-3K`Up_{zPDlO4_!%nwTstwziEUdw!>c1bk&aGId>2Aj%bUZsy4Dca zQ^7M)6{^LMyF^9KB#&5$4U2=O8ZV-3pdGy)s6%?YP^<{q+)mZ#}D6+l;v5mD_aPGQYt)DW5U zFsI|@QZ-%CJhf^_B#ik&YCn#~2G&yM&*eBSu+GFXgayR;pI8VPXd>li*2W36KKFPn zUD?vcuA->!Q`WYDD=DBFAMSQgm1t4Fs6$`N)yw54Nh|2Hh7xs8mN-wJ)k$nCtNbqF``m&O3!-6HEXTL_5KmS zdKVb?H~K#7<*-J>4E{d$3E{L9HE}FE6dO24HY8m4r`SSBTlL?dS6bPPx#nk^tcr*D zNF$}5xL0QPIBCrWDtHS6nrCSf0pqNHj?b|xqV5Vg_j)&sB5pM*I^9L|1WFBTuN7-L z8R(J;a4(R8&wy1A9vX%in8O4S60E7LH~54lB3Zg(_Qfag-TtGt^7*q`KVsW?HZb0bfC(Ryj-9}BrJ@QV4or2xTO z*p3q5mqE{E?wUV|r>f6^-&uV}x!1tR7OT!WOJy%7yihF4-q?BLYu8c~zqT~^a*1x+ zmriFi1)w;m(&v5j4`zw&OiP{m;#&I)ov`R+j@s;!6Q(3lI-dmOgr}fWlm~N-iXq4F zIaOSU3dcQ{0#5BQ#BVE!+M?6a+KzL?{}S$uHEyNvzva z?u2zoq|iBcvgEYA^b+|;XA8hV`mp@sA=R)jVV`&rPUACmWIv_eqWZHW=34xH-ooJ4rSsyt5X!XJ zoB8HoR>GS}b^5dc z4%X04xF^3o3nRz_?Og5HTb~13E?Rx^*Q`qGpW_Z)V7T60AMgeF6Y;J*@38%N5ceAU zLdb5H(VS~Nug&|6TRx?X`SgSH1o9}botLYtZeYO%-7D-g#o75|rTin72pJmBzOst^ zAn|gKtwer1!Cw7}z0y9h64c+;L(I~-NX=M~BcqP8QXbPnpWm+v_mw5Is5mB3Q3d?q zGxWXgCv{%m=)7i7#qA_H8)>0c;2w%Er0^3)YI(E@#Vw1v!g7me{zhMn5nU&)8H}nR zqUxvW5X$9l8~AxJf1RI3QhO_|dZbU-(S@em%1hK(gN)1kcRJDMYeq zvkw0xp0t<0u`~@{MR+}tdyXa6hFkxgCtS~>=_o8GmD9J(k@!^9-?N4=X#WR?Pl(Dc z$_K0;3*SPLtCUdN)BI4meGaf5+q6=(1|Ds_&B&bj3I52mMzY+42(j`pa!iN0iQ@lA zLjDh(Gk@+HSdo1<#%ATg&A#>Nf=8l`=@PZh+t_$O<(5;#C7eD-jgQW^|K5+?sC862 zHr4?JZ%^z9MCnps2WtAtZ5fI*+JUb-K5jke8z-Qk2^# za=NvcXI!*?U^{6$RVT`1jkd)Q9@a75%{Lm3(Z~llY$2m$AuB=u6K$44=8UAh_eL&82mdt8nSC|jV|5!@%FB3aI6fxYjMg{rSFid&ZvcuE zt5u0Ez@5a00vYzY@=Av}S-tlSNp_QRexqDrzvLs|G?Cj!BI56N1n_8fi{f%od*wU6~m>+J^iOJYyf=Zrti z@viCx6zvMt^OsJfLqc{t7GBgN*An+Z`i-H$Kz1D8s1ODD=v(gFE=IbR&S_H=`*1-O zs7zGUULTZ-L%_H{bO$^zVS~HdaFG(iv%jLoix!O5khNV7|h`@cAzXjcdT{4_VkoP4XH33tUMLCNIWzchIL`lqF=t8_>szsmc7! z=FU!=z~XrIk^X~*pDKh@^FjPAgKpj-o9EohGA($%=#C|#dksn`&rK&4;S4n)1KTPL zT4Yd&g#Nd~ei&YiUfoFTehB4R^2)sEWe)MV?MDhv?`n68KQF#x{91JwHerYzAP(lIttW+I z?|TwIY*o|K`oFsr0f=Uy2ID?hUpH{v3R!if6nl^xY+or5w~{$QJH>j zlk2uJpq|+641SJ`y5t<7e``x;<^#WoCLE|XzMJMTT07M3R^!3IM#&%Jl ziE!3&rb4v3*LzAqR_|`{@)Q0OVou2ZgH#|etZ5b9gs!hxA|L^p2FFTzi7`fy9z?{4 z4|hYs{q#GrBbpdrxWcrnI*~~E2;Gll>3>4G_bigxK;jg%If7>Wd5-h8CrxAPwxme% za9*T7R=7Dm7hZWDSxt%*AncmwV|y&=s)P`$Eoz#7R_UMVm~B308lW$p&Ioe7kvL|i zPI`~PohT*8f#AEwKPJ9_{vhuvhcejTyWRyQa;l|=1Q)El(Zu`HNq74b6^SihZA#>R z1@AoohA1QXAiIwz@f_EJx-WJ;{7+EdbF9%Z%s1tr^ih-hWdcSyz2uTZOvQ@0-OvE0?}xfe_ua0Y$?UGYOvR z-w;z;oC#JmjTMs84SpX}RE9Dk@Yj4(Ls+tPqWL_gD|-}SWp}?~@l`;WuvJRhAp%?2 z`=LCq&O?=cmOMQYKN>wh`y2N_);I6pI zIc#o!&g+CLfV8otY}ThA{q)v5MsCL4#2=*%MZr4wY87fKxNFM*)hcR`ItvZ9Rgr>> zX)Ip0XBI`z0^O5J1EHt~jxs~oZ*X^5eKG$g>4cx^``AkMU8)Zp%x8C#oRRIL?J1pT zNp8IZxRtY?V+BpGG5GuGpQvpm+I^b1R_bWTnO2%KD;f04w7!U1F}5+*+hU$V(v3Uni7>=Hi%$bFw8{pSC&a>jTjCog^qnaA63MJuY^*8adkZp3Z-44xrFp^#2^&hsed3i{ni zA7#n-Gkbaub}bX~X;Bc|kMf8>2mM9S+d5yoR-< z&_#*{!^L{`5OEJyb=3fQ8TCus2)ohyTqQ<11lrlya}~4yC5>!9T~e4fDq?ZcgI731 z6+=G3r-Fg6Aicp#NUC!*e-Bz{1WOk1jnl!BrMNdKlNbKB%?x!YK`yF3`juIrhTU~T zlrwS&s9VeJWn0V+`bj9CYC$lJ|G<^2aR1@26eNbW#3_!(vfpwWEcbQrNjAuQa3HDz zk6a(M47iuSZSc~qSY7|CyO{r|<@Q~;4FeIu=04em_xa%#(=qx&^V#H_*eVDtm8QS6 zSHHLpx~-lu9`wL^E!y7ubyj#ThnSI1_p@I+m>-l%n<^3DXy^Gu;Q2Q-&gHqi3eRhI z>n?i_UJctGSCV2qwHjsVTh+y}t)@*U5$9h`UmU`bt|SgZO0bU_;`|UwQPS|F?0#<{ z@f!~2OB$a&#aM2gs5lo)x?C7pg1OXKCbPV?b`wDMjrrp5yLxYHF8PXn!A|z67R;`v zDLzkII~)<#U(|^Kkz)G!7A_KIGLX{Zl3USTbcU4KsjZ1E$LxVR56^EvMFwiLX!9U@ zzcNCWljxIySxZa&0(ztxcw=4w?HQ;wT2Q*510sKmAslF7)yG%ZLCWmY;D_K(N7#F; zR9#Ye)FwpjmBNkf!1@q??JYgBgMx`HnAV z_z6rpe~xrH<6fc?-iEGiOuXosx)0Kq*7y>TTYg6zfaY(Yp8Z;h{p>A~_1G`KLmh!q|yX1|fwRqKe8V-+s^A5gWs;aNY?P(`P16Yjh;i!MnWMB*T?Ic6* zuol(^wvA}k_=@5>Br~O^3IVV^?I1tpDg34^)18i=L929)3B+xeFPVCZd(oOt7K>2h zvKG8d`=l0yMQN+x$K%giGddDqqeieOk;-l2kv>+%CQB|KME_;h$(p@7$<*!>h8#1g zhYIHTedsSJ4ed4C*p58q|ygwE;zteSXUdnpyyLaiQ)8KcQJ9Uf*C98^b zVkGh`rotFw zPo?Fm8pmtW#Sgp~yGvpb*c)xqso+WNPF9~JVXHMr5~XZgs&`!W$(Klv!9y+ZD*gz! z-yF_KuV!Q7HnL2|B2~U4Q<7t@s7l8p4@&oyr{t}s=XS6fGGp#kR%U)R72bmGwv(lQ zhX=kz@6H`0woIH%qQ|?~MfD~5=<6nNg4>fw{PFGT&SUJKECd>W=!pgmS#HoQZl4o}*rhf>TyR9U z4}F&1k=5Vo%I!I>ijtFfSa%iJ-!Xx$5r182bq?OILilRZ1dGhkn*3{wk_d#=1h=ad z)+p#lfiWr@V+M}6esVNqTBZ<1w~X5xx9#U25PY)t2~%?RqIpd_vQSc4bH23sCjHfX zYhr~92ws`~gXKA10GYN^rE7 zp;FpAgwW!Gi9df>{+}UK8*e^888O=AK@w>q48V5Ke{Zx%>@S^y!Z}I#x*x@TG zo({h*J9oF_+6=#{RPW0wGGPU2ciCS{R}D3n`L`lIvI5HJz7%Mj(fO6e{x@E&2l;u0 zd@T4vm+QD;M1D(-f!`?s;}a&8|HY5CdkbuI^r>q0*G@D;%7|@iP1$s#=hazX8m4$8 z|Ic#><@^ueZ4{%dskrcexxX1fowU2zbF!eD+(A(lo8Pu7c$+!(I2-$NgWonU9#Cih z_>rnBnPg@Y$E`Z1R>W?gI>STAkiL;q2~8ars20)>j&%D?_3!vXjGKXGf+#0|0>Z>u@KgMLhr``LcSKey)j8V^GB8vQgFzd#5QGQ;y!7RtJ z(;mCs`my)=jq2t>O&5Fq?XD^>c)iP6Y(Z=rzd|!eb*DZ66>yVsh(i;ra(jF(boJ(Z z>^(9wCVK53I;f6^n#Jag_p@#_1q%7%biziDJZM^e*Ba&ZpXKvmMz$li`BB41ULV5F|f(-+IX3@!1rW zN<873Kkqn#c6suLh`_V4wJH)#{WyNX2!h|$olrsX zl=L2?;&aA2@dw{K@mK$Z!o|_Vt>#sY>FxGg#fVJlCz<;Y-y#eA!!7{wI)HDv&R1@y z5#*H%%pyL(Pt84|nH#kIas}j_>1sJ-#5b-T9X_kiRg+&Z@-@8#iu^B_mlZ~ z2{=;ngRDkF3hVwHndrDbmid%Zhv_>Ex{!=!r>fdKqi&B}OtA{9par_bUgFBmF+oPT z&3Fm@4Z1?qupjVO9xP39%3*U;4`imK_9TK6errjAx53CpzyTU3a^o{Dlfkw@ppRH z9dOo5XDkH?^LIcBJ1VsW-5tK8jRV$GS2e1;uKgJTk#;|E-EMAZ%lQq$uX?t!fyVqz zBDR8dWeH1Qb)%RgR3)3GnCMDl@jI#O+<`+TnnripElxx?`nHW(-tCv(n+{IPVgg|P z20#8h+#hvcS~*bAs%{&hcE5^@%pmrZNTy z`z*DS+F!jLt=s9^4EfMcTWkLTPFKFk-UvNwO}8ExCkuRnezO)PJR8hp@+Q2i#LS_GE!3!NLxjaaA&uPqCxKYx5kt zIicw#jd9!1?y+245B?MLjvlwFrJ-vgf`yvwxy4?o!9ES!tYqLLy0bhP-uP1^@ikFd z+IpuUQXgCl|JOtTe6f{{cEf9s^|dTLUc_svy@h&5F#D2DCekD9Z_B7@$!^Xjz7ErS zRWspFxHOYEYC~|SwLp#ICRl$1b{%>|`KpGLy*lf6?R7<9Uep<-u?BuIrKI;_o%9J4 zu6qDC0V0gUTc|i zg5em^47D^AgZ7JTlM&M%GPOdD8AoiKZu?MLZ?-7WJIiG4_O(V|XOAr9t3U+2yoP@2gR%U4 zn*jV;72uc|BJ9a_9c{iu*!~W(1HeVV&Q4(La@ZEB^n#bH6!mX#5nCCslyctW6VF(d zA4Fd;ekvC(CissTWyHsXm|CaUF+unZo|;)PfEdDk#i0C(MsN#)*sQr4ov_JT!u`ao$T8Lq`}c9for&WYbYF^tz6;rlehp6F=hlZhK07!h`on?X7FGD1 zj(<`117rl#PV}@hbWSAhg87jP4+B>6=4iI{ebk{3^t_(tuM%(ahoYL6&8=YR9i;N= zTycwpv0dmg5tz@J(CkSbUGd3N3X+b0qyLIAn_U$fX=ytGO3){5kN4(x+5?)lJIZv2 z%H0kz4_5it@R!d%c~c+s0Epn(YA}Fspf>IC35eBurFW_D>FcCZ~eQQ{)WBo+7=0!)a_PS z&jkkl??diF(au3~0IX^nd8e9%nWB;^wCV3|H*gPf z&emZ2^(IeEy`ETuckeE%i=SBG=&-IBX)^IJG^{=npo_;{rOIP0HZSDn1s1CV`z!Ej zCwN+~hs7F^0det*vw(@}uk2@U`WKtiY?`DG&56cm<1_OlRv~;#eKvdtj61<=?Y9I<_eN6kO2%%Ml~}=yAS`Z z2^rnuuYbU7gKuG-!og#DsPC;q`3&_9hVAhnWaVsnElaZo1FzR+-Af}YcOdO+yT<

    i&a<)xnE*i^MX_DWoPkcx;ejl zq&(xEPv2DxDbLxv7ub{TUxkY5LazZA%OMxpwc>-h+CwVDa+j5guCy+<`re0(c56x! z;r*sX=9SNxn?8Wg;o+->ckrw|(N=1g;?u}Bp82(5oWer3CfngtAL+{Ra{t3Y(emXx z0Qmrn(wmS%fiss~E7<9z(x+iAi%dmsJTH22~XKkKy$6IMU9`)*%~Bd&BWpkM7lef@w~7 zFA{X2ZEh3&XpAFyUk#((KuJ%I_e)|(Ts+`~n(uc2XEk36?^t)*X6KFM&TUI5v~1GH z&TUtG1|djN4F!MQAHgA;nDzc=?R2BP+>%}nIUO5G2a7p9nKumw4lw@9zjxuOtns2M zQ&g!qBYWIgR+6b*F;{|bV--X2wwV;b)PNr>kk*tr*TOnua+G#J&g^ror4c-(XjFtd zNkJJS)n+Ols`U44zwiQ1^CRW-d!3*ZXam2`6bEmClgy57jRJ~GGIhFR&>N>RR0znE z(iU^XRqI#MtRTd<>`U9Ah_sOH^waUP_;%bfIJjq(jx;j2kjz7GT$L*zhlV8DpN(y+ zNw^RGhmr)Yw3bZQg!u0m9f7=V7|RWF$QeCr?V&KwJBVo3^TA64_`}_aa?(2V?;AKL zV0TAut9aN7j2P&5uo@7!jb({fAXh`>G|k_wkXc6`(XD=iD4poRY~4r~gtVUu`N?tx z8RBU!xk5D396Qouqvh1XcEW1vole4K;*<&1f*8Rr=JTf~_DQ*tEPu!0%*c4jJn@)2 zXRUtH(9s0Beo0`-;<`o}J%v1ITQPc{HOJw@BHT09ggi9v%(b{EsO7eDi*&y#ai4Pb ze;~fKr3?J%N~jXF30O2t7&U4fPqSIM#KVH79&7Wg?Kb<;jQJec7IqxvWP}vS0!8z* zj;Oh|4(9kA@m#xzCQ6O1UZtw02+6g80;-tvI{#V(332ZSmB-Anof8!4I~?W?%8y#B zzbO>SKgk^b(&9JJSDT`-w>7Vhn`?{UQ*aOZO?0vj7IXwLLUvILpFY`ZzMkIn{;7^`m&OhSY`AMx3 z`+PRQ-yTWTtv-qO>=vCOtw4K8Q}*e-Q`?RQf$n6j$%CZm+mJ1MVpirt^U8m2IJ}Af z+gIW@wpXV3R~70QulG_PP`9+|BPV!{?jOv3`i6SzcI$6zP zL@_RWmN5`YcH)%|kvR|KWmNM)JZT@b+t@I6(eNanJC+5}*AS!S9Q?9RsR4;Qxn85x zlX~+%zTMR>_@uMa_oAD!y=q+Ee9SFH%b*vF~A-dh|K@Hw$E_c zu3v@IgX~X6w3ym?;}4iQD3aB%AyQeiX|Aijr1M0Dy6J+|Ibj75qi4?qZ0di~NPJZ3 zC}9kX4=O`1O?Vra<8|$YVDg4j2$^81>ONtrwQs?)GD| zh+(vJNiKd!*5*k?e~hX$lf6R5`Z=Nx&N_z%pidPF4}X^HxSxxWtQH;<{%Y}RPm~7g z+3N^Kh$^Dm%~f0eLmXY(c8O_NRMJd(r&*E@PZcX8m^_0}mP*6kiD8w_@gLg;a3sm8 z;4^--%f^Jp^!;1fLfOxk)*f|~^gd|jHx-db+eAWo6~P%miD}8FSv%&UK4y&%?V^OU z$)=}FAO0+N0R5$2or{flfpXbol!>}X*Z|F~6Ah>yQ52lKj=pVW{sPfSuf8gw zI1HdVvXT>@>4Wa@=Ni(Ir3bNA-#C^FueaE38i{LeEVq{DIXW~)BZLn!+&E9Z0ROH# zYX!KK4okGiAss)iVlQud3l`fd9ZSDV+>rGy(`$aK1o@EAKyV3p&~4?|K5ylIoz}|w zifsth?OUi$xQLl(~IJ3GldB`>o2`Oz#) z43n4PMMeN_0cqm80P=xwa?L1B6q-H8N|sHin=_QXb{Z5|0*JM&Or!f9!V*GUL zWK4fbjfx+JUD|m?ky^&G2 zBq58eZ3jt20?R)ERQ$*$2?MSFt`prpY&`^CZb=#M$><_HxKC4@mbc8G!+p5xXq4sxqpYeb185 zU2HFkVRhyu%Bk1SRp4)nouBj;aD=#od*g<2r+T~zG7%dMgkR%vtQ{Qz-)Ka;N&CBl z4^39@ge3&4PQoq*(a#6Om(a30ww?6i21J5v!c=1aR%+r#{~{^Bo8}8d_nVJ^|0l?> z`OLHH9Iem8Y^Q6uuI6!M>!B1>^C|Kh!r@*YjjUwcQTJx6n^s0+Iy4i6clXMjhg zLz5TPFn?}ut%rC)3#xF=o(|BA-}m~;=P<5h9f7%Ovj#Po_h2OJ zReF3Yi0h-|raQJU!0j1sJguHZ?9M%I*=^a^;u12oyGmS56U^wA$^mnF1eFWkkA{0m+JA)ieFPb#UFP9)^qRx z=v^h*7W-ALjCIalsasoS%yZZscH>=2Fn(8dD3?jOG(kAw?)i_=0NYSQib#W%^UwHr zL&ICBrKG|R?^F1ZOt?7E=?`G;aUXponr^Uun~UEyTH6eFW~zcHpMjrr$A(zWovHv= z!_x{JREtlM{TVoakg>Y;!hM&#*~x)9Cp6fyhX$P0->5@{&!B&><{rx7**{sTn;ovo zZ*i&)POm~{w&fHlaSJKCVSf>--88%*I({!ZhSfB9aWiAESSv zyx4*W3=BZbM)1l^WOX*C`Z*p7rV;B-BcVvH4-qnP?9?W_v~j+sJ8+*u-J$$lhcD-jT;O z(h?*g%5NItbBU|bIg;pWI~|jIQ&AM0+HwflUW+F`W~<`;@A6hovPxHnGnaUfc{@q! z)Zn>fAN}IQoDDR!bBLEL*rTZGrp~De8}S#CC3s2qa!jd)u@39WI+%$Y*r5JLy#b#} zN_}s+5`!UJQutMB5x;;QwT@50iY+q-(mhgpPr&B*B>nyC^xpKMlsdx`_M`V+@cfz* zcSG_#3W1X{syF3~wbKDC%_mSyOh96OY~2UoOO0#Q_AB<0F%k8pifA+6v;l)svNo4K zHsC8ForG5PpzF$R%YpI%T}xhKw*lM{U)PM(0~6Tq95K3?Rk!4K?Ud4P>F#zw?05*M zQ90LVY!|~az^Al{-l^?+8c>EIyS|lOOvtR99vQL?Em|9sZ;{<5*d5Xz39(G)=E=l5 zHTNR3j-yQm%`fNMY$k~j390b=#m=AQB%owbq}g!i+!P=G+?CeGKoI(&*O$0`HImI7 zWOiPq)TVT)G2{a(+q+He+9K79o4ce?=D|HpfG?Rqo3{wn@fMTeR8wZpW%ZD+DB}y+!GbUctjtCB7xhndguWkfr~Jp>=N;Denmo8R z|7#p8t3lQ-*^@T159YPBKg#bG;YeOST8^K^21jCc#1s|pe%dQWzy8mI-a?z(>{j(W zH_a?wgKc7?4V>%=KZKasnLRZjH*JA;+xy*{LP_WUGxyOOFpU|qg=HcbYhtPLs z*;<>}uPZy4YoC*L2Z|E!3DDkLw$lUE&J{%=;61}HtZ^wXgZvRt>GjHo0@0nrC58j5 zG0jxhTG%W5U;|prX`Ks)H>#tbP@aG*8NOTdQq@-MGFP&+5aDxKKUcr;0Hk$L#MG@2 zki0B}Ufx1KBgijWHwjahsm^ z4PQBmUKqk&aK(&f|LWBjsTJh&$PnpS(o#{x4ushgv=U=hr@Q6>9<|^QtKIXheate< z1$Xx6{dR;?TiWoV0~115=9?n^qK6@sm>e0w9!ei>JIH@7^_`=K>R-s|vgv=yzsPcq zzk`g*J~2vhAINtvvV~ry$MJ|b;vSKS`0;O^b>?m0EW0?_0{d6}jJ($%RpV6BjP;Ae zuor-w`HPAtNbl;@(6=!lcf(jr>Toi=-^b<;*|C@CUL)Mu<$!+*_ZC>~r!Wd&yS({? zLxls;ZPF%$J{;K~54jRs8>fjR|ptFGl{k5kexnA zB;PTd1-_LD&q(R@sZ@%&VVCa%h0R$+MzyMVWl#V3_i3v|b&eT+lEimIT*kJ1!2WX# zU5UA&9mcW~kd-avg@Wv(UAAu??f7Gkp>lfN3u6?OVPwZlviyChz9NcQo8j1sY0qjX z>WN*YYnNo=jP@L$7xP7H5MI2ivlnlK;f*Ovtc0& zp8}st>?@}ZzPw3p3PM{KLKD*pc9{?AePj+KU}-vE^$gB;eIP4FqLnF^1GVn4x^sB9 zc80fFe4Zynxhg%GV!opt?orLaXeT0M9L+Zx90~C6 zmrZT2mOB3mS%4J#NVoiMIx`I=I{WR&JI{ZMmpU^U_m-~^=>b6uJoEtVY-C+4QN3+B zIM#cz+d@nVK%8jVFCRn=EviF>3kCk9zxl&C<>8bw1SW0KcN|j=)1TdYnmk!O?9R4a z;UsHOwURFtv}O7;LDyyMbxA$p^r*R1_W?hHmhpi0Z0mVJO|`RV`b`I2b{I+DIhh3G ze}7$%@js=gy{}UfDLa-70IoJ&cl3L55E-=>PfSXPG|VBpaiR$SK|I1GbRhsLI4PlY z9wJ-j;ZGCvN5FlB#eh(tn%qeNJtT#@|vncv*(TX+G^H(`8j<~i( z!H{6@9+lcWBIfo@e@Och46dpsyr#fnbd;s8#6VfqxYyBU${E0;y@qtm>E~_Q9_76X zRgV-ou9(s;ay4DVwnFC~#ZltrUl>l3MY4-pe2(@rVQaDoe6!bzR}=~+>;){qGlq0Y z%)~7WVLyk49>yiZgM2^Ww=1RIej;x6v~Kba`?ZtxY;&k$qi@HWf7z3;NA!(i*xJvd zKVS(a8^~9j5*+n2%V-yP35M0iiP>hG=u0`}eA=4c?lin3ZR;$~ufNG7RFRFA9`Jq) z0n)U6b~z@ay~hbB;i5d%3i!<~)p3G%nQt6&U8yHesAn}aD(q=Mhi(u*k0oThtVabc z1QZI7KXhZP2rFd#A)O|TZgesb%Sz^8jX96i1nnWVKNmS$ctGq|Hs-MBJr8tC20 zBNyu%AcJGFfi{rG&D=!^hWVeAeK37O_7TL@h=#!TSqYVaN8WOx$z}tnZGi-AWemI6 zunJU(dWUEvkKsqV6Y*w%htMGnD(_)xU?j5ZPu z%ggp?sCL2beti<-^Ft~d<|o#l7C&bqM;R89NT?K4pXL3wJVXrffxcruCtAsof5{Z- zboCiM(Q?f-(0oP&l*oV*5^{>&vJN;*6v^}DnT`47 z&%kkR40bre87}x*yn9f*Y30@ZXr|^IR=oLNNFjsl*_Cy`Al|uu90@*(RaTNA+|Z#_ zw}j4bR~8~(vYx3NEe+4+r;&AN#!7H@Y$7OX<=GTCACC!K|L4d1+mvn(0122P*1v-c$pb~z)aMZ9N{H-)5|h;Z z9i+MS-U*)e0y{uGkLh~vz^VlUm=n~zQS^>8kB-(~m*2sUImpOCp7j#TiDrQ!QTH3A z?a&(kk|$X?#mbp2VK!m3%qlhzEfy4ml(Q&f6Kf-DBO{{^p>AuG9(v)oIFzx`CuX~i z%%LN{w;RU)-ny22`C8nSbf@H$?Ef!RsZB@bAO-UiBO|llRnYo|@k8i#kT3acN1H@PDK$KWzLi2-5rj^^c(bd%I!mbHUgs{kuykSN=%m teJUzPp*AvVwc7jNBmQ3*0Ld<2!e9CymFb`UeZ~}8_CX_G3p0WC{sqYN!L$GX literal 0 HcmV?d00001 diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index fca24c4f1..b88f07a1f 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -672,7 +672,7 @@ def test_hamming_reference(): def test_normalized_hamming(): - hamming_calculator = HammingDistanceCalculator(1, 1, 50, True) + hamming_calculator = HammingDistanceCalculator(1, 1, 50, normalize = True) seq1 = np.array(["AAAA", "AAB", "AABB", "ABA"]) seq2 = np.array(["ABB", "ABBB", "ABBB"]) expected_result = np.array([[0, 0, 0], [34, 0, 0], [0, 26, 26], [34, 0, 0]]) @@ -681,53 +681,34 @@ def test_normalized_hamming(): assert res.shape == expected_result.shape assert np.array_equal(res.todense(), expected_result) + def test_normalized_hamming_reference(): from . import TESTDATA - from decimal import Decimal, ROUND_HALF_UP seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") - hamming_calculator = HammingDistanceCalculator(1, 1, 1000, False) - res = hamming_calculator.calc_dist_mat(seqs, seqs) - - cutoff = 50 - - normalized_hamming_calculator = HammingDistanceCalculator(2, 2, cutoff, True) - res_norm = normalized_hamming_calculator.calc_dist_mat(seqs, seqs) + reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_normalized_csr_result.npz") - lengths = np.array([len(s) for s in seqs]) + normalized_hamming_calculator = HammingDistanceCalculator(2, 2, 50, normalize = True) + res = normalized_hamming_calculator.calc_dist_mat(seqs, seqs) - for i in range(len(res.data)): - if res.data[i] > 0: - distance = (res.data[i]-1) / lengths[res.indices[i]] * 100 + 1 - distance = Decimal(distance).quantize(0, ROUND_HALF_UP) - if(distance <= cutoff + 1): - res.data[i] = distance - else: - res.data[i] = 0 - - nz = np.nonzero(res.data) - res.data = res.data[nz] - - assert np.array_equal(res.data, res_norm.data) + assert np.array_equal(res.data, reference_result.data) + assert np.array_equal(res.indices, reference_result.indices) + assert np.array_equal(res.indptr, reference_result.indptr) def test_hamming_histogram(): - hamming_calculator = HammingDistanceCalculator(1, 1, 100, True, True) + hamming_calculator = HammingDistanceCalculator(1, 1, 100, normalize=True, histogram=True) seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) - expected_result = np.array([50, 100, 50, 100]) - _, _, _, res = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) - assert np.array_equal(res, expected_result) + row_mins_expected = np.array([50, 100, 50, 100]) + _, _, _, row_mins = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) + assert np.array_equal(row_mins_expected, row_mins) + def test_hamming_histogram_reference(): from . import TESTDATA seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") - hamming_calculator = HammingDistanceCalculator(2, 2, 100, True, True) - res = hamming_calculator.calc_dist_mat(seqs, seqs) - res_dense = res.todense() - res_dense = np.where(res_dense == 0, 101, res_dense) - res_dense = np.where(res_dense == 1, 101, res_dense) - row_mins_ref = np.min(res_dense, axis=1) - 1 + hamming_calculator = HammingDistanceCalculator(2, 2, 100, normalize=True, histogram=True) + row_mins_ref = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_histogram_result.npy") _, _, _, row_mins = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) - assert np.array_equal(row_mins_ref, row_mins) From 6a220a74d61cd23906ad05d747815b565d0c6387 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 08:04:38 +0000 Subject: [PATCH 36/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/tests/test_ir_dist_metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index b88f07a1f..c755a1f74 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -672,7 +672,7 @@ def test_hamming_reference(): def test_normalized_hamming(): - hamming_calculator = HammingDistanceCalculator(1, 1, 50, normalize = True) + hamming_calculator = HammingDistanceCalculator(1, 1, 50, normalize=True) seq1 = np.array(["AAAA", "AAB", "AABB", "ABA"]) seq2 = np.array(["ABB", "ABBB", "ABBB"]) expected_result = np.array([[0, 0, 0], [34, 0, 0], [0, 26, 26], [34, 0, 0]]) @@ -684,11 +684,11 @@ def test_normalized_hamming(): def test_normalized_hamming_reference(): from . import TESTDATA - + seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_normalized_csr_result.npz") - normalized_hamming_calculator = HammingDistanceCalculator(2, 2, 50, normalize = True) + normalized_hamming_calculator = HammingDistanceCalculator(2, 2, 50, normalize=True) res = normalized_hamming_calculator.calc_dist_mat(seqs, seqs) assert np.array_equal(res.data, reference_result.data) @@ -706,9 +706,9 @@ def test_hamming_histogram(): def test_hamming_histogram_reference(): from . import TESTDATA + seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") hamming_calculator = HammingDistanceCalculator(2, 2, 100, normalize=True, histogram=True) row_mins_ref = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_histogram_result.npy") _, _, _, row_mins = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) assert np.array_equal(row_mins_ref, row_mins) - From 8683ab66eba4a484e38730c21bdb9e2b5371fb97 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 10:49:59 +0200 Subject: [PATCH 37/94] docstring for normalized hamming distance and tcrdist distance added --- src/scirpy/ir_dist/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 51181753b..41878e570 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -56,6 +56,10 @@ def IrNeighbors(*args, **kwargs): BLOSUM62 matrix. Faster implementation of `alignment` with some loss. This option is incompatible with nucleotide sequences. See :class:`~scirpy.ir_dist.metrics.FastAlignmentDistanceCalculator`. + * `normalized_hamming` -- Normalized Hamming distance (in percent) for CDR3 sequences of equal length. + See :class:`~scirpy.ir_dist.metrics.HammingDistanceCalculator`. + * `tcrdist` -- Distance based on pairwise sequence alignments between TCR CDR3 sequences based on the tcrdist metric. + See :class:`~scirpy.ir_dist.metrics.TCRdistDistanceCalculator`. * any instance of :class:`~scirpy.ir_dist.metrics.DistanceCalculator`. """ From bfe35fd72d5e2e59b69e3549fc231b602f973a1d Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 17:26:14 +0200 Subject: [PATCH 38/94] adapted default parameters and tests for n_jobs and n_blocks --- src/scirpy/ir_dist/metrics.py | 20 ++++++++---------- src/scirpy/tests/test_ir_dist_metrics.py | 26 ++++++++++++++++++------ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index ac1f1730f..f889681be 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -426,7 +426,7 @@ class MetricDistanceCalculator(abc.ABC): Overall number of blocks given to the workers (processes) """ - def __init__(self, n_jobs: int = 1, n_blocks: int = 1): + def __init__(self, n_jobs: int = -1, n_blocks: int = 1): super().__init__() self.n_jobs = n_jobs self.n_blocks = n_blocks @@ -631,13 +631,12 @@ def _hamming_mat( is_symmetric *= histogram start_column *= is_symmetric - nb.set_num_threads(self.n_jobs) + if(self.n_jobs > -1): + nb.set_num_threads(self.n_jobs) + num_threads = nb.get_num_threads() - if num_threads > 1: - jit_parallel = True - else: - jit_parallel = False + jit_parallel = num_threads > 1 @nb.jit(nopython=True, parallel=jit_parallel, nogil=True) def _nb_hamming_mat(): @@ -837,13 +836,12 @@ def _tcrdist_mat( dist_mat_weighted = self.tcr_nb_distance_matrix * dist_weight start_column *= is_symmetric - nb.set_num_threads(self.n_jobs) + if(self.n_jobs > -1): + nb.set_num_threads(self.n_jobs) + num_threads = nb.get_num_threads() - if num_threads > 1: - jit_parallel = True - else: - jit_parallel = False + jit_parallel = num_threads > 1 @nb.jit(nopython=True, parallel=jit_parallel, nogil=True) def _nb_tcrdist_mat(): diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index b88f07a1f..c8f2b33a1 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -305,19 +305,33 @@ def test_fast_alignment_dist_with_two_seq_arrays(): npt.assert_almost_equal(res.toarray(), np.array([[0, 1, 5], [0, 5, 10], [0, 0, 0], [1, 0, 0]])) +metrics_with_n_blocks = ["hamming", "normalized_hamming", "tcrdist"] +n_blocks_params = [1,2] + @pytest.mark.parametrize( "metric", ["alignment", "fastalignment", "identity", "hamming", "normalized_hamming", "levenshtein", "tcrdist"] ) -def test_sequence_dist_all_metrics(metric): +@pytest.mark.parametrize( + "n_jobs", [-1, 1, 2] +) +def test_sequence_dist_all_metrics(metric, n_jobs): # Smoke test, no assertions! # Smoke test, no assertions! unique_seqs = np.array(["AAA", "ARA", "AFFFFFA", "FAFAFA", "FFF"]) seqs2 = np.array(["RRR", "FAFA", "WWWWWWW"]) - dist_mat = ir.ir_dist.sequence_dist(unique_seqs, metric=metric, cutoff=8, n_jobs=2) - assert dist_mat.shape == (5, 5) - - dist_mat = ir.ir_dist.sequence_dist(unique_seqs, seqs2, metric=metric, cutoff=8, n_jobs=2) - assert dist_mat.shape == (5, 3) + cutoff = 8 + + if metric in metrics_with_n_blocks: + for n_blocks in n_blocks_params: + dist_mat = ir.ir_dist.sequence_dist(unique_seqs, metric=metric, cutoff=cutoff, n_jobs=n_jobs, n_blocks=n_blocks) + assert dist_mat.shape == (5, 5) + dist_mat = ir.ir_dist.sequence_dist(unique_seqs, seqs2, metric=metric, cutoff=cutoff, n_jobs=n_jobs, n_blocks=n_blocks) + assert dist_mat.shape == (5, 3) + else: + dist_mat = ir.ir_dist.sequence_dist(unique_seqs, metric=metric, cutoff=cutoff, n_jobs=n_jobs) + assert dist_mat.shape == (5, 5) + dist_mat = ir.ir_dist.sequence_dist(unique_seqs, seqs2, metric=metric, cutoff=cutoff, n_jobs=n_jobs) + assert dist_mat.shape == (5, 3) @pytest.mark.parametrize( From 15c04bd0024df12a474d20aa90e2733f0a313dd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:28:04 +0000 Subject: [PATCH 39/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 6 +++--- src/scirpy/tests/test_ir_dist_metrics.py | 17 ++++++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index f889681be..558b03c54 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -631,9 +631,9 @@ def _hamming_mat( is_symmetric *= histogram start_column *= is_symmetric - if(self.n_jobs > -1): + if self.n_jobs > -1: nb.set_num_threads(self.n_jobs) - + num_threads = nb.get_num_threads() jit_parallel = num_threads > 1 @@ -836,7 +836,7 @@ def _tcrdist_mat( dist_mat_weighted = self.tcr_nb_distance_matrix * dist_weight start_column *= is_symmetric - if(self.n_jobs > -1): + if self.n_jobs > -1: nb.set_num_threads(self.n_jobs) num_threads = nb.get_num_threads() diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 7feb0c065..daa495fdc 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -306,26 +306,29 @@ def test_fast_alignment_dist_with_two_seq_arrays(): metrics_with_n_blocks = ["hamming", "normalized_hamming", "tcrdist"] -n_blocks_params = [1,2] +n_blocks_params = [1, 2] + @pytest.mark.parametrize( "metric", ["alignment", "fastalignment", "identity", "hamming", "normalized_hamming", "levenshtein", "tcrdist"] ) -@pytest.mark.parametrize( - "n_jobs", [-1, 1, 2] -) +@pytest.mark.parametrize("n_jobs", [-1, 1, 2]) def test_sequence_dist_all_metrics(metric, n_jobs): # Smoke test, no assertions! # Smoke test, no assertions! unique_seqs = np.array(["AAA", "ARA", "AFFFFFA", "FAFAFA", "FFF"]) seqs2 = np.array(["RRR", "FAFA", "WWWWWWW"]) cutoff = 8 - + if metric in metrics_with_n_blocks: for n_blocks in n_blocks_params: - dist_mat = ir.ir_dist.sequence_dist(unique_seqs, metric=metric, cutoff=cutoff, n_jobs=n_jobs, n_blocks=n_blocks) + dist_mat = ir.ir_dist.sequence_dist( + unique_seqs, metric=metric, cutoff=cutoff, n_jobs=n_jobs, n_blocks=n_blocks + ) assert dist_mat.shape == (5, 5) - dist_mat = ir.ir_dist.sequence_dist(unique_seqs, seqs2, metric=metric, cutoff=cutoff, n_jobs=n_jobs, n_blocks=n_blocks) + dist_mat = ir.ir_dist.sequence_dist( + unique_seqs, seqs2, metric=metric, cutoff=cutoff, n_jobs=n_jobs, n_blocks=n_blocks + ) assert dist_mat.shape == (5, 3) else: dist_mat = ir.ir_dist.sequence_dist(unique_seqs, metric=metric, cutoff=cutoff, n_jobs=n_jobs) From d45ab64766c79c5cd72a34858a059a25123cafab Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 17:44:18 +0200 Subject: [PATCH 40/94] test_sequence_dist_all_metrics adaptions --- src/scirpy/tests/test_ir_dist_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 7feb0c065..64d03a238 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -305,9 +305,6 @@ def test_fast_alignment_dist_with_two_seq_arrays(): npt.assert_almost_equal(res.toarray(), np.array([[0, 1, 5], [0, 5, 10], [0, 0, 0], [1, 0, 0]])) -metrics_with_n_blocks = ["hamming", "normalized_hamming", "tcrdist"] -n_blocks_params = [1,2] - @pytest.mark.parametrize( "metric", ["alignment", "fastalignment", "identity", "hamming", "normalized_hamming", "levenshtein", "tcrdist"] ) @@ -317,6 +314,9 @@ def test_fast_alignment_dist_with_two_seq_arrays(): def test_sequence_dist_all_metrics(metric, n_jobs): # Smoke test, no assertions! # Smoke test, no assertions! + metrics_with_n_blocks = ["hamming", "normalized_hamming", "tcrdist"] + n_blocks_params = [1,2] + unique_seqs = np.array(["AAA", "ARA", "AFFFFFA", "FAFAFA", "FFF"]) seqs2 = np.array(["RRR", "FAFA", "WWWWWWW"]) cutoff = 8 From 1730b3fffcd0e2a0e9041f0ae9d229cf48823ed7 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 17:52:37 +0200 Subject: [PATCH 41/94] n_jobs default value set to -1 --- src/scirpy/ir_dist/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index f889681be..f49744536 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -551,7 +551,7 @@ class HammingDistanceCalculator(MetricDistanceCalculator): def __init__( self, - n_jobs: int = 1, + n_jobs: int = -1, n_blocks: int = 1, cutoff: int = 2, *, @@ -764,7 +764,7 @@ def __init__( ntrim: int = 3, ctrim: int = 2, fixed_gappos: bool = True, - n_jobs: int = 1, + n_jobs: int = -1, n_blocks: int = 1, ): self.dist_weight = dist_weight From 8f1821073427f65ece329ed256ff21196f488915 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 18:43:47 +0200 Subject: [PATCH 42/94] docstring of ir_dist for n_jobs adapted --- src/scirpy/ir_dist/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 41878e570..2f3073c0a 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -35,7 +35,7 @@ def IrNeighbors(*args, **kwargs): MetricType = Union[ - Literal["alignment", "fastalignment", "identity", "levenshtein", "hamming"], + Literal["alignment", "fastalignment", "identity", "levenshtein", "hamming", "normalized_hamming", "tcrdist"], metrics.DistanceCalculator, ] @@ -167,10 +167,12 @@ def _ir_dist( If true, store the result in `adata.uns`. Otherwise return a dictionary with the results. n_jobs - Number of cores to use for distance calculation. Passed on to - :class:`scirpy.ir_dist.metrics.DistanceCalculator`. :class:`joblib.Parallel` is + Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. + The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` + for parallelization with multithreading instead. + {airr_mod} {airr_key} {chain_idx_key} From 8cfc1c6decd9f6bcb2ddd4bb5a543da20b6f0314 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:47:09 +0000 Subject: [PATCH 43/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/__init__.py | 2 +- src/scirpy/tests/test_ir_dist_metrics.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 2f3073c0a..742d89ee9 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -170,7 +170,7 @@ def _ir_dist( Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. - The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` + The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` for parallelization with multithreading instead. {airr_mod} diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 81c00a605..8bf20f27e 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -313,8 +313,8 @@ def test_sequence_dist_all_metrics(metric, n_jobs): # Smoke test, no assertions! # Smoke test, no assertions! metrics_with_n_blocks = ["hamming", "normalized_hamming", "tcrdist"] - n_blocks_params = [1,2] - + n_blocks_params = [1, 2] + unique_seqs = np.array(["AAA", "ARA", "AFFFFFA", "FAFAFA", "FFF"]) seqs2 = np.array(["RRR", "FAFA", "WWWWWWW"]) cutoff = 8 From efbc37bc18bc88612fa58a81f6f29fb70b625bd7 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 18:59:27 +0200 Subject: [PATCH 44/94] docstring change to test cicd pipeline --- src/scirpy/ir_dist/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 2f3073c0a..59683537d 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -170,8 +170,6 @@ def _ir_dist( Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. - The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` - for parallelization with multithreading instead. {airr_mod} {airr_key} From 9398d7ccffb8747e7c284a5cbe1beb7ba2834e6a Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 19:12:27 +0200 Subject: [PATCH 45/94] docstring for n_jobs of _ir_dist changed --- src/scirpy/ir_dist/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 59683537d..6c8459fc0 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -170,7 +170,9 @@ def _ir_dist( Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. - + The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` + for parallelization with multithreading instead. + {airr_mod} {airr_key} {chain_idx_key} From 66d6f709d30f9f461ac17fc4be4fdff7129dfda7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:12:52 +0000 Subject: [PATCH 46/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 6c8459fc0..742d89ee9 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -170,9 +170,9 @@ def _ir_dist( Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. - The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` + The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` for parallelization with multithreading instead. - + {airr_mod} {airr_key} {chain_idx_key} From afc03baa8f754cc0488eefc1a99b2faf50455067 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 15 Aug 2024 19:24:49 +0200 Subject: [PATCH 47/94] docstring for n_jobs of _ir_dist changed --- src/scirpy/ir_dist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 6c8459fc0..f8a96a2fb 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -170,7 +170,7 @@ def _ir_dist( Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. - The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba.jit(parallel=True)` + The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba` for parallelization with multithreading instead. {airr_mod} From dc8dae411b1851a7e03d6ec97e7d4689601f6895 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 15 Aug 2024 17:26:34 +0000 Subject: [PATCH 48/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 53b9e09d2..77aa04346 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -170,7 +170,7 @@ def _ir_dist( Number of cores to use for distance calculation. :class:`joblib.Parallel` is used internally. Via the :class:`joblib.parallel_config` context manager, you can set another backend (e.g. `dask`) and adjust other configuration options. - The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba` + The metrics `hamming`, `normalized_hamming`, and `tcrdist` utilize `numba` for parallelization with multithreading instead. {airr_mod} From 758feed94ea37387ea35885cc3d5d551413dba3a Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 10:14:00 +0200 Subject: [PATCH 49/94] moved histogram creation to parent class of hamming distance calculator --- src/scirpy/ir_dist/metrics.py | 81 +++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 761263db0..d815f0026 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -426,10 +426,11 @@ class MetricDistanceCalculator(abc.ABC): Overall number of blocks given to the workers (processes) """ - def __init__(self, n_jobs: int = -1, n_blocks: int = 1): + def __init__(self, n_jobs: int = -1, n_blocks: int = 1, histogram: bool = False): super().__init__() self.n_jobs = n_jobs self.n_blocks = n_blocks + self.histogram = histogram @abc.abstractmethod def _metric_mat( @@ -439,7 +440,7 @@ def _metric_mat( seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, - ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray, Union[np.ndarray, None]]: """ Abstract method that should be implemented by the derived class in a way such that it computes the pairwise distances for gene sequences in seqs and seqs2 based on a certain distance metric. The result should be a distance matrix @@ -471,24 +472,45 @@ def _metric_mat( row_element_counts: Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later + row_mins: + Minimum distance per row, ignoring equal sequences and ignoring the cutoff. + Should be None if the computation of row_mins is not implemented. Used to create a nearest neighbor + histogram later. """ pass + def _make_histogram(self, row_mins: np.ndarray): + if self.normalize: + bins = np.arange(0, 101, 2) + else: + max_value = np.max(row_mins) + bin_step = np.ceil(max_value / 100) + bins = np.arange(0, max_value + 1, bin_step) + + plt.hist(row_mins, bins=bins, histtype="bar", edgecolor="black") + plt.axvline(x=self.cutoff, color="r", linestyle="-", label="cutoff") + plt.legend() + plt.xlabel("Distance to nearest neighbor") + plt.ylabel("Count") + plt.title('Histogram of "distance-to-nearest"-distribution') + plt.show() + def _calc_dist_mat_block( self, seqs: Sequence[str], seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, - ) -> csr_matrix: - """Computes a block of the final distance matrix and returns it as CSR matrix. + ) -> tuple[csr_matrix, np.ndarray]: + """Computes a block of the final distance matrix and returns it as CSR matrix. Also computes + the minimum distance per row, for which equal sequences and the cutoff are ignored. If the final result matrix that consists of all blocks together is symmetric, only the part of the block that would contribute to the upper triangular matrix of the final result will be computed. """ if len(seqs) == 0 or len(seqs2) == 0: return csr_matrix((len(seqs), len(seqs2))) - data_rows, indices_rows, row_element_counts = self._metric_mat( + data_rows, indices_rows, row_element_counts, row_mins = self._metric_mat( seqs=seqs, seqs2=seqs2, is_symmetric=is_symmetric, @@ -499,7 +521,7 @@ def _calc_dist_mat_block( indptr[1:] = np.cumsum(row_element_counts) data, indices = np.concatenate(data_rows), np.concatenate(indices_rows) sparse_distance_matrix = csr_matrix((data, indices, indptr), shape=(len(seqs), len(seqs2))) - return sparse_distance_matrix + return sparse_distance_matrix, row_mins def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = None) -> csr_matrix: """Calculates the pairwise distances between two vectors of gene sequences based on the distance metric @@ -519,9 +541,13 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No delayed_jobs = [joblib.delayed(self._calc_dist_mat_block)(*args) for args in arguments] results = joblib.Parallel(return_as="list")(delayed_jobs) - distance_matrix_csr = scipy.sparse.vstack(results) + + block_matrices_csr, block_row_mins = zip(*results) + distance_matrix_csr = scipy.sparse.vstack(block_matrices_csr) + row_mins = np.concatenate(block_row_mins) else: - distance_matrix_csr = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) + distance_matrix_csr, block_row_mins = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) + row_mins = np.array(block_row_mins) if is_symmetric: upper_triangular_distance_matrix = distance_matrix_csr @@ -529,6 +555,12 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No else: full_distance_matrix = distance_matrix_csr + if self.histogram : + if None in row_mins: + raise NotImplementedError("Creating a histogram is not implemented for this metric") + else: + self._make_histogram(row_mins) + return full_distance_matrix @@ -556,27 +588,10 @@ def __init__( cutoff: int = 2, *, normalize: bool = False, - histogram: bool = False, ): - super().__init__(n_jobs=n_jobs, n_blocks=n_blocks) + super().__init__(n_jobs=n_jobs, n_blocks=n_blocks, histogram=False) self.cutoff = cutoff self.normalize = normalize - self.histogram = histogram - - def _make_histogram(self, row_mins): - if self.normalize: - bins = np.arange(0, 101, 2) - else: - max_value = np.max(row_mins) - bin_step = np.ceil(max_value / 100) - bins = np.arange(0, max_value + 1, bin_step) - plt.hist(row_mins, bins=bins, histtype="bar", edgecolor="black") - plt.axvline(x=self.cutoff, color="r", linestyle="-", label="cutoff") - plt.legend() - plt.xlabel("Distance to nearest neighbor") - plt.ylabel("Count") - plt.title('Histogram of "distance-to-nearest"-distribution') - plt.show() def _hamming_mat( self, @@ -618,6 +633,10 @@ def _hamming_mat( row_element_counts: Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later + row_mins: + Minimum distance per row, ignoring equal sequences and ignoring the cutoff. + Should be None if the computation of row_mins is not implemented. Used to create a nearest neighbor + histogram later. """ unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) @@ -710,12 +729,9 @@ def _nb_hamming_mat(): data_rows, indices_rows, row_element_counts, row_mins = _nb_hamming_mat() - if histogram: - self._make_histogram(row_mins) - return data_rows, indices_rows, row_element_counts, row_mins - _metric_mat = lambda self, *args, **kwargs: self._hamming_mat(*args, **kwargs)[:3] + _metric_mat = _hamming_mat class TCRdistDistanceCalculator(MetricDistanceCalculator): @@ -820,6 +836,9 @@ def _tcrdist_mat( row_element_counts: Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later + row_mins: + Returns always None because the computation of the minimum distance per row is not implemented for + the tcrdist calculator yet. """ max_seq_len = max(len(s) for s in (*seqs, *seqs2)) @@ -929,7 +948,7 @@ def _nb_tcrdist_mat(): return data_rows_flat, indices_rows_flat, row_element_counts data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() - return data_rows, indices_rows, row_element_counts + return data_rows, indices_rows, row_element_counts, None _metric_mat = _tcrdist_mat From 04d0db79b4432eaf05c233090ea12b18c889dc7d Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 11:13:40 +0200 Subject: [PATCH 50/94] histogram computation adaptions --- src/scirpy/ir_dist/metrics.py | 63 +++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index d815f0026..baab03562 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -440,7 +440,7 @@ def _metric_mat( seqs2: Sequence[str], is_symmetric: bool = False, start_column: int = 0, - ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray, Union[np.ndarray, None]]: + ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray, np.ndarray]: """ Abstract method that should be implemented by the derived class in a way such that it computes the pairwise distances for gene sequences in seqs and seqs2 based on a certain distance metric. The result should be a distance matrix @@ -473,27 +473,19 @@ def _metric_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later row_mins: - Minimum distance per row, ignoring equal sequences and ignoring the cutoff. - Should be None if the computation of row_mins is not implemented. Used to create a nearest neighbor + Array containing the minimum distance per row, ignoring equal sequences and ignoring the cutoff. + Should contain None if the computation of row_mins is not implemented. Used to create a nearest neighbor histogram later. """ pass + def _make_histogram(self, row_mins: np.ndarray): - if self.normalize: - bins = np.arange(0, 101, 2) - else: - max_value = np.max(row_mins) - bin_step = np.ceil(max_value / 100) - bins = np.arange(0, max_value + 1, bin_step) + """ + Subclass should override this method if the computation of a nearest neighbor histogram is implemented. + """ + raise NotImplementedError("Creating a histogram is not implemented for this metric") - plt.hist(row_mins, bins=bins, histtype="bar", edgecolor="black") - plt.axvline(x=self.cutoff, color="r", linestyle="-", label="cutoff") - plt.legend() - plt.xlabel("Distance to nearest neighbor") - plt.ylabel("Count") - plt.title('Histogram of "distance-to-nearest"-distribution') - plt.show() def _calc_dist_mat_block( self, @@ -508,7 +500,7 @@ def _calc_dist_mat_block( of the block that would contribute to the upper triangular matrix of the final result will be computed. """ if len(seqs) == 0 or len(seqs2) == 0: - return csr_matrix((len(seqs), len(seqs2))) + return csr_matrix((len(seqs), len(seqs2))), np.array([None]) data_rows, indices_rows, row_element_counts, row_mins = self._metric_mat( seqs=seqs, @@ -546,8 +538,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No distance_matrix_csr = scipy.sparse.vstack(block_matrices_csr) row_mins = np.concatenate(block_row_mins) else: - distance_matrix_csr, block_row_mins = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) - row_mins = np.array(block_row_mins) + distance_matrix_csr, row_mins = self._calc_dist_mat_block(seqs, seqs2, is_symmetric) if is_symmetric: upper_triangular_distance_matrix = distance_matrix_csr @@ -556,10 +547,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No full_distance_matrix = distance_matrix_csr if self.histogram : - if None in row_mins: - raise NotImplementedError("Creating a histogram is not implemented for this metric") - else: - self._make_histogram(row_mins) + self._make_histogram(row_mins) return full_distance_matrix @@ -588,11 +576,28 @@ def __init__( cutoff: int = 2, *, normalize: bool = False, + histogram: bool = False, ): - super().__init__(n_jobs=n_jobs, n_blocks=n_blocks, histogram=False) + super().__init__(n_jobs=n_jobs, n_blocks=n_blocks, histogram=histogram) self.cutoff = cutoff self.normalize = normalize + def _make_histogram(self, row_mins: np.ndarray): + if self.normalize: + bins = np.arange(0, 101, 2) + else: + max_value = np.max(row_mins) + bin_step = np.ceil(max_value / 100) + bins = np.arange(0, max_value + 1, bin_step) + + plt.hist(row_mins, bins=bins, histtype="bar", edgecolor="black") + plt.axvline(x=self.cutoff, color="r", linestyle="-", label="cutoff") + plt.legend() + plt.xlabel("Distance to nearest neighbor") + plt.ylabel("Count") + plt.title('Histogram of "distance-to-nearest"-distribution') + plt.show() + def _hamming_mat( self, *, @@ -634,8 +639,8 @@ def _hamming_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later row_mins: - Minimum distance per row, ignoring equal sequences and ignoring the cutoff. - Should be None if the computation of row_mins is not implemented. Used to create a nearest neighbor + Array containing the minimum distance per row, ignoring equal sequences and ignoring the cutoff. + Should contain None if the computation of row_mins is not implemented. Used to create a nearest neighbor histogram later. """ unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) @@ -789,7 +794,7 @@ def __init__( self.ctrim = ctrim self.fixed_gappos = fixed_gappos self.cutoff = cutoff - super().__init__(n_jobs=n_jobs, n_blocks=n_blocks) + super().__init__(n_jobs=n_jobs, n_blocks=n_blocks, histogram=False) def _tcrdist_mat( self, @@ -837,7 +842,7 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later row_mins: - Returns always None because the computation of the minimum distance per row is not implemented for + Always returns numpy array containing None because the computation of the minimum distance per row is not implemented for the tcrdist calculator yet. """ max_seq_len = max(len(s) for s in (*seqs, *seqs2)) @@ -948,7 +953,7 @@ def _nb_tcrdist_mat(): return data_rows_flat, indices_rows_flat, row_element_counts data_rows, indices_rows, row_element_counts = _nb_tcrdist_mat() - return data_rows, indices_rows, row_element_counts, None + return data_rows, indices_rows, row_element_counts, np.array([None]) _metric_mat = _tcrdist_mat From b7ed4ca10d0062c9a229b08d29158a45d191de83 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 11:26:32 +0200 Subject: [PATCH 51/94] test case test_tcrdist_histogram_not_implemented added --- src/scirpy/ir_dist/metrics.py | 4 +++- src/scirpy/tests/test_ir_dist_metrics.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index baab03562..11cd79e7b 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -787,6 +787,7 @@ def __init__( fixed_gappos: bool = True, n_jobs: int = -1, n_blocks: int = 1, + histogram: bool = False, ): self.dist_weight = dist_weight self.gap_penalty = gap_penalty @@ -794,7 +795,8 @@ def __init__( self.ctrim = ctrim self.fixed_gappos = fixed_gappos self.cutoff = cutoff - super().__init__(n_jobs=n_jobs, n_blocks=n_blocks, histogram=False) + self.histogram = histogram + super().__init__(n_jobs=n_jobs, n_blocks=n_blocks, histogram=histogram) def _tcrdist_mat( self, diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 8bf20f27e..66397d4f5 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -728,3 +728,10 @@ def test_hamming_histogram_reference(): row_mins_ref = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_histogram_result.npy") _, _, _, row_mins = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) assert np.array_equal(row_mins_ref, row_mins) + +def test_tcrdist_histogram_not_implemented(): + #Change once histogram is implemented for tcrdist + with pytest.raises(NotImplementedError, match=None): + tcrdist_calculator = TCRdistDistanceCalculator(histogram=True) + seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) + _ = tcrdist_calculator.calc_dist_mat(seqs, seqs) From d40e1935b1348ef940a9363a69653615ae82264e Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 11:39:38 +0200 Subject: [PATCH 52/94] documentation for histogram adapted --- src/scirpy/ir_dist/metrics.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 11cd79e7b..f00ab2037 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -411,7 +411,7 @@ def _seqs2mat( class MetricDistanceCalculator(abc.ABC): """ - Abstract base class for distance calculator classes that compute parwise distances between + Abstract base class for distance calculator classes that computes parwise distances between gene sequences in parallel based on a certain distance metric. The result is a (scipy) compressed sparse row distance matrix. Derived classes just need to implement the method _metric_mat (see method comments for more details). @@ -446,6 +446,8 @@ def _metric_mat( for gene sequences in seqs and seqs2 based on a certain distance metric. The result should be a distance matrix that is returned in the form of the data, indices and intptr arrays of a (scipy) compressed sparse row matrix. + In case that a nearest neighbour histogram should be created later, the minimum value per row is returned. + If this function is used to compute a block of a bigger result matrix, is_symmetric and start_column can be used to only compute the part of the block that would be part of the upper triangular matrix of the result matrix. @@ -474,8 +476,8 @@ def _metric_mat( needed to create the final scipy CSR result matrix later row_mins: Array containing the minimum distance per row, ignoring equal sequences and ignoring the cutoff. - Should contain None if the computation of row_mins is not implemented. Used to create a nearest neighbor - histogram later. + Contains None if the computation of row_mins is not implemented. Used to create a nearest neighbor + histogram later. Is empty if the histogram should not be created. """ pass @@ -640,8 +642,7 @@ def _hamming_mat( needed to create the final scipy CSR result matrix later row_mins: Array containing the minimum distance per row, ignoring equal sequences and ignoring the cutoff. - Should contain None if the computation of row_mins is not implemented. Used to create a nearest neighbor - histogram later. + Used to create a nearest neighbor histogram later. Is empty if the histogram should not be created. """ unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) @@ -844,8 +845,8 @@ def _tcrdist_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later row_mins: - Always returns numpy array containing None because the computation of the minimum distance per row is not implemented for - the tcrdist calculator yet. + Always returns a numpy array containing None because the computation of the minimum distance per row is + not implemented for the tcrdist calculator yet. """ max_seq_len = max(len(s) for s in (*seqs, *seqs2)) From 54c0cc226653963842e4e0a93d0765a2d97f0937 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:44:44 +0000 Subject: [PATCH 53/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 6 ++---- src/scirpy/tests/test_ir_dist_metrics.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index f00ab2037..3fa92d9b7 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -481,14 +481,12 @@ def _metric_mat( """ pass - def _make_histogram(self, row_mins: np.ndarray): """ Subclass should override this method if the computation of a nearest neighbor histogram is implemented. """ raise NotImplementedError("Creating a histogram is not implemented for this metric") - def _calc_dist_mat_block( self, seqs: Sequence[str], @@ -535,7 +533,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No delayed_jobs = [joblib.delayed(self._calc_dist_mat_block)(*args) for args in arguments] results = joblib.Parallel(return_as="list")(delayed_jobs) - + block_matrices_csr, block_row_mins = zip(*results) distance_matrix_csr = scipy.sparse.vstack(block_matrices_csr) row_mins = np.concatenate(block_row_mins) @@ -548,7 +546,7 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No else: full_distance_matrix = distance_matrix_csr - if self.histogram : + if self.histogram: self._make_histogram(row_mins) return full_distance_matrix diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 66397d4f5..e8bd38ca8 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -729,8 +729,9 @@ def test_hamming_histogram_reference(): _, _, _, row_mins = hamming_calculator._hamming_mat(seqs=seqs, seqs2=seqs) assert np.array_equal(row_mins_ref, row_mins) + def test_tcrdist_histogram_not_implemented(): - #Change once histogram is implemented for tcrdist + # Change once histogram is implemented for tcrdist with pytest.raises(NotImplementedError, match=None): tcrdist_calculator = TCRdistDistanceCalculator(histogram=True) seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) From b16c7056d9ee68f97c3fa1662156dc6fcac4a4a8 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 11:47:18 +0200 Subject: [PATCH 54/94] reformatted doc string --- src/scirpy/ir_dist/metrics.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index f00ab2037..3e7e59f41 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -483,9 +483,7 @@ def _metric_mat( def _make_histogram(self, row_mins: np.ndarray): - """ - Subclass should override this method if the computation of a nearest neighbor histogram is implemented. - """ + """Subclass should override this method if the computation of a nearest neighbor histogram is implemented.""" raise NotImplementedError("Creating a histogram is not implemented for this metric") From 2419bfbde7672b6d227e490adaee55b1864ee01a Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 15:19:59 +0200 Subject: [PATCH 55/94] handling of symmetric matrices with respect to histogram variable changed --- src/scirpy/ir_dist/metrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 7f761395c..65ae90ec9 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -649,7 +649,10 @@ def _hamming_mat( cutoff = self.cutoff normalize = self.normalize histogram = self.histogram - is_symmetric *= histogram + + if(histogram): + is_symmetric = False + start_column *= is_symmetric if self.n_jobs > -1: From e516a863246aa32ac86544fe3cc86d61f577f361 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:20:32 +0000 Subject: [PATCH 56/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 65ae90ec9..087cb419d 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -649,10 +649,10 @@ def _hamming_mat( cutoff = self.cutoff normalize = self.normalize histogram = self.histogram - - if(histogram): + + if histogram: is_symmetric = False - + start_column *= is_symmetric if self.n_jobs > -1: From 20c14ebcbbaa4542d1e791c648871418991fce43 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Fri, 16 Aug 2024 21:33:18 +0200 Subject: [PATCH 57/94] retrieval of usable cpus for numba adapted --- src/scirpy/ir_dist/metrics.py | 10 ++++------ src/scirpy/util/__init__.py | 15 ++++++++++----- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 65ae90ec9..023543118 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -14,7 +14,7 @@ from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix -from scirpy.util import _doc_params, _parallelize_with_joblib, deprecated +from scirpy.util import _doc_params, _parallelize_with_joblib, deprecated, _get_usable_cpus _doc_params_parallel_distance_calculator = """\ n_jobs @@ -655,9 +655,8 @@ def _hamming_mat( start_column *= is_symmetric - if self.n_jobs > -1: - nb.set_num_threads(self.n_jobs) - + nb.set_num_threads(_get_usable_cpus(n_jobs=self.n_jobs, use_numba=True)) + num_threads = nb.get_num_threads() jit_parallel = num_threads > 1 @@ -862,8 +861,7 @@ def _tcrdist_mat( dist_mat_weighted = self.tcr_nb_distance_matrix * dist_weight start_column *= is_symmetric - if self.n_jobs > -1: - nb.set_num_threads(self.n_jobs) + nb.set_num_threads(_get_usable_cpus(n_jobs=self.n_jobs, use_numba=True)) num_threads = nb.get_num_threads() diff --git a/src/scirpy/util/__init__.py b/src/scirpy/util/__init__.py index 0a4059351..d8e4d526f 100644 --- a/src/scirpy/util/__init__.py +++ b/src/scirpy/util/__init__.py @@ -585,19 +585,24 @@ def _parallelize_with_joblib(delayed_objects, *, total=None, **kwargs): return Parallel(return_as="list", **kwargs)(delayed_objects) -def _get_usable_cpus(n_jobs: int = 0): +def _get_usable_cpus(n_jobs: int = 0, use_numba: bool = False): """Get the number of CPUs available to the process - If `n_jobs` is specified and > 0 that value will be returned unaltered. Otherwise will try to determine the number of CPUs available to the process which is not necessarily the number of CPUs available on the system. - On MacOS, `os.sched_getaffinity` is not implemented, therefore we just return the cpu count there. """ if n_jobs > 0: return n_jobs try: - return len(os.sched_getaffinity(0)) + usable_cpus = len(os.sched_getaffinity(0)) except AttributeError: - return os.cpu_count() + usable_cpus = os.cpu_count() + + if use_numba: + # When using numba, the `NUMBA_NUM_THREADS` variable should additionally be respected as upper limit + from numba import config + usable_cpus = min(usable_cpus, config.NUMBA_NUM_THREADS) + + return usable_cpus From f5662110659f921d3553987d0231af88775e0822 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 19:35:07 +0000 Subject: [PATCH 58/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 4 ++-- src/scirpy/util/__init__.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 0c6e3a59a..1ffe9e85f 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -14,7 +14,7 @@ from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix -from scirpy.util import _doc_params, _parallelize_with_joblib, deprecated, _get_usable_cpus +from scirpy.util import _doc_params, _get_usable_cpus, _parallelize_with_joblib, deprecated _doc_params_parallel_distance_calculator = """\ n_jobs @@ -656,7 +656,7 @@ def _hamming_mat( start_column *= is_symmetric nb.set_num_threads(_get_usable_cpus(n_jobs=self.n_jobs, use_numba=True)) - + num_threads = nb.get_num_threads() jit_parallel = num_threads > 1 diff --git a/src/scirpy/util/__init__.py b/src/scirpy/util/__init__.py index d8e4d526f..c4057ebf7 100644 --- a/src/scirpy/util/__init__.py +++ b/src/scirpy/util/__init__.py @@ -603,6 +603,7 @@ def _get_usable_cpus(n_jobs: int = 0, use_numba: bool = False): if use_numba: # When using numba, the `NUMBA_NUM_THREADS` variable should additionally be respected as upper limit from numba import config + usable_cpus = min(usable_cpus, config.NUMBA_NUM_THREADS) return usable_cpus From f68dd70f76a813ef26c7a8ebbe9dec90cd667090 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 19 Aug 2024 08:38:36 +0200 Subject: [PATCH 59/94] more documentation for histogram and (hamming) normalize added --- src/scirpy/ir_dist/metrics.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 0c6e3a59a..040ff6f6a 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -424,6 +424,8 @@ class MetricDistanceCalculator(abc.ABC): Number of threads per process to use for the pairwise distance calculation n_blocks: Overall number of blocks given to the workers (processes) + histogram: + Determines whether a nearest neighbor histogram should be created """ def __init__(self, n_jobs: int = -1, n_blocks: int = 1, histogram: bool = False): @@ -482,7 +484,7 @@ def _metric_mat( pass def _make_histogram(self, row_mins: np.ndarray): - """Subclass should override this method if the computation of a nearest neighbor histogram is implemented.""" + """Subclass should override this method if the creation of a nearest neighbor histogram is implemented.""" raise NotImplementedError("Creating a histogram is not implemented for this metric") def _calc_dist_mat_block( @@ -515,7 +517,8 @@ def _calc_dist_mat_block( def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = None) -> csr_matrix: """Calculates the pairwise distances between two vectors of gene sequences based on the distance metric - of the derived class and returns a CSR distance matrix + of the derived class and returns a CSR distance matrix. Also creates a histogram based on the minimum value + per row of the distance matrix if histogram is set to True. """ if seqs2 is None: seqs2 = seqs @@ -553,6 +556,11 @@ def calc_dist_mat(self, seqs: Sequence[str], seqs2: Optional[Sequence[str]] = No class HammingDistanceCalculator(MetricDistanceCalculator): """Computes pairwise distances between gene sequences based on the "hamming" distance metric. + Set `normalize` to True to use the normalized hamming distance metric instead of the standard hamming distance + metric. Then the distance will be calculated as percentage of different positions relative to the sequence length + (e.g. AAGG and AAAA -> 50 (%) normalized hamming distance). The cutoff is then also given as normalized hamming + distance in percent. + The code of this class is based on `pwseqdist `_. Reused under MIT license, Copyright (c) 2020 Andrew Fiore-Gartland. @@ -565,6 +573,11 @@ class HammingDistanceCalculator(MetricDistanceCalculator): Number of numba parallel threads to use for the pairwise distance calculation n_blocks: Number of joblib delayed objects (blocks to compute) given to joblib.Parallel + normalize: + Determines whether the normalized hamming distance metric should be used instead of the standard + hamming distance + histogram: + Determines whether a nearest neighbor histogram should be created """ def __init__( @@ -766,6 +779,8 @@ class TCRdistDistanceCalculator(MetricDistanceCalculator): Number of numba parallel threads to use for the pairwise distance calculation n_blocks: Number of joblib delayed objects (blocks to compute) given to joblib.Parallel + histogram: + Determines whether a nearest neighbor histogram should be created """ parasail_aa_alphabet = "ARNDCQEGHILKMFPSTWYVBZX" From d9dd20e25082122bfb63e48bf136b2db64ac3eca Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:40:16 +0000 Subject: [PATCH 60/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 0718f4858..d68601260 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -425,7 +425,7 @@ class MetricDistanceCalculator(abc.ABC): n_blocks: Overall number of blocks given to the workers (processes) histogram: - Determines whether a nearest neighbor histogram should be created + Determines whether a nearest neighbor histogram should be created """ def __init__(self, n_jobs: int = -1, n_blocks: int = 1, histogram: bool = False): @@ -577,7 +577,7 @@ class HammingDistanceCalculator(MetricDistanceCalculator): Determines whether the normalized hamming distance metric should be used instead of the standard hamming distance histogram: - Determines whether a nearest neighbor histogram should be created + Determines whether a nearest neighbor histogram should be created """ def __init__( @@ -780,7 +780,7 @@ class TCRdistDistanceCalculator(MetricDistanceCalculator): n_blocks: Number of joblib delayed objects (blocks to compute) given to joblib.Parallel histogram: - Determines whether a nearest neighbor histogram should be created + Determines whether a nearest neighbor histogram should be created """ parasail_aa_alphabet = "ARNDCQEGHILKMFPSTWYVBZX" From 636b8e0484e3643ae4e3c6bc0ce9f3bb7b080894 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 19 Aug 2024 20:41:00 +0200 Subject: [PATCH 61/94] added GPUHammingDistanceCalculator --- src/scirpy/ir_dist/__init__.py | 2 + src/scirpy/ir_dist/metrics.py | 167 +++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) diff --git a/src/scirpy/ir_dist/__init__.py b/src/scirpy/ir_dist/__init__.py index 77aa04346..cd0f73d19 100644 --- a/src/scirpy/ir_dist/__init__.py +++ b/src/scirpy/ir_dist/__init__.py @@ -105,6 +105,8 @@ def _get_distance_calculator(metric: MetricType, cutoff: Union[int, None], *, n_ dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, **kwargs) elif metric == "normalized_hamming": dist_calc = metrics.HammingDistanceCalculator(n_jobs=n_jobs, normalize=True, **kwargs) + elif metric == "gpu_hamming": + dist_calc = metrics.GPUHammingDistanceCalculator(**kwargs) elif metric == "tcrdist": dist_calc = metrics.TCRdistDistanceCalculator(n_jobs=n_jobs, **kwargs) else: diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index d68601260..c3e979848 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -7,6 +7,7 @@ import joblib import matplotlib.pyplot as plt import numba as nb +from numba import cuda import numpy as np import scipy.sparse import scipy.spatial @@ -751,6 +752,172 @@ def _nb_hamming_mat(): _metric_mat = _hamming_mat +class GPUHammingDistanceCalculator(MetricDistanceCalculator): + """Computes pairwise distances between gene sequences based on the "hamming" distance metric with GPU support. + + The code of this class is based on `pwseqdist `_. + Reused under MIT license, Copyright (c) 2020 Andrew Fiore-Gartland. + + Parameters + ---------- + cutoff: + Will eleminate distances > cutoff to make efficient + use of sparse matrices. + n_jobs: + Number of numba parallel threads to use for the pairwise distance calculation + n_blocks: + Number of joblib delayed objects (blocks to compute) given to joblib.Parallel + """ + + def __init__( + self, + *, + cutoff: int = 2, + ): + super().__init__(n_jobs=1, n_blocks=1) + self.cutoff = cutoff + + def _gpu_hamming_mat( + self, + *, + seqs: Sequence[str], + seqs2: Sequence[str], + is_symmetric: bool = False, + start_column: int = 0, + ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: + """Computes the pairwise hamming distances for sequences in seqs_mat1 and seqs_mat2 with GPU support. + + Parameters + ---------- + seqs_mat1/2: + Matrix containing sequences created by seqs2mat with padding to accomodate + sequences of different lengths (-1 padding) + seqs_L1/2: + A vector containing the length of each sequence in the respective seqs_mat matrix, + without the padding in seqs_mat + is_symmetric: + Determines whether the final result matrix is symmetric, assuming that this function is + only used to compute a block of a bigger result matrix + start_column: + Determines at which column the calculation should be started. This is only used if this function is + used to compute a block of a bigger result matrix that is symmetric + + Returns + ------- + data_rows: + List with arrays containing the non-zero data values of the result matrix per row, + needed to create the final scipy CSR result matrix later + indices_rows: + List with arrays containing the non-zero entry column indeces of the result matrix per row, + needed to create the final scipy CSR result matrix later + row_element_counts: + Array with integers that indicate the amount of non-zero values of the result matrix per row, + needed to create the final scipy CSR result matrix later + """ + + unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) + max_seq_len = max(len(s) for s in (*seqs, *seqs2)) + + seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + + @cuda.jit + def hamming_kernel(seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset): + row = cuda.grid(1) + if row < seqs_mat1.shape[0]: + row_end_index = 0 + seq1_len = seqs_L1[row] + for col in range(seqs_mat2.shape[0]): + if (not is_symmetric) or ((col + block_offset) >= row): + seq2_len = seqs_L2[col] + distance = 1 + if(seq1_len == seq2_len): + for i in range(0, seq1_len): + distance += seqs_mat1[row, i] != seqs_mat2[col, i] + if(distance <= cutoff + 1): + data[row, row_end_index] = distance + indices[row, row_end_index] = col + row_end_index += 1 + row_element_counts[row] = row_end_index + + @cuda.jit + def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): + row, col = cuda.grid(2) + if row < data_matrix.shape[0] and col < data_matrix.shape[1]: + row_start = indptr[row] + row_end = indptr[row + 1] + row_end_index = row_end - row_start + data_index = row_start + col + if (data_index < data.shape[0]) and (col < row_end_index): + data[data_index] = data_matrix[row, col] + indices[data_index] = indices_matrix[row, col] + + def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_offset): + + d_seqs_mat1 = cuda.to_device(seqs_mat1_block) + d_seqs_mat2 = cuda.to_device(seqs_mat2) + d_seqs_L1 = cuda.to_device(seqs_L1_block) + d_seqs_L2 = cuda.to_device(seqs_L2) + + data_matrix = np.zeros((seqs_mat1_block.shape[0],seqs_mat2.shape[0]), dtype=np.uint8) + d_data_matrix = cuda.device_array_like(data_matrix) + + indices_matrix = np.zeros((seqs_mat1_block.shape[0],seqs_mat2.shape[0]), dtype=np.uint32) + d_indices_matrix = cuda.device_array_like(indices_matrix) + + row_element_counts = np.zeros(seqs_mat1_block.shape[0], dtype=np.uint32) + d_row_element_counts = cuda.device_array_like(row_element_counts) + + threads_per_block = 256 + blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block + hamming_kernel[blocks_per_grid, threads_per_block](d_seqs_mat1, d_seqs_mat2, d_seqs_L1, d_seqs_L2, self.cutoff, d_data_matrix, d_indices_matrix, d_row_element_counts, block_offset) + + row_element_counts = d_row_element_counts.copy_to_host() + + indptr = np.zeros(seqs_mat1_block.shape[0]+1, dtype=np.uint32) + indptr[1:] = np.cumsum(row_element_counts) + d_indptr = cuda.to_device(indptr) + + n_elements = indptr[-1] + + data = np.zeros(n_elements, dtype=np.uint8) + d_data = cuda.device_array_like(data) + + indices = np.zeros(n_elements, dtype=np.uint32) + d_indices = cuda.device_array_like(indices) + + threads_per_block = (1, 256) + blocks_per_grid_x = (d_data_matrix.shape[0] + threads_per_block[0] - 1) // threads_per_block[0] + blocks_per_grid_y = (d_data_matrix.shape[1] + threads_per_block[1] - 1) // threads_per_block[1] + blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) + create_csr_kernel[blocks_per_grid, threads_per_block](d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr) + + data = d_data.copy_to_host() + indices = d_indices.copy_to_host() + + return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0],seqs_mat2.shape[0])) + + block_width = 4096 + n_blocks = seqs_mat2.shape[0] // block_width + 1 + + seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) + seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) + result_blocks = [None] * n_blocks + + block_offset = start_column + for i in range(0, n_blocks): + result_blocks[i] = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], block_offset) + block_offset += seqs_mat2_blocks[i].shape[0] + + result_sparse = scipy.sparse.hstack(result_blocks) + + row_element_counts_gpu = np.diff(result_sparse.indptr) + + return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) + + _metric_mat = _gpu_hamming_mat + + class TCRdistDistanceCalculator(MetricDistanceCalculator): """Computes pairwise distances between TCR CDR3 sequences based on the "tcrdist" distance metric. From 3e4f0d3379eed4637d44e600bc4c1f5dd36f8daf Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 19 Aug 2024 20:43:04 +0200 Subject: [PATCH 62/94] added test case for gpu hamming distance --- src/scirpy/tests/test_ir_dist_metrics.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index e8bd38ca8..3bc9d6825 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -11,6 +11,7 @@ DistanceCalculator, FastAlignmentDistanceCalculator, HammingDistanceCalculator, + GPUHammingDistanceCalculator, IdentityDistanceCalculator, LevenshteinDistanceCalculator, ParallelDistanceCalculator, @@ -736,3 +737,18 @@ def test_tcrdist_histogram_not_implemented(): tcrdist_calculator = TCRdistDistanceCalculator(histogram=True) seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) _ = tcrdist_calculator.calc_dist_mat(seqs, seqs) + + +def test_gpu_hamming_reference(): + # test hamming distance against reference implementation + from . import TESTDATA + + seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") + reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_csr_result.npz") + + gpu_hamming_calculator = GPUHammingDistanceCalculator(cutoff=2) + res = gpu_hamming_calculator.calc_dist_mat(seqs, seqs) + + assert np.array_equal(res.data, reference_result.data) + assert np.array_equal(res.indices, reference_result.indices) + assert np.array_equal(res.indptr, reference_result.indptr) \ No newline at end of file From 30f6947808caccbf16b2f399425c698157f7260b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 18:55:57 +0000 Subject: [PATCH 63/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 48 +++++++++++++++--------- src/scirpy/tests/test_ir_dist_metrics.py | 6 +-- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index c3e979848..bfe1dd612 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -7,11 +7,11 @@ import joblib import matplotlib.pyplot as plt import numba as nb -from numba import cuda import numpy as np import scipy.sparse import scipy.spatial from Levenshtein import distance as levenshtein_dist +from numba import cuda from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix @@ -814,7 +814,6 @@ def _gpu_hamming_mat( Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later """ - unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) @@ -822,7 +821,9 @@ def _gpu_hamming_mat( seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) @cuda.jit - def hamming_kernel(seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset): + def hamming_kernel( + seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset + ): row = cuda.grid(1) if row < seqs_mat1.shape[0]: row_end_index = 0 @@ -831,10 +832,10 @@ def hamming_kernel(seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices if (not is_symmetric) or ((col + block_offset) >= row): seq2_len = seqs_L2[col] distance = 1 - if(seq1_len == seq2_len): + if seq1_len == seq2_len: for i in range(0, seq1_len): distance += seqs_mat1[row, i] != seqs_mat2[col, i] - if(distance <= cutoff + 1): + if distance <= cutoff + 1: data[row, row_end_index] = distance indices[row, row_end_index] = col row_end_index += 1 @@ -850,36 +851,45 @@ def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): data_index = row_start + col if (data_index < data.shape[0]) and (col < row_end_index): data[data_index] = data_matrix[row, col] - indices[data_index] = indices_matrix[row, col] + indices[data_index] = indices_matrix[row, col] def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_offset): - d_seqs_mat1 = cuda.to_device(seqs_mat1_block) d_seqs_mat2 = cuda.to_device(seqs_mat2) d_seqs_L1 = cuda.to_device(seqs_L1_block) d_seqs_L2 = cuda.to_device(seqs_L2) - data_matrix = np.zeros((seqs_mat1_block.shape[0],seqs_mat2.shape[0]), dtype=np.uint8) + data_matrix = np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint8) d_data_matrix = cuda.device_array_like(data_matrix) - indices_matrix = np.zeros((seqs_mat1_block.shape[0],seqs_mat2.shape[0]), dtype=np.uint32) + indices_matrix = np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint32) d_indices_matrix = cuda.device_array_like(indices_matrix) row_element_counts = np.zeros(seqs_mat1_block.shape[0], dtype=np.uint32) d_row_element_counts = cuda.device_array_like(row_element_counts) - + threads_per_block = 256 blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block - hamming_kernel[blocks_per_grid, threads_per_block](d_seqs_mat1, d_seqs_mat2, d_seqs_L1, d_seqs_L2, self.cutoff, d_data_matrix, d_indices_matrix, d_row_element_counts, block_offset) + hamming_kernel[blocks_per_grid, threads_per_block]( + d_seqs_mat1, + d_seqs_mat2, + d_seqs_L1, + d_seqs_L2, + self.cutoff, + d_data_matrix, + d_indices_matrix, + d_row_element_counts, + block_offset, + ) row_element_counts = d_row_element_counts.copy_to_host() - indptr = np.zeros(seqs_mat1_block.shape[0]+1, dtype=np.uint32) + indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.uint32) indptr[1:] = np.cumsum(row_element_counts) d_indptr = cuda.to_device(indptr) n_elements = indptr[-1] - + data = np.zeros(n_elements, dtype=np.uint8) d_data = cuda.device_array_like(data) @@ -890,13 +900,15 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off blocks_per_grid_x = (d_data_matrix.shape[0] + threads_per_block[0] - 1) // threads_per_block[0] blocks_per_grid_y = (d_data_matrix.shape[1] + threads_per_block[1] - 1) // threads_per_block[1] blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) - create_csr_kernel[blocks_per_grid, threads_per_block](d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr) + create_csr_kernel[blocks_per_grid, threads_per_block]( + d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr + ) data = d_data.copy_to_host() indices = d_indices.copy_to_host() - return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0],seqs_mat2.shape[0])) - + return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])) + block_width = 4096 n_blocks = seqs_mat2.shape[0] // block_width + 1 @@ -908,13 +920,13 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off for i in range(0, n_blocks): result_blocks[i] = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], block_offset) block_offset += seqs_mat2_blocks[i].shape[0] - + result_sparse = scipy.sparse.hstack(result_blocks) row_element_counts_gpu = np.diff(result_sparse.indptr) return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) - + _metric_mat = _gpu_hamming_mat diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 3bc9d6825..5fa95ef78 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -10,8 +10,8 @@ AlignmentDistanceCalculator, DistanceCalculator, FastAlignmentDistanceCalculator, - HammingDistanceCalculator, GPUHammingDistanceCalculator, + HammingDistanceCalculator, IdentityDistanceCalculator, LevenshteinDistanceCalculator, ParallelDistanceCalculator, @@ -742,7 +742,7 @@ def test_tcrdist_histogram_not_implemented(): def test_gpu_hamming_reference(): # test hamming distance against reference implementation from . import TESTDATA - + seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_csr_result.npz") @@ -751,4 +751,4 @@ def test_gpu_hamming_reference(): assert np.array_equal(res.data, reference_result.data) assert np.array_equal(res.indices, reference_result.indices) - assert np.array_equal(res.indptr, reference_result.indptr) \ No newline at end of file + assert np.array_equal(res.indptr, reference_result.indptr) From 2f3bfb3fcae344f38e96e1f9ceb758985df8cd8c Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 29 Aug 2024 11:48:47 +0200 Subject: [PATCH 64/94] documentation for GPUHammingDistanceCalculator adapted --- src/scirpy/ir_dist/metrics.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 30cf090ff..7d9bf81cb 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -765,10 +765,6 @@ class GPUHammingDistanceCalculator(_MetricDistanceCalculator): cutoff: Will eleminate distances > cutoff to make efficient use of sparse matrices. - n_jobs: - Number of numba parallel threads to use for the pairwise distance calculation - n_blocks: - Number of joblib delayed objects (blocks to compute) given to joblib.Parallel """ def __init__( @@ -787,16 +783,12 @@ def _gpu_hamming_mat( is_symmetric: bool = False, start_column: int = 0, ) -> tuple[list[np.ndarray], list[np.ndarray], np.ndarray]: - """Computes the pairwise hamming distances for sequences in seqs_mat1 and seqs_mat2 with GPU support. + """Computes the pairwise hamming distances for sequences in seqs and seqs2 with GPU support. Parameters ---------- - seqs_mat1/2: - Matrix containing sequences created by seqs2mat with padding to accomodate - sequences of different lengths (-1 padding) - seqs_L1/2: - A vector containing the length of each sequence in the respective seqs_mat matrix, - without the padding in seqs_mat + seqs/2: + A python sequence of strings representing gene sequences is_symmetric: Determines whether the final result matrix is symmetric, assuming that this function is only used to compute a block of a bigger result matrix @@ -807,14 +799,17 @@ def _gpu_hamming_mat( Returns ------- data_rows: - List with arrays containing the non-zero data values of the result matrix per row, + List with array containing the non-zero data values of the result matrix, needed to create the final scipy CSR result matrix later indices_rows: - List with arrays containing the non-zero entry column indeces of the result matrix per row, + List with array containing the non-zero entry column indeces of the result matrix, needed to create the final scipy CSR result matrix later row_element_counts: Array with integers that indicate the amount of non-zero values of the result matrix per row, needed to create the final scipy CSR result matrix later + row_mins: + Always returns a numpy array containing None because the computation of the minimum distance per row is + not implemented for the GPU hamming calculator yet. """ unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) From 3b49d167682855a841b979a77a44938ccb1bfdc2 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 29 Aug 2024 11:51:20 +0200 Subject: [PATCH 65/94] adapted documentation of _tcrdist_mat --- src/scirpy/ir_dist/metrics.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 7d9bf81cb..e41467570 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -1012,9 +1012,6 @@ def _tcrdist_mat( ---------- seqs/2: A python sequence of strings representing gene sequences - seqs_L1/2: - A vector containing the length of each sequence in the respective seqs_mat matrix, - without the padding in seqs_mat is_symmetric: Determines whether the final result matrix is symmetric, assuming that this function is only used to compute a block of a bigger result matrix From 03afdd3c440b942daeeb66544f750f711ca13863 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sun, 29 Sep 2024 12:10:22 +0200 Subject: [PATCH 66/94] cuda numba experiments --- src/scirpy/ir_dist/metrics.py | 78 ++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 29794d126..a6728dc8c 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -16,6 +16,9 @@ from scirpy.util import _doc_params, _get_usable_cpus, _parallelize_with_joblib, deprecated +import time +import math + _doc_params_parallel_distance_calculator = """\ n_jobs Number of jobs to use for the pairwise distance calculation, passed to @@ -816,6 +819,10 @@ def _gpu_hamming_mat( seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + # import matplotlib.pyplot as plt + # plt.hist(seqs_L1, bins=np.max(seqs_L1), edgecolor='black') + # plt.show() + @cuda.jit def hamming_kernel( seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset @@ -837,6 +844,26 @@ def hamming_kernel( row_end_index += 1 row_element_counts[row] = row_end_index + # @cuda.jit + # def hamming_kernel( + # seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset + # ): + # row = cuda.grid(1) + # if row < seqs_mat1.shape[0]: + # # row_end_index = 0 + # seq1_len = seqs_L1[row] + # for col in range(seqs_mat2.shape[0]): + # seq2_len = seqs_L2[col] + # if seq1_len == seq2_len: # (col%1000 == 0): + # # data[row, col] = 1 + # # indices[row, col] = col + # data[row, col] = 1 + # data[row, col] = 0 + + + # # row_end_index += 1 + # # row_element_counts[row] = row_end_index + @cuda.jit def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): row, col = cuda.grid(2) @@ -855,10 +882,12 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off d_seqs_L1 = cuda.to_device(seqs_L1_block) d_seqs_L2 = cuda.to_device(seqs_L2) - data_matrix = np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint8) + result_len = 50 + + data_matrix = np.zeros((seqs_mat1_block.shape[0], result_len), dtype=np.uint8) # np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint8) d_data_matrix = cuda.device_array_like(data_matrix) - indices_matrix = np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint32) + indices_matrix = np.zeros((seqs_mat1_block.shape[0], result_len), dtype=np.uint32) # np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint32) d_indices_matrix = cuda.device_array_like(indices_matrix) row_element_counts = np.zeros(seqs_mat1_block.shape[0], dtype=np.uint32) @@ -866,6 +895,14 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off threads_per_block = 256 blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block + + # threads_per_block = (1, 256) + # blocks_per_grid_x = math.ceil(data_matrix.shape[0] / threads_per_block[0]) + # blocks_per_grid_y = math.ceil(data_matrix.shape[1] / threads_per_block[1]) + # blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) + + cuda.synchronize() + start_kernel = time.time() hamming_kernel[blocks_per_grid, threads_per_block]( d_seqs_mat1, d_seqs_mat2, @@ -877,6 +914,26 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off d_row_element_counts, block_offset, ) + cuda.synchronize() + end_kernel = time.time() + time_taken = (end_kernel-start_kernel) + print("first time hamming kernel time taken: ", time_taken) + + # start_kernel = time.time() + # hamming_kernel[blocks_per_grid, threads_per_block]( + # d_seqs_mat1, + # d_seqs_mat2, + # d_seqs_L1, + # d_seqs_L2, + # self.cutoff, + # d_data_matrix, + # d_indices_matrix, + # d_row_element_counts, + # block_offset, + # ) + # cuda.synchronize() + # end_kernel = time.time() + # print("second time hamming kernel time taken: ", end_kernel-start_kernel) row_element_counts = d_row_element_counts.copy_to_host() @@ -903,24 +960,33 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off data = d_data.copy_to_host() indices = d_indices.copy_to_host() - return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])) + return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])), time_taken - block_width = 4096 - n_blocks = seqs_mat2.shape[0] // block_width + 1 + block_width = 512 + n_blocks = 50 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) result_blocks = [None] * n_blocks block_offset = start_column + time_sum = 0 for i in range(0, n_blocks): - result_blocks[i] = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], block_offset) + result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], block_offset) + time_sum += time_taken block_offset += seqs_mat2_blocks[i].shape[0] + print("time_sum: ", time_sum) result_sparse = scipy.sparse.hstack(result_blocks) + size_in_bytes = result_sparse.data.nbytes + result_sparse.indices.nbytes + result_sparse.indptr.nbytes + size_in_gb = size_in_bytes / (1024 ** 3) + print(f"Size of the CSR matrix: {size_in_gb:.6f} GB") + row_element_counts_gpu = np.diff(result_sparse.indptr) + print("max row element count: ", np.max(row_element_counts_gpu)) + return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) _metric_mat = _gpu_hamming_mat From 0b0c5fbd1e7b3733b524e88de60834a174a10b48 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sun, 29 Sep 2024 15:18:41 +0200 Subject: [PATCH 67/94] cupy experiments --- src/scirpy/ir_dist/metrics.py | 255 ++++++++++++++--------- src/scirpy/tests/test_ir_dist_metrics.py | 9 +- 2 files changed, 161 insertions(+), 103 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index a6728dc8c..0d42e6eb4 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -10,7 +10,8 @@ import scipy.sparse import scipy.spatial from Levenshtein import distance as levenshtein_dist -from numba import cuda +# from numba import cuda +import cupy as cp from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix @@ -819,107 +820,130 @@ def _gpu_hamming_mat( seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) - # import matplotlib.pyplot as plt - # plt.hist(seqs_L1, bins=np.max(seqs_L1), edgecolor='black') - # plt.show() - - @cuda.jit - def hamming_kernel( - seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset - ): - row = cuda.grid(1) - if row < seqs_mat1.shape[0]: - row_end_index = 0 - seq1_len = seqs_L1[row] - for col in range(seqs_mat2.shape[0]): - if (not is_symmetric) or ((col + block_offset) >= row): - seq2_len = seqs_L2[col] - distance = 1 - if seq1_len == seq2_len: - for i in range(0, seq1_len): - distance += seqs_mat1[row, i] != seqs_mat2[col, i] - if distance <= cutoff + 1: - data[row, row_end_index] = distance - indices[row, row_end_index] = col - row_end_index += 1 - row_element_counts[row] = row_end_index - # @cuda.jit # def hamming_kernel( # seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset # ): # row = cuda.grid(1) # if row < seqs_mat1.shape[0]: - # # row_end_index = 0 + # row_end_index = 0 # seq1_len = seqs_L1[row] # for col in range(seqs_mat2.shape[0]): - # seq2_len = seqs_L2[col] - # if seq1_len == seq2_len: # (col%1000 == 0): - # # data[row, col] = 1 - # # indices[row, col] = col - # data[row, col] = 1 - # data[row, col] = 0 - - - # # row_end_index += 1 - # # row_element_counts[row] = row_end_index - - @cuda.jit - def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): - row, col = cuda.grid(2) - if row < data_matrix.shape[0] and col < data_matrix.shape[1]: - row_start = indptr[row] - row_end = indptr[row + 1] - row_end_index = row_end - row_start - data_index = row_start + col - if (data_index < data.shape[0]) and (col < row_end_index): - data[data_index] = data_matrix[row, col] - indices[data_index] = indices_matrix[row, col] + # if (not is_symmetric) or ((col + block_offset) >= row): + # seq2_len = seqs_L2[col] + # distance = 1 + # if seq1_len == seq2_len: + # for i in range(0, seq1_len): + # distance += seqs_mat1[row, i] != seqs_mat2[col, i] + # if distance <= cutoff + 1: + # data[row, row_end_index] = distance + # indices[row, row_end_index] = col + # row_end_index += 1 + # row_element_counts[row] = row_end_index + + hamming_kernel = cp.RawKernel(r''' + extern "C" __global__ + void hamming_kernel( + const int* seqs_mat1, const int* seqs_mat2, + const int* seqs_L1, const int* seqs_L2, const int cutoff, + int* data, int* indices, int* row_element_counts, const int block_offset, + const int seqs_mat1_rows, const int seqs_mat2_rows, const int seqs_mat1_cols, const int seqs_mat2_cols, const bool is_symmetric + ) { + int row = blockDim.x * blockIdx.x + threadIdx.x; // Get thread index + if (row < seqs_mat1_rows) { + if(row == 0){ + printf("*********%d********", cutoff); + } + int seq1_len = seqs_L1[row]; + int row_end_index = 0; + for (int col = 0; col < seqs_mat2_rows; col++) { + if ((! is_symmetric ) || (col + block_offset) >= row) { + int seq2_len = seqs_L2[col]; + int distance = 1; + if (seq1_len == seq2_len) { + for (int i = 0; i < seq1_len; i++) { + if(seqs_mat1[row * seqs_mat1_cols + i] != seqs_mat2[col * seqs_mat2_cols + i]) { + distance++; + } + } + if (distance <= cutoff + 1) { + data[row * seqs_mat2_cols + row_end_index] = distance; + indices[row * seqs_mat2_cols + row_end_index] = col; + row_end_index++; + } + } + } + } + row_element_counts[row] = row_end_index; + } + } + ''', 'hamming_kernel') - def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_offset): - d_seqs_mat1 = cuda.to_device(seqs_mat1_block) - d_seqs_mat2 = cuda.to_device(seqs_mat2) - d_seqs_L1 = cuda.to_device(seqs_L1_block) - d_seqs_L2 = cuda.to_device(seqs_L2) + # @cuda.jit + # def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): + # row, col = cuda.grid(2) + # if row < data_matrix.shape[0] and col < data_matrix.shape[1]: + # row_start = indptr[row] + # row_end = indptr[row + 1] + # row_end_index = row_end - row_start + # data_index = row_start + col + # if (data_index < data.shape[0]) and (col < row_end_index): + # data[data_index] = data_matrix[row, col] + # indices[data_index] = indices_matrix[row, col] + + create_csr_kernel = cp.RawKernel(r''' + extern "C" __global__ + void create_csr_kernel( + int* data, int* indices, + int* data_matrix, int* indices_matrix, + int* indptr, int data_matrix_rows, int data_matrix_cols, int data_rows, int indices_matrix_cols + ) { + int row = blockDim.x * blockIdx.x + threadIdx.x; + int col = blockDim.y * blockIdx.y + threadIdx.y; + + if (row < data_matrix_rows && col < data_matrix_cols) { + unsigned int row_start = indptr[row]; + unsigned int row_end = indptr[row + 1]; + unsigned int row_end_index = row_end - row_start; + unsigned int data_index = row_start + col; + + if (data_index < data_rows && col < row_end_index) { + data[data_index] = data_matrix[row * data_matrix_cols + col]; + indices[data_index] = indices_matrix[row * indices_matrix_cols + col]; + } + } + } + ''', 'create_csr_kernel') - result_len = 50 + def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_offset): + + # Transfer data to GPU (CuPy automatically places arrays on GPU) + d_seqs_mat1 = cp.asarray(seqs_mat1_block.astype(int)) + d_seqs_mat2 = cp.asarray(seqs_mat2.astype(int)) + d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) + d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) + + result_len = 100 - data_matrix = np.zeros((seqs_mat1_block.shape[0], result_len), dtype=np.uint8) # np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint8) - d_data_matrix = cuda.device_array_like(data_matrix) + # Create output arrays (on GPU) using CuPy + d_data_matrix = cp.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=cp.int_) + d_indices_matrix = cp.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=cp.int_) + d_row_element_counts = cp.zeros(seqs_mat1_block.shape[0], dtype=cp.int_) - indices_matrix = np.zeros((seqs_mat1_block.shape[0], result_len), dtype=np.uint32) # np.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=np.uint32) - d_indices_matrix = cuda.device_array_like(indices_matrix) + # Configure the grid and block sizes + threads_per_block = 256 + blocks_per_grid = (seqs_mat1_block.shape[0] + (threads_per_block - 1)) // threads_per_block + + print('seqs_mat1_block shape:', seqs_mat1_block.shape) + print('seqs_mat2 shape:', seqs_mat2.shape) + print("d_seqs_L1:", np.min(d_seqs_L1)) - row_element_counts = np.zeros(seqs_mat1_block.shape[0], dtype=np.uint32) - d_row_element_counts = cuda.device_array_like(row_element_counts) + seqs_mat1_rows, seqs_mat1_cols = seqs_mat1_block.shape + seqs_mat2_rows, seqs_mat2_cols = seqs_mat2.shape - threads_per_block = 256 - blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block + cp.cuda.Device().synchronize() - # threads_per_block = (1, 256) - # blocks_per_grid_x = math.ceil(data_matrix.shape[0] / threads_per_block[0]) - # blocks_per_grid_y = math.ceil(data_matrix.shape[1] / threads_per_block[1]) - # blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) - - cuda.synchronize() start_kernel = time.time() - hamming_kernel[blocks_per_grid, threads_per_block]( - d_seqs_mat1, - d_seqs_mat2, - d_seqs_L1, - d_seqs_L2, - self.cutoff, - d_data_matrix, - d_indices_matrix, - d_row_element_counts, - block_offset, - ) - cuda.synchronize() - end_kernel = time.time() - time_taken = (end_kernel-start_kernel) - print("first time hamming kernel time taken: ", time_taken) - - # start_kernel = time.time() # hamming_kernel[blocks_per_grid, threads_per_block]( # d_seqs_mat1, # d_seqs_mat2, @@ -931,39 +955,70 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off # d_row_element_counts, # block_offset, # ) - # cuda.synchronize() - # end_kernel = time.time() - # print("second time hamming kernel time taken: ", end_kernel-start_kernel) - row_element_counts = d_row_element_counts.copy_to_host() + hamming_kernel( + (blocks_per_grid,), (threads_per_block,), + ( + d_seqs_mat1, + d_seqs_mat2, + d_seqs_L1, + d_seqs_L2, + self.cutoff, + d_data_matrix, + d_indices_matrix, + d_row_element_counts, + block_offset, + seqs_mat1_rows, + seqs_mat2_rows, + seqs_mat1_cols, + seqs_mat2_cols, + is_symmetric, + ) + ) + + cp.cuda.Device().synchronize() + + exit() + + end_kernel = time.time() + time_taken = (end_kernel-start_kernel) + print("first time hamming kernel time taken: ", time_taken) + + row_element_counts = d_row_element_counts.get() indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.uint32) indptr[1:] = np.cumsum(row_element_counts) - d_indptr = cuda.to_device(indptr) + d_indptr = cp.asarray(indptr) n_elements = indptr[-1] data = np.zeros(n_elements, dtype=np.uint8) - d_data = cuda.device_array_like(data) + d_data = cp.zeros_like(data) indices = np.zeros(n_elements, dtype=np.uint32) - d_indices = cuda.device_array_like(indices) + d_indices = cp.zeros_like(indices) threads_per_block = (1, 256) blocks_per_grid_x = (d_data_matrix.shape[0] + threads_per_block[0] - 1) // threads_per_block[0] blocks_per_grid_y = (d_data_matrix.shape[1] + threads_per_block[1] - 1) // threads_per_block[1] blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) - create_csr_kernel[blocks_per_grid, threads_per_block]( - d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr + + # create_csr_kernel[blocks_per_grid, threads_per_block]( + # d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr + # ) + + create_csr_kernel( + (blocks_per_grid_x, blocks_per_grid_y), threads_per_block, + (d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr, d_data_matrix.shape[0], d_data_matrix.shape[1],d_data.shape[0], d_indices_matrix.shape[1]) ) - data = d_data.copy_to_host() - indices = d_indices.copy_to_host() + data = d_data.get() + indices = d_indices.get() return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])), time_taken - block_width = 512 - n_blocks = 50 # seqs_mat2.shape[0] // block_width + 1 + block_width = 4096 + n_blocks = 1 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) @@ -1550,4 +1605,4 @@ def _num_different_characters(self, s1, s2, len_diff): for c in shorter: if c in longer: longer = longer.replace(c, "", 1) - return len(longer) - len_diff + return len(longer) - len_diff \ No newline at end of file diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 2e9b8b4dc..87ada8b4b 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -753,9 +753,12 @@ def test_gpu_hamming_reference(): seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_csr_result.npz") + gpu_hamming_calculator = GPUHammingDistanceCalculator(cutoff=2) res = gpu_hamming_calculator.calc_dist_mat(seqs, seqs) - assert np.array_equal(res.data, reference_result.data) - assert np.array_equal(res.indices, reference_result.indices) - assert np.array_equal(res.indptr, reference_result.indptr) + print(res.indptr[-10:-1]) + + # assert np.array_equal(res.data, reference_result.data) + # assert np.array_equal(res.indices, reference_result.indices) + # assert np.array_equal(res.indptr, reference_result.indptr) From babdf9a7d72cae39369ef29e2ba8d222d1aaead7 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 30 Sep 2024 11:31:47 +0200 Subject: [PATCH 68/94] cupy experiments --- src/scirpy/ir_dist/metrics.py | 60 +++++++++++++++++------- src/scirpy/tests/test_ir_dist_metrics.py | 9 ++-- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 0d42e6eb4..9bc49e481 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -851,13 +851,10 @@ def _gpu_hamming_mat( ) { int row = blockDim.x * blockIdx.x + threadIdx.x; // Get thread index if (row < seqs_mat1_rows) { - if(row == 0){ - printf("*********%d********", cutoff); - } int seq1_len = seqs_L1[row]; int row_end_index = 0; for (int col = 0; col < seqs_mat2_rows; col++) { - if ((! is_symmetric ) || (col + block_offset) >= row) { + //if ((! is_symmetric ) || (col + block_offset) >= row) { int seq2_len = seqs_L2[col]; int distance = 1; if (seq1_len == seq2_len) { @@ -867,12 +864,12 @@ def _gpu_hamming_mat( } } if (distance <= cutoff + 1) { - data[row * seqs_mat2_cols + row_end_index] = distance; - indices[row * seqs_mat2_cols + row_end_index] = col; + data[row * seqs_mat2_rows + row_end_index] = distance; + indices[row * seqs_mat2_rows + row_end_index] = col; row_end_index++; } } - } + //} } row_element_counts[row] = row_end_index; } @@ -902,12 +899,12 @@ def _gpu_hamming_mat( int col = blockDim.y * blockIdx.y + threadIdx.y; if (row < data_matrix_rows && col < data_matrix_cols) { - unsigned int row_start = indptr[row]; - unsigned int row_end = indptr[row + 1]; - unsigned int row_end_index = row_end - row_start; - unsigned int data_index = row_start + col; + int row_start = indptr[row]; + int row_end = indptr[row + 1]; + int row_end_index = row_end - row_start; + int data_index = row_start + col; - if (data_index < data_rows && col < row_end_index) { + if ((data_index < data_rows) && (col < row_end_index)) { data[data_index] = data_matrix[row * data_matrix_cols + col]; indices[data_index] = indices_matrix[row * indices_matrix_cols + col]; } @@ -978,24 +975,39 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off cp.cuda.Device().synchronize() - exit() - end_kernel = time.time() time_taken = (end_kernel-start_kernel) print("first time hamming kernel time taken: ", time_taken) + data_matrix = d_data_matrix.get() + indices_matrix = d_indices_matrix.get() + + print('*****data_matrix shape: ', data_matrix.shape) + print('*****indices_matrix shape: ', indices_matrix.shape) + print('*****data_matrix nonzero: ', np.count_nonzero(data_matrix)) + print('*****d_indices_matrix nonzero: ', np.count_nonzero(indices_matrix)) + np.savetxt('data_matrix.csv', data_matrix, delimiter=',', fmt='%d') + np.savetxt('indices_matrix.csv', indices_matrix, delimiter=',', fmt='%d') + row_element_counts = d_row_element_counts.get() - indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.uint32) + indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.int_) indptr[1:] = np.cumsum(row_element_counts) d_indptr = cp.asarray(indptr) + + + # np.savetxt('data_matrix.csv', data_matrix, delimiter=',', fmt='%d') + + # print('*****indptr: ', indptr[-10:-1]) + # exit() + n_elements = indptr[-1] - data = np.zeros(n_elements, dtype=np.uint8) + data = np.zeros(n_elements, dtype=np.int_) d_data = cp.zeros_like(data) - indices = np.zeros(n_elements, dtype=np.uint32) + indices = np.zeros(n_elements, dtype=np.int_) d_indices = cp.zeros_like(indices) threads_per_block = (1, 256) @@ -1007,12 +1019,26 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off # d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr # ) + # void create_csr_kernel( + # int* data, int* indices, + # int* data_matrix, int* indices_matrix, + # int* indptr, int data_matrix_rows, int data_matrix_cols, int data_rows, int indices_matrix_cols + # ) + create_csr_kernel( (blocks_per_grid_x, blocks_per_grid_y), threads_per_block, (d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr, d_data_matrix.shape[0], d_data_matrix.shape[1],d_data.shape[0], d_indices_matrix.shape[1]) ) data = d_data.get() + indptr = d_indptr.get() + print('*****data nonzero: ', np.count_nonzero(data)) + print('*****data shape: ',data.shape) + np.savetxt('data.csv', data, delimiter=',', fmt='%d') + np.savetxt('indptr.csv', indptr, delimiter=',', fmt='%d') + + + indices = d_indices.get() return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])), time_taken diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 87ada8b4b..2e9b8b4dc 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -753,12 +753,9 @@ def test_gpu_hamming_reference(): seqs = np.load(TESTDATA / "hamming_test_data/hamming_WU3k_seqs.npy") reference_result = scipy.sparse.load_npz(TESTDATA / "hamming_test_data/hamming_WU3k_csr_result.npz") - gpu_hamming_calculator = GPUHammingDistanceCalculator(cutoff=2) res = gpu_hamming_calculator.calc_dist_mat(seqs, seqs) - print(res.indptr[-10:-1]) - - # assert np.array_equal(res.data, reference_result.data) - # assert np.array_equal(res.indices, reference_result.indices) - # assert np.array_equal(res.indptr, reference_result.indptr) + assert np.array_equal(res.data, reference_result.data) + assert np.array_equal(res.indices, reference_result.indices) + assert np.array_equal(res.indptr, reference_result.indptr) From f22ae38a75303ccbb29e12061fe7595e318ab4b7 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 30 Sep 2024 11:56:52 +0200 Subject: [PATCH 69/94] scaled cupy to 1 million cells --- src/scirpy/ir_dist/metrics.py | 91 +++++++++++------------------------ 1 file changed, 29 insertions(+), 62 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 9bc49e481..cdbe0579c 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -844,17 +844,29 @@ def _gpu_hamming_mat( hamming_kernel = cp.RawKernel(r''' extern "C" __global__ void hamming_kernel( - const int* seqs_mat1, const int* seqs_mat2, - const int* seqs_L1, const int* seqs_L2, const int cutoff, - int* data, int* indices, int* row_element_counts, const int block_offset, - const int seqs_mat1_rows, const int seqs_mat2_rows, const int seqs_mat1_cols, const int seqs_mat2_cols, const bool is_symmetric + const int* seqs_mat1, + const int* seqs_mat2, + const int* seqs_L1, + const int* seqs_L2, + const int cutoff, + int* data, + int* indices, + int* row_element_counts, + const int block_offset, + const int seqs_mat1_rows, + const int seqs_mat2_rows, + const int seqs_mat1_cols, + const int seqs_mat2_cols, + const int data_cols, + const int indices_cols, + const bool is_symmetric ) { int row = blockDim.x * blockIdx.x + threadIdx.x; // Get thread index if (row < seqs_mat1_rows) { int seq1_len = seqs_L1[row]; int row_end_index = 0; for (int col = 0; col < seqs_mat2_rows; col++) { - //if ((! is_symmetric ) || (col + block_offset) >= row) { + if ((! is_symmetric ) || (col + block_offset) >= row) { int seq2_len = seqs_L2[col]; int distance = 1; if (seq1_len == seq2_len) { @@ -864,12 +876,12 @@ def _gpu_hamming_mat( } } if (distance <= cutoff + 1) { - data[row * seqs_mat2_rows + row_end_index] = distance; - indices[row * seqs_mat2_rows + row_end_index] = col; + data[row * data_cols + row_end_index] = distance; + indices[row * indices_cols + row_end_index] = col; row_end_index++; } } - //} + } } row_element_counts[row] = row_end_index; } @@ -920,38 +932,25 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) - result_len = 100 + result_len = 200 # Create output arrays (on GPU) using CuPy - d_data_matrix = cp.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=cp.int_) - d_indices_matrix = cp.zeros((seqs_mat1_block.shape[0], seqs_mat2.shape[0]), dtype=cp.int_) + d_data_matrix = cp.zeros((seqs_mat1_block.shape[0], result_len), dtype=cp.int_) + d_indices_matrix = cp.zeros((seqs_mat1_block.shape[0], result_len), dtype=cp.int_) d_row_element_counts = cp.zeros(seqs_mat1_block.shape[0], dtype=cp.int_) # Configure the grid and block sizes threads_per_block = 256 blocks_per_grid = (seqs_mat1_block.shape[0] + (threads_per_block - 1)) // threads_per_block - - print('seqs_mat1_block shape:', seqs_mat1_block.shape) - print('seqs_mat2 shape:', seqs_mat2.shape) - print("d_seqs_L1:", np.min(d_seqs_L1)) seqs_mat1_rows, seqs_mat1_cols = seqs_mat1_block.shape seqs_mat2_rows, seqs_mat2_cols = seqs_mat2.shape + d_data_matrix_cols = result_len + d_indices_matrix_cols = result_len cp.cuda.Device().synchronize() start_kernel = time.time() - # hamming_kernel[blocks_per_grid, threads_per_block]( - # d_seqs_mat1, - # d_seqs_mat2, - # d_seqs_L1, - # d_seqs_L2, - # self.cutoff, - # d_data_matrix, - # d_indices_matrix, - # d_row_element_counts, - # block_offset, - # ) hamming_kernel( (blocks_per_grid,), (threads_per_block,), @@ -969,6 +968,8 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off seqs_mat2_rows, seqs_mat1_cols, seqs_mat2_cols, + d_data_matrix_cols, + d_indices_matrix_cols, is_symmetric, ) ) @@ -979,29 +980,12 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off time_taken = (end_kernel-start_kernel) print("first time hamming kernel time taken: ", time_taken) - data_matrix = d_data_matrix.get() - indices_matrix = d_indices_matrix.get() - - print('*****data_matrix shape: ', data_matrix.shape) - print('*****indices_matrix shape: ', indices_matrix.shape) - print('*****data_matrix nonzero: ', np.count_nonzero(data_matrix)) - print('*****d_indices_matrix nonzero: ', np.count_nonzero(indices_matrix)) - np.savetxt('data_matrix.csv', data_matrix, delimiter=',', fmt='%d') - np.savetxt('indices_matrix.csv', indices_matrix, delimiter=',', fmt='%d') - row_element_counts = d_row_element_counts.get() indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.int_) indptr[1:] = np.cumsum(row_element_counts) d_indptr = cp.asarray(indptr) - - - # np.savetxt('data_matrix.csv', data_matrix, delimiter=',', fmt='%d') - - # print('*****indptr: ', indptr[-10:-1]) - # exit() - n_elements = indptr[-1] data = np.zeros(n_elements, dtype=np.int_) @@ -1014,17 +998,7 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off blocks_per_grid_x = (d_data_matrix.shape[0] + threads_per_block[0] - 1) // threads_per_block[0] blocks_per_grid_y = (d_data_matrix.shape[1] + threads_per_block[1] - 1) // threads_per_block[1] blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) - - # create_csr_kernel[blocks_per_grid, threads_per_block]( - # d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr - # ) - - # void create_csr_kernel( - # int* data, int* indices, - # int* data_matrix, int* indices_matrix, - # int* indptr, int data_matrix_rows, int data_matrix_cols, int data_rows, int indices_matrix_cols - # ) - + create_csr_kernel( (blocks_per_grid_x, blocks_per_grid_y), threads_per_block, (d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr, d_data_matrix.shape[0], d_data_matrix.shape[1],d_data.shape[0], d_indices_matrix.shape[1]) @@ -1032,19 +1006,12 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off data = d_data.get() indptr = d_indptr.get() - print('*****data nonzero: ', np.count_nonzero(data)) - print('*****data shape: ',data.shape) - np.savetxt('data.csv', data, delimiter=',', fmt='%d') - np.savetxt('indptr.csv', indptr, delimiter=',', fmt='%d') - - - indices = d_indices.get() return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])), time_taken block_width = 4096 - n_blocks = 1 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) From 730cb8007f461bd673ed03729eaa8c326d48ae8f Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 2 Oct 2024 12:25:42 +0200 Subject: [PATCH 70/94] sorted sequences by length --- src/scirpy/ir_dist/metrics.py | 50 +++++++++++++++++++++--- src/scirpy/tests/test_ir_dist_metrics.py | 7 ++-- 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index cdbe0579c..d5412aebb 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -814,6 +814,30 @@ def _gpu_hamming_mat( Always returns a numpy array containing None because the computation of the minimum distance per row is not implemented for the GPU hamming calculator yet. """ + + print("seqs unsorted: ", seqs[0:20]) + start_sorting = time.time() + # seqs = np.array(sorted(seqs, key=len)) + # seqs2 = np.array(sorted(seqs2, key=len)) + + seqs_lengths = np.vectorize(len)(seqs) + seqs_original_indices = np.argsort(seqs_lengths) + seqs = seqs[seqs_original_indices] + + seqs2_lengths = np.vectorize(len)(seqs2) + seqs2_original_indices = np.argsort(seqs2_lengths) + seqs2 = seqs2[seqs2_original_indices] + + end_sorting = time.time() + print("sorting time taken: ", end_sorting-start_sorting) + print("seqs sorted: ", seqs[0:20]) + + # seqs_original_indices = np.arange(0, len(seqs)) + # seqs2_original_indices = np.arange(0, len(seqs2)) + + is_symmetric = False + print("is_symmetric: ", is_symmetric) + unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) @@ -848,6 +872,8 @@ def _gpu_hamming_mat( const int* seqs_mat2, const int* seqs_L1, const int* seqs_L2, + const int* seqs_original_indices, + const int* seqs2_original_indices, const int cutoff, int* data, int* indices, @@ -861,8 +887,9 @@ def _gpu_hamming_mat( const int indices_cols, const bool is_symmetric ) { - int row = blockDim.x * blockIdx.x + threadIdx.x; // Get thread index + int row = blockDim.x * blockIdx.x + threadIdx.x; if (row < seqs_mat1_rows) { + int seqs_original_index = seqs_original_indices[row]; int seq1_len = seqs_L1[row]; int row_end_index = 0; for (int col = 0; col < seqs_mat2_rows; col++) { @@ -876,14 +903,15 @@ def _gpu_hamming_mat( } } if (distance <= cutoff + 1) { - data[row * data_cols + row_end_index] = distance; - indices[row * indices_cols + row_end_index] = col; + int seqs2_original_index = seqs2_original_indices[col]; + data[seqs_original_index * data_cols + row_end_index] = distance; + indices[seqs_original_index * indices_cols + row_end_index] = seqs2_original_index; row_end_index++; } } } } - row_element_counts[row] = row_end_index; + row_element_counts[seqs_original_index] = row_end_index; } } ''', 'hamming_kernel') @@ -959,6 +987,8 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off d_seqs_mat2, d_seqs_L1, d_seqs_L2, + cp.asarray(seqs_original_indices.astype(int), dtype=cp.int_), + cp.asarray(seqs2_original_indices.astype(int), dtype=cp.int_), self.cutoff, d_data_matrix, d_indices_matrix, @@ -982,12 +1012,17 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off row_element_counts = d_row_element_counts.get() + # np.savetxt('d_data_matrix_ref.csv', d_data_matrix.get(), delimiter=',', fmt='%d') + # np.savetxt('d_indices_matrix_ref.csv', d_data_matrix.get(), delimiter=',', fmt='%d') + indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.int_) indptr[1:] = np.cumsum(row_element_counts) d_indptr = cp.asarray(indptr) n_elements = indptr[-1] + print("***n_elements: ", n_elements) + data = np.zeros(n_elements, dtype=np.int_) d_data = cp.zeros_like(data) @@ -1008,10 +1043,15 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off indptr = d_indptr.get() indices = d_indices.get() + # np.savetxt('data.csv', data, delimiter=',', fmt='%d') + # print("np.count_nonzero(data):", np.count_nonzero(data)) + # print("np.count_nonzero(indices):", np.count_nonzero(indices)) + # print("indptr[-1]: ", indptr[-1]) + return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])), time_taken block_width = 4096 - n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 1 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 2e9b8b4dc..a2d108154 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -756,6 +756,7 @@ def test_gpu_hamming_reference(): gpu_hamming_calculator = GPUHammingDistanceCalculator(cutoff=2) res = gpu_hamming_calculator.calc_dist_mat(seqs, seqs) - assert np.array_equal(res.data, reference_result.data) - assert np.array_equal(res.indices, reference_result.indices) - assert np.array_equal(res.indptr, reference_result.indptr) + # assert np.array_equal(res.data, reference_result.data) + # assert np.array_equal(res.indices, reference_result.indices) + # assert np.array_equal(res.indptr, reference_result.indptr) + assert np.array_equal(res.todense(), reference_result.todense()) From 5e4776ca20c789d06fff235a33095ebacb0665f1 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 16 Oct 2024 09:20:02 +0200 Subject: [PATCH 71/94] textures used for seqs_mat1 and seqs_mat2 --- src/scirpy/ir_dist/metrics.py | 78 +++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 9 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index d5412aebb..ad148c734 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -832,8 +832,8 @@ def _gpu_hamming_mat( print("sorting time taken: ", end_sorting-start_sorting) print("seqs sorted: ", seqs[0:20]) - # seqs_original_indices = np.arange(0, len(seqs)) - # seqs2_original_indices = np.arange(0, len(seqs2)) + seqs_original_indices = cp.asarray(seqs_original_indices.astype(int), dtype=cp.int_) + seqs2_original_indices = cp.asarray(seqs2_original_indices.astype(int), dtype=cp.int_) is_symmetric = False print("is_symmetric: ", is_symmetric) @@ -841,6 +841,8 @@ def _gpu_hamming_mat( unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) + print(f"***max_seq_len: {max_seq_len}") + seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) @@ -868,8 +870,10 @@ def _gpu_hamming_mat( hamming_kernel = cp.RawKernel(r''' extern "C" __global__ void hamming_kernel( - const int* seqs_mat1, - const int* seqs_mat2, + const int* seqs_mat1, //cudaTextureObject_t seqs_mat1, + const int* seqs_mat2, //cudaTextureObject_t seqs_mat2, + cudaTextureObject_t tex_mat1, + cudaTextureObject_t tex_mat2, const int* seqs_L1, const int* seqs_L2, const int* seqs_original_indices, @@ -898,9 +902,27 @@ def _gpu_hamming_mat( int distance = 1; if (seq1_len == seq2_len) { for (int i = 0; i < seq1_len; i++) { - if(seqs_mat1[row * seqs_mat1_cols + i] != seqs_mat2[col * seqs_mat2_cols + i]) { + /* + int val1 = seqs_mat1[row, i];//tex2D(seqs_mat1, row, i); + int val2 = seqs_mat2[col, i];//tex2D(seqs_mat2, col, i); + if(val1 != val2) { + distance++; + } + */ + /* + int val1 = seqs_mat1[row * seqs_mat1_cols + i]; + int val2 = seqs_mat2[col * seqs_mat2_cols + i]; + if( val1 != val2) { distance++; } + */ + + int tex_val1 = tex2D(tex_mat1, i, row); + int tex_val2 = tex2D(tex_mat2, i, col); + if( tex_val1 != tex_val2) { + distance++; + } + } if (distance <= cutoff + 1) { int seqs2_original_index = seqs2_original_indices[col]; @@ -976,6 +998,42 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off d_data_matrix_cols = result_len d_indices_matrix_cols = result_len + from cupy.cuda import texture + + def create_texture(cupy_array) -> texture.TextureObject: + width, height, depth = (cupy_array.shape[1], cupy_array.shape[0], 0) + dim = 3 if depth != 0 else 2 if height != 0 else 1 + + # generate input data and allocate output buffer + shape = (depth, height, width) if dim == 3 else \ + (height, width) if dim == 2 else \ + (width,) + + # prepare input, output, and texture memory + # self.data holds the data stored in the texture memory + tex_data = cupy_array #cupy_array + ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, + cp.cuda.runtime.cudaChannelFormatKindSigned) + + print(f"{tex_data.shape}, {width}, {height}") + arr = texture.CUDAarray(ch, width, height) + + arr.copy_from(tex_data) + # create resource and texture descriptors + res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) + address_mode = (cp.cuda.runtime.cudaAddressModeClamp, + ) + tex = texture.TextureDescriptor(address_mode, cp.cuda.runtime.cudaFilterModePoint, + cp.cuda.runtime.cudaReadModeElementType) + + # create a texture object + return texture.TextureObject(res, tex) + + tex_mat1 = create_texture(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + tex_mat2 = create_texture(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + #tex_mat3 = create_texture(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + #tex_mat4 = create_texture(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + cp.cuda.Device().synchronize() start_kernel = time.time() @@ -983,12 +1041,14 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off hamming_kernel( (blocks_per_grid,), (threads_per_block,), ( - d_seqs_mat1, - d_seqs_mat2, + d_seqs_mat1, # tex_seqs_mat1, + d_seqs_mat2, #tex_seqs_mat2, + tex_mat1, + tex_mat2, d_seqs_L1, d_seqs_L2, - cp.asarray(seqs_original_indices.astype(int), dtype=cp.int_), - cp.asarray(seqs2_original_indices.astype(int), dtype=cp.int_), + seqs_original_indices, + seqs2_original_indices, self.cutoff, d_data_matrix, d_indices_matrix, From e6bf3933b35fdc8a46ca361d4374ca18f0f55095 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 16 Oct 2024 11:15:17 +0200 Subject: [PATCH 72/94] texture mit up to 100k cells --- src/scirpy/ir_dist/metrics.py | 93 ++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 34 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index ad148c734..815d8b58a 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -870,14 +870,14 @@ def _gpu_hamming_mat( hamming_kernel = cp.RawKernel(r''' extern "C" __global__ void hamming_kernel( - const int* seqs_mat1, //cudaTextureObject_t seqs_mat1, - const int* seqs_mat2, //cudaTextureObject_t seqs_mat2, + const int* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, + const int* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, cudaTextureObject_t tex_mat1, cudaTextureObject_t tex_mat2, - const int* seqs_L1, - const int* seqs_L2, - const int* seqs_original_indices, - const int* seqs2_original_indices, + cudaTextureObject_t tex_L1, //const int* seqs_L1, + cudaTextureObject_t tex_L2, //const int* seqs_L2, + cudaTextureObject_t seqs_original_indices, //const int* seqs_original_indices, + cudaTextureObject_t seqs2_original_indices, //const int* seqs2_original_indices, const int cutoff, int* data, int* indices, @@ -893,12 +893,12 @@ def _gpu_hamming_mat( ) { int row = blockDim.x * blockIdx.x + threadIdx.x; if (row < seqs_mat1_rows) { - int seqs_original_index = seqs_original_indices[row]; - int seq1_len = seqs_L1[row]; + int seqs_original_index = tex1D(seqs_original_indices, row); //seqs_original_indices[row]; + int seq1_len = tex1D(tex_L1, row);//seqs_L1[row]; int row_end_index = 0; for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { - int seq2_len = seqs_L2[col]; + int seq2_len = tex1D(tex_L2, col); // seqs_L2[col]; int distance = 1; if (seq1_len == seq2_len) { for (int i = 0; i < seq1_len; i++) { @@ -910,22 +910,22 @@ def _gpu_hamming_mat( } */ /* - int val1 = seqs_mat1[row * seqs_mat1_cols + i]; - int val2 = seqs_mat2[col * seqs_mat2_cols + i]; + int val1 = __ldg(&seqs_mat1[row * seqs_mat1_cols + i]); + int val2 = __ldg(&seqs_mat2[col * seqs_mat2_cols + i]); if( val1 != val2) { distance++; } */ - int tex_val1 = tex2D(tex_mat1, i, row); - int tex_val2 = tex2D(tex_mat2, i, col); + int tex_val1 = tex2D(tex_mat1, row, i); + int tex_val2 = tex2D(tex_mat2, col, i); if( tex_val1 != tex_val2) { distance++; } } if (distance <= cutoff + 1) { - int seqs2_original_index = seqs2_original_indices[col]; + int seqs2_original_index = tex1D(seqs2_original_indices, col);//seqs2_original_indices[col]; data[seqs_original_index * data_cols + row_end_index] = distance; indices[seqs_original_index * indices_cols + row_end_index] = seqs2_original_index; row_end_index++; @@ -1000,40 +1000,65 @@ def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_off from cupy.cuda import texture - def create_texture(cupy_array) -> texture.TextureObject: - width, height, depth = (cupy_array.shape[1], cupy_array.shape[0], 0) + def create_texture1D(cupy_array) -> texture.TextureObject: + width, height, depth = (cupy_array.shape[0], 0, 0) dim = 3 if depth != 0 else 2 if height != 0 else 1 - # generate input data and allocate output buffer shape = (depth, height, width) if dim == 3 else \ (height, width) if dim == 2 else \ (width,) - # prepare input, output, and texture memory - # self.data holds the data stored in the texture memory - tex_data = cupy_array #cupy_array + tex_data = cupy_array ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, cp.cuda.runtime.cudaChannelFormatKindSigned) print(f"{tex_data.shape}, {width}, {height}") - arr = texture.CUDAarray(ch, width, height) + arr = texture.CUDAarray(ch, width) arr.copy_from(tex_data) - # create resource and texture descriptors res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) address_mode = (cp.cuda.runtime.cudaAddressModeClamp, ) tex = texture.TextureDescriptor(address_mode, cp.cuda.runtime.cudaFilterModePoint, cp.cuda.runtime.cudaReadModeElementType) + return texture.TextureObject(res, tex) + + def create_texture2D(cupy_array) -> texture.TextureObject: + width, height, depth = (cupy_array.shape[0], cupy_array.shape[1], 0) + dim = 3 if depth != 0 else 2 if height != 0 else 1 - # create a texture object + shape = (depth, height, width) if dim == 3 else \ + (height, width) if dim == 2 else \ + (width,) + + tex_data = cp.transpose(cupy_array) + ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, + cp.cuda.runtime.cudaChannelFormatKindSigned) + + print(f"{tex_data.shape}, {width}, {height}") + arr = texture.CUDAarray(ch, width, height) + + arr.copy_from(tex_data) + res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) + address_mode = (cp.cuda.runtime.cudaAddressModeClamp, + ) + tex = texture.TextureDescriptor(address_mode, cp.cuda.runtime.cudaFilterModePoint, + cp.cuda.runtime.cudaReadModeElementType) return texture.TextureObject(res, tex) - tex_mat1 = create_texture(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - tex_mat2 = create_texture(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) - #tex_mat3 = create_texture(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - #tex_mat4 = create_texture(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + tex_seqs_mat1 = create_texture2D(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + tex_seqs_mat2 = create_texture2D(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + tex_seqs2_original_indices = create_texture1D(seqs2_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + + start_compile = time.time() + hamming_kernel.compile() + end_compile = time.time() + print("compile time taken: ", end_compile-start_compile) + cp.get_default_memory_pool().free_all_blocks() cp.cuda.Device().synchronize() start_kernel = time.time() @@ -1043,12 +1068,12 @@ def create_texture(cupy_array) -> texture.TextureObject: ( d_seqs_mat1, # tex_seqs_mat1, d_seqs_mat2, #tex_seqs_mat2, - tex_mat1, - tex_mat2, - d_seqs_L1, - d_seqs_L2, - seqs_original_indices, - seqs2_original_indices, + tex_seqs_mat1, + tex_seqs_mat2, + tex_seqs_L1, # d_seqs_L1, + tex_seqs_L2, # d_seqs_L2, + tex_seqs_original_indices, #seqs_original_indices, + tex_seqs2_original_indices, #seqs2_original_indices, self.cutoff, d_data_matrix, d_indices_matrix, @@ -1068,7 +1093,7 @@ def create_texture(cupy_array) -> texture.TextureObject: end_kernel = time.time() time_taken = (end_kernel-start_kernel) - print("first time hamming kernel time taken: ", time_taken) + print("hamming kernel time taken: ", time_taken) row_element_counts = d_row_element_counts.get() From da251d00085531bad10c81e4914a176c83bdca5d Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 16 Oct 2024 15:04:21 +0200 Subject: [PATCH 73/94] sorted seqs with multiple blocks --- src/scirpy/ir_dist/metrics.py | 56 +++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 815d8b58a..105ac95c9 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -974,27 +974,27 @@ def _gpu_hamming_mat( } ''', 'create_csr_kernel') - def calc_block_gpu(seqs_mat1_block, seqs_mat2, seqs_L1_block, seqs_L2, block_offset): + def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset): # Transfer data to GPU (CuPy automatically places arrays on GPU) - d_seqs_mat1 = cp.asarray(seqs_mat1_block.astype(int)) - d_seqs_mat2 = cp.asarray(seqs_mat2.astype(int)) + d_seqs_mat1 = cp.asarray(seqs_mat1.astype(int)) + d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(int)) d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) result_len = 200 # Create output arrays (on GPU) using CuPy - d_data_matrix = cp.zeros((seqs_mat1_block.shape[0], result_len), dtype=cp.int_) - d_indices_matrix = cp.zeros((seqs_mat1_block.shape[0], result_len), dtype=cp.int_) - d_row_element_counts = cp.zeros(seqs_mat1_block.shape[0], dtype=cp.int_) + d_data_matrix = cp.zeros((seqs_mat1.shape[0], result_len), dtype=cp.int_) + d_indices_matrix = cp.zeros((seqs_mat1.shape[0], result_len), dtype=cp.int_) + d_row_element_counts = cp.zeros(seqs_mat1.shape[0], dtype=cp.int_) # Configure the grid and block sizes threads_per_block = 256 - blocks_per_grid = (seqs_mat1_block.shape[0] + (threads_per_block - 1)) // threads_per_block + blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block - seqs_mat1_rows, seqs_mat1_cols = seqs_mat1_block.shape - seqs_mat2_rows, seqs_mat2_cols = seqs_mat2.shape + seqs_mat1_rows, seqs_mat1_cols = seqs_mat1.shape + seqs_mat2_rows, seqs_mat2_cols = seqs_mat2_block.shape d_data_matrix_cols = result_len d_indices_matrix_cols = result_len @@ -1051,7 +1051,7 @@ def create_texture2D(cupy_array) -> texture.TextureObject: tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - tex_seqs2_original_indices = create_texture1D(seqs2_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + tex_seqs2_original_indices = create_texture1D(seqs2_original_indices_blocks) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) start_compile = time.time() hamming_kernel.compile() @@ -1100,7 +1100,7 @@ def create_texture2D(cupy_array) -> texture.TextureObject: # np.savetxt('d_data_matrix_ref.csv', d_data_matrix.get(), delimiter=',', fmt='%d') # np.savetxt('d_indices_matrix_ref.csv', d_data_matrix.get(), delimiter=',', fmt='%d') - indptr = np.zeros(seqs_mat1_block.shape[0] + 1, dtype=np.int_) + indptr = np.zeros(seqs_mat1.shape[0] + 1, dtype=np.int_) indptr[1:] = np.cumsum(row_element_counts) d_indptr = cp.asarray(indptr) @@ -1132,25 +1132,49 @@ def create_texture2D(cupy_array) -> texture.TextureObject: # print("np.count_nonzero(data):", np.count_nonzero(data)) # print("np.count_nonzero(indices):", np.count_nonzero(indices)) # print("indptr[-1]: ", indptr[-1]) - - return csr_matrix((data, indices, indptr), shape=(seqs_mat1_block.shape[0], seqs_mat2.shape[0])), time_taken + res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2_block.shape[0])) + return res, time_taken block_width = 4096 - n_blocks = 1 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 2 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) + seqs2_original_indices_blocks = np.array_split(seqs2_original_indices, n_blocks) result_blocks = [None] * n_blocks block_offset = start_column time_sum = 0 for i in range(0, n_blocks): - result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], block_offset) + result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], seqs2_original_indices_blocks[i], block_offset) time_sum += time_taken block_offset += seqs_mat2_blocks[i].shape[0] print("time_sum: ", time_sum) - result_sparse = scipy.sparse.hstack(result_blocks) + # result_sparse = scipy.sparse.hstack(result_blocks) + # print("block indptr shape: ", [block.indptr.shape for block in result_blocks]) + # print(result_blocks[0].shape[0]) + + data_blocks = [] + indices_blocks = [] + indptr = np.zeros(result_blocks[0].shape[0] + 1) + for result in result_blocks: + indptr+=result.indptr + size_counter = 0 + for row in range(result_blocks[0].shape[0]): + for result in result_blocks: + start = result.indptr[row] + end = result.indptr[row+1] + data = result.data[start:end] + indices = result.indices[start:end] + data_blocks.append(data) + indices_blocks.append(indices) + size_counter += len(data) + + data = np.concatenate(data_blocks) + indices = np.concatenate(indices_blocks) + + result_sparse = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0])) size_in_bytes = result_sparse.data.nbytes + result_sparse.indices.nbytes + result_sparse.indptr.nbytes size_in_gb = size_in_bytes / (1024 ** 3) From 60ec6516d6ce643971ad084652ebc8b839a31709 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Wed, 16 Oct 2024 16:06:14 +0200 Subject: [PATCH 74/94] scaled textures to 1 million cells --- src/scirpy/ir_dist/metrics.py | 36 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 105ac95c9..eb70f862e 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -870,13 +870,13 @@ def _gpu_hamming_mat( hamming_kernel = cp.RawKernel(r''' extern "C" __global__ void hamming_kernel( - const int* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, - const int* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, - cudaTextureObject_t tex_mat1, + const int* seqs_mat1, //cudaTextureObject_t seqs_mat1, + const int* seqs_mat2, //cudaTextureObject_t seqs_mat2, + //cudaTextureObject_t tex_mat1, cudaTextureObject_t tex_mat2, - cudaTextureObject_t tex_L1, //const int* seqs_L1, + const int* seqs_L1, //cudaTextureObject_t tex_L1, // cudaTextureObject_t tex_L2, //const int* seqs_L2, - cudaTextureObject_t seqs_original_indices, //const int* seqs_original_indices, + const int* seqs_original_indices, //cudaTextureObject_t seqs_original_indices, cudaTextureObject_t seqs2_original_indices, //const int* seqs2_original_indices, const int cutoff, int* data, @@ -893,9 +893,15 @@ def _gpu_hamming_mat( ) { int row = blockDim.x * blockIdx.x + threadIdx.x; if (row < seqs_mat1_rows) { - int seqs_original_index = tex1D(seqs_original_indices, row); //seqs_original_indices[row]; - int seq1_len = tex1D(tex_L1, row);//seqs_L1[row]; + int seqs_original_index = seqs_original_indices[row]; //tex1D(seqs_original_indices, row); + int seq1_len = seqs_L1[row]; // tex1D(tex_L1, row); int row_end_index = 0; + + int seq1[80]; + for(int i = 0; i<80; i++){ + seq1[i] = seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); + } + for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { int seq2_len = tex1D(tex_L2, col); // seqs_L2[col]; @@ -917,7 +923,7 @@ def _gpu_hamming_mat( } */ - int tex_val1 = tex2D(tex_mat1, row, i); + int tex_val1 = seq1[i];//tex2D(tex_mat1, row, i); int tex_val2 = tex2D(tex_mat2, col, i); if( tex_val1 != tex_val2) { distance++; @@ -1046,11 +1052,11 @@ def create_texture2D(cupy_array) -> texture.TextureObject: cp.cuda.runtime.cudaReadModeElementType) return texture.TextureObject(res, tex) - tex_seqs_mat1 = create_texture2D(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + #tex_seqs_mat1 = create_texture2D(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) tex_seqs_mat2 = create_texture2D(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) - tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + #tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) - tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) + #tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) tex_seqs2_original_indices = create_texture1D(seqs2_original_indices_blocks) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) start_compile = time.time() @@ -1068,11 +1074,11 @@ def create_texture2D(cupy_array) -> texture.TextureObject: ( d_seqs_mat1, # tex_seqs_mat1, d_seqs_mat2, #tex_seqs_mat2, - tex_seqs_mat1, + #tex_seqs_mat1, tex_seqs_mat2, - tex_seqs_L1, # d_seqs_L1, + d_seqs_L1, #tex_seqs_L1, tex_seqs_L2, # d_seqs_L2, - tex_seqs_original_indices, #seqs_original_indices, + seqs_original_indices, # tex_seqs_original_indices, tex_seqs2_original_indices, #seqs2_original_indices, self.cutoff, d_data_matrix, @@ -1136,7 +1142,7 @@ def create_texture2D(cupy_array) -> texture.TextureObject: return res, time_taken block_width = 4096 - n_blocks = 2 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) From 6f9d6bde5c272d0a749b094fe0c6b589122165b4 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 17 Oct 2024 12:33:30 +0200 Subject: [PATCH 75/94] use char for sequences --- src/scirpy/ir_dist/metrics.py | 46 ++++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index eb70f862e..5b1cfae18 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -843,8 +843,8 @@ def _gpu_hamming_mat( print(f"***max_seq_len: {max_seq_len}") - seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=32) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=32) # @cuda.jit # def hamming_kernel( @@ -870,18 +870,18 @@ def _gpu_hamming_mat( hamming_kernel = cp.RawKernel(r''' extern "C" __global__ void hamming_kernel( - const int* seqs_mat1, //cudaTextureObject_t seqs_mat1, - const int* seqs_mat2, //cudaTextureObject_t seqs_mat2, + const char* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, + const int* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, //cudaTextureObject_t tex_mat1, cudaTextureObject_t tex_mat2, - const int* seqs_L1, //cudaTextureObject_t tex_L1, // + const int* __restrict__ seqs_L1, //cudaTextureObject_t tex_L1, // cudaTextureObject_t tex_L2, //const int* seqs_L2, - const int* seqs_original_indices, //cudaTextureObject_t seqs_original_indices, + const int* __restrict__ seqs_original_indices, //cudaTextureObject_t seqs_original_indices, cudaTextureObject_t seqs2_original_indices, //const int* seqs2_original_indices, const int cutoff, - int* data, - int* indices, - int* row_element_counts, + int* __restrict__ data, + int* __restrict__ indices, + int* __restrict__ row_element_counts, const int block_offset, const int seqs_mat1_rows, const int seqs_mat2_rows, @@ -897,8 +897,9 @@ def _gpu_hamming_mat( int seq1_len = seqs_L1[row]; // tex1D(tex_L1, row); int row_end_index = 0; - int seq1[80]; - for(int i = 0; i<80; i++){ + char seq1[32]; + #pragma unroll + for(int i = 0; i<32; i++){ seq1[i] = seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); } @@ -923,8 +924,8 @@ def _gpu_hamming_mat( } */ - int tex_val1 = seq1[i];//tex2D(tex_mat1, row, i); - int tex_val2 = tex2D(tex_mat2, col, i); + char tex_val1 = seq1[i];//tex2D(tex_mat1, row, i); + char tex_val2 = tex2D(tex_mat2, col, i); if( tex_val1 != tex_val2) { distance++; } @@ -942,7 +943,7 @@ def _gpu_hamming_mat( row_element_counts[seqs_original_index] = row_end_index; } } - ''', 'hamming_kernel') + ''', 'hamming_kernel') #, options=('--maxrregcount=256', '--ptxas-options=-v')) # @cuda.jit # def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): @@ -983,8 +984,8 @@ def _gpu_hamming_mat( def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset): # Transfer data to GPU (CuPy automatically places arrays on GPU) - d_seqs_mat1 = cp.asarray(seqs_mat1.astype(int)) - d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(int)) + d_seqs_mat1 = cp.asarray(seqs_mat1.astype(np.int8)) + d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(np.int8)) d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) @@ -1038,7 +1039,7 @@ def create_texture2D(cupy_array) -> texture.TextureObject: (width,) tex_data = cp.transpose(cupy_array) - ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, + ch = texture.ChannelFormatDescriptor(8, 0, 0, 0, cp.cuda.runtime.cudaChannelFormatKindSigned) print(f"{tex_data.shape}, {width}, {height}") @@ -1103,9 +1104,6 @@ def create_texture2D(cupy_array) -> texture.TextureObject: row_element_counts = d_row_element_counts.get() - # np.savetxt('d_data_matrix_ref.csv', d_data_matrix.get(), delimiter=',', fmt='%d') - # np.savetxt('d_indices_matrix_ref.csv', d_data_matrix.get(), delimiter=',', fmt='%d') - indptr = np.zeros(seqs_mat1.shape[0] + 1, dtype=np.int_) indptr[1:] = np.cumsum(row_element_counts) d_indptr = cp.asarray(indptr) @@ -1134,10 +1132,6 @@ def create_texture2D(cupy_array) -> texture.TextureObject: indptr = d_indptr.get() indices = d_indices.get() - # np.savetxt('data.csv', data, delimiter=',', fmt='%d') - # print("np.count_nonzero(data):", np.count_nonzero(data)) - # print("np.count_nonzero(indices):", np.count_nonzero(indices)) - # print("indptr[-1]: ", indptr[-1]) res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2_block.shape[0])) return res, time_taken @@ -1157,10 +1151,6 @@ def create_texture2D(cupy_array) -> texture.TextureObject: block_offset += seqs_mat2_blocks[i].shape[0] print("time_sum: ", time_sum) - # result_sparse = scipy.sparse.hstack(result_blocks) - # print("block indptr shape: ", [block.indptr.shape for block in result_blocks]) - # print(result_blocks[0].shape[0]) - data_blocks = [] indices_blocks = [] indptr = np.zeros(result_blocks[0].shape[0] + 1) From adbd2390d00e1730c5fc198218ca80449d1894ab Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 17 Oct 2024 15:51:15 +0200 Subject: [PATCH 76/94] shared memory used --- src/scirpy/ir_dist/metrics.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 5b1cfae18..31c8e3c65 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -401,7 +401,7 @@ def _seqs2mat( mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) L = np.zeros(len(seqs), dtype=np.int8) for si, s in enumerate(seqs): - L[si] = len(s) + L[si] = min(len(s), max_len) for aai in range(max_len): if aai >= len(s): break @@ -868,7 +868,7 @@ def _gpu_hamming_mat( # row_element_counts[row] = row_end_index hamming_kernel = cp.RawKernel(r''' - extern "C" __global__ + extern "C" __global__ __launch_bounds__(256) void hamming_kernel( const char* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, const int* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, @@ -897,16 +897,23 @@ def _gpu_hamming_mat( int seq1_len = seqs_L1[row]; // tex1D(tex_L1, row); int row_end_index = 0; + /* char seq1[32]; #pragma unroll for(int i = 0; i<32; i++){ seq1[i] = seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); } + */ + __shared__ char seq1[32*256]; + for(int i = 0; i<32; i++){ + seq1[i * 256 + threadIdx.x] = seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); + } for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { int seq2_len = tex1D(tex_L2, col); // seqs_L2[col]; int distance = 1; + if (seq1_len == seq2_len) { for (int i = 0; i < seq1_len; i++) { /* @@ -923,8 +930,7 @@ def _gpu_hamming_mat( distance++; } */ - - char tex_val1 = seq1[i];//tex2D(tex_mat1, row, i); + char tex_val1 = seq1[i * 256 + threadIdx.x];//seq1[i];//tex2D(tex_mat1, row, i); char tex_val2 = tex2D(tex_mat2, col, i); if( tex_val1 != tex_val2) { distance++; @@ -943,7 +949,7 @@ def _gpu_hamming_mat( row_element_counts[seqs_original_index] = row_end_index; } } - ''', 'hamming_kernel') #, options=('--maxrregcount=256', '--ptxas-options=-v')) + ''', 'hamming_kernel', options=('--maxrregcount=256', '--ptxas-options=-v', '-lineinfo')) # @cuda.jit # def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): @@ -1136,7 +1142,7 @@ def create_texture2D(cupy_array) -> texture.TextureObject: return res, time_taken block_width = 4096 - n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 2 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) From a39dbebfd36b2d56cf5cd27f0e7940e495d16383 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sat, 19 Oct 2024 10:33:06 +0200 Subject: [PATCH 77/94] experiments, run 1 million cells with global memory --- src/scirpy/ir_dist/metrics.py | 166 +++++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 41 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 31c8e3c65..c681735dd 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -838,14 +838,19 @@ def _gpu_hamming_mat( is_symmetric = False print("is_symmetric: ", is_symmetric) - unique_characters = "".join({char for string in (*seqs, *seqs2) for char in string}) + unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) print(f"***max_seq_len: {max_seq_len}") + print("seqs2 +++", seqs2[0:10]) + print("unique_characters +++", unique_characters) + print("seqs +++", seqs[0:10]) seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=32) seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=32) + print("seqs +++", seqs_mat1[0]) + # @cuda.jit # def hamming_kernel( # seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset @@ -867,11 +872,11 @@ def _gpu_hamming_mat( # row_end_index += 1 # row_element_counts[row] = row_end_index - hamming_kernel = cp.RawKernel(r''' + hamming_kernel = cp.RawKernel(r''' extern "C" __global__ __launch_bounds__(256) void hamming_kernel( const char* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, - const int* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, + const char* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, //cudaTextureObject_t tex_mat1, cudaTextureObject_t tex_mat2, const int* __restrict__ seqs_L1, //cudaTextureObject_t tex_L1, // @@ -896,22 +901,64 @@ def _gpu_hamming_mat( int seqs_original_index = seqs_original_indices[row]; //tex1D(seqs_original_indices, row); int seq1_len = seqs_L1[row]; // tex1D(tex_L1, row); int row_end_index = 0; + /* + int num_threads_per_block = 256; + __shared__ char seq1[32*256]; + __shared__ char seq1_2[32*256]; + const char4* seqs_mat1_pointer = reinterpret_cast(seqs_mat1); + char4* seq1_2_pointer = reinterpret_cast(seq1_2); + */ + /* + for(int i = 0; i<32/4; i++){ + seq1[(i*4+0) * 256 + threadIdx.x] = seqs_mat1[(i*4+0) * seqs_mat1_cols + row];//seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); + seq1[(i*4+1) * 256 + threadIdx.x] = seqs_mat1[(i*4+1) * seqs_mat1_cols + row]; + seq1[(i*4+2) * 256 + threadIdx.x] = seqs_mat1[(i*4+2) * seqs_mat1_cols + row]; + seq1[(i*4+3) * 256 + threadIdx.x] = seqs_mat1[(i*4+3) * seqs_mat1_cols + row]; + + }*/ /* - char seq1[32]; - #pragma unroll - for(int i = 0; i<32; i++){ - seq1[i] = seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); - } - */ - __shared__ char seq1[32*256]; - for(int i = 0; i<32; i++){ - seq1[i * 256 + threadIdx.x] = seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); - } - + for(int i = 0; i<32/4; i++){ + seq1[(i*4+0)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+0) * seqs_mat1_rows + row]; + seq1[(i*4+1)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+1) * seqs_mat1_rows + row]; + seq1[(i*4+2)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+2) * seqs_mat1_rows + row]; + seq1[(i*4+3)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+3) * seqs_mat1_rows + row]; + + + seq1[i * 4 * num_threads_per_block + threadIdx.x*4 + 0] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 0]; + seq1[i * 4 * num_threads_per_block + threadIdx.x*4 + 1] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 1]; + seq1[i * 4 * num_threads_per_block + threadIdx.x*4 + 2] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 2]; + seq1[i * 4 *num_threads_per_block + threadIdx.x*4 + 3] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 3]; + + if(threadIdx.x == 255 && row == 255 && i<8){ + printf("***index1 mat: %d\n***", i * 4 * seqs_mat1_rows + row*4 + 0); + printf("***index1: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 0); + printf("***index2: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 1); + printf("***index3: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 2); + printf("***index4: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 3); + + } + + seq1_2_pointer[i * num_threads_per_block + threadIdx.x] = seqs_mat1_pointer[i * seqs_mat1_rows + row]; + __syncthreads(); + if(threadIdx.x == 1 && row == 1 && i<8){ + printf("***value1: %d\n***", seqs_mat1[(i*4+0) * seqs_mat1_rows + row]); + printf("***value2: %d\n***", seqs_mat1[(i*4+1) * seqs_mat1_rows + row]); + printf("***value3: %d\n***", seqs_mat1[(i*4+2) * seqs_mat1_rows + row]); + printf("***value4: %d\n***", seqs_mat1[(i*4+3) * seqs_mat1_rows + row]); + printf("..\n"); + char4 input = seqs_mat1_pointer[i * 388 + row]; + printf("***values pointer: (%d, %d, %d, %d)***\n", input.x, input.y, input.z, input.w); + printf("-----\n"); + //printf("***index: %d***", i * num_threads_per_block + threadIdx.x); + //printf("***value: (%d, %d, %d, %d)***", input.x, input.y, input.z, input.w); + //printf("***row: %d, i: %d, value: %d\n***",row, i , seqs_mat1[i * seqs_mat1_rows + row]); + } + }*/ + for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { - int seq2_len = tex1D(tex_L2, col); // seqs_L2[col]; + int seq2_len = tex1Dfetch(tex_L2, col); // seqs_L2[col]; int distance = 1; if (seq1_len == seq2_len) { @@ -930,15 +977,27 @@ def _gpu_hamming_mat( distance++; } */ - char tex_val1 = seq1[i * 256 + threadIdx.x];//seq1[i];//tex2D(tex_mat1, row, i); + + /* + char tex_val1 = /seq1[i * 256 + threadIdx.x];//seq1[i];//tex2D(tex_mat1, row, i); char tex_val2 = tex2D(tex_mat2, col, i); if( tex_val1 != tex_val2) { distance++; } + */ + char tex_val1 = seqs_mat1[i*seqs_mat1_rows+row];//tex2D(tex_mat1,row,i);//tex1Dfetch(tex_mat1, i * seqs_mat1_rows + row);;//seqs_mat1[i * seqs_mat1_rows + row]; //tex2D(tex_mat1, row, i);//seq1[i * num_threads_per_block + threadIdx.x];//seq1[i];//tex2D(tex_mat1, row, i); + char tex_val2 = seqs_mat2[i*seqs_mat2_rows+col];//tex2D(tex_mat2,col,i);//tex1Dfetch(tex_mat2, col * seqs_mat1_cols + i); + /*char4 tex_val3 = seq1_2[i * num_threads_per_block + threadIdx.x]; + if(tex_val1 != tex_val3.x && tex_val1 != tex_val3.y && tex_val1 != tex_val3.z && tex_val1 != tex_val3.w){ + printf("***unequal!!**"); + }*/ + if( tex_val1 != tex_val2) { + distance++; + } } if (distance <= cutoff + 1) { - int seqs2_original_index = tex1D(seqs2_original_indices, col);//seqs2_original_indices[col]; + int seqs2_original_index = tex1Dfetch(seqs2_original_indices, col);//seqs2_original_indices[col]; data[seqs_original_index * data_cols + row_end_index] = distance; indices[seqs_original_index * indices_cols + row_end_index] = seqs2_original_index; row_end_index++; @@ -1014,35 +1073,57 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori from cupy.cuda import texture def create_texture1D(cupy_array) -> texture.TextureObject: - width, height, depth = (cupy_array.shape[0], 0, 0) - dim = 3 if depth != 0 else 2 if height != 0 else 1 + ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, + cp.cuda.runtime.cudaChannelFormatKindSigned) + + res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeLinear, arr=cupy_array, chDesc=ch) - shape = (depth, height, width) if dim == 3 else \ - (height, width) if dim == 2 else \ - (width,) + tex_desc = texture.TextureDescriptor(None, cp.cuda.runtime.cudaFilterModePoint, + cp.cuda.runtime.cudaReadModeElementType) - tex_data = cupy_array - ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, - cp.cuda.runtime.cudaChannelFormatKindSigned) + return texture.TextureObject(res, tex_desc) - print(f"{tex_data.shape}, {width}, {height}") - arr = texture.CUDAarray(ch, width) - arr.copy_from(tex_data) - res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) - address_mode = (cp.cuda.runtime.cudaAddressModeClamp, - ) - tex = texture.TextureDescriptor(address_mode, cp.cuda.runtime.cudaFilterModePoint, - cp.cuda.runtime.cudaReadModeElementType) - return texture.TextureObject(res, tex) + # width, height, depth = (cupy_array.shape[0], 0, 0) + # # dim = 3 if depth != 0 else 2 if height != 0 else 1 + + # # shape = (depth, height, width) if dim == 3 else \ + # # (height, width) if dim == 2 else \ + # # (width,) + + # # tex_data = cupy_array + # ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, + # cp.cuda.runtime.cudaChannelFormatKindSigned) + + # # print(f"{tex_data.shape}, {width}, {height}") + # # arr = texture.CUDAarray(ch, width) + + # # arr.copy_from(tex_data) + # # res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) + # res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeLinear, arr=cupy_array, chDesc=ch) + # # address_mode = (cp.cuda.runtime.cudaAddressModeClamp, + # # ) + # tex = texture.TextureDescriptor(None, cp.cuda.runtime.cudaFilterModePoint, + # cp.cuda.runtime.cudaReadModeElementType) + # return texture.TextureObject(res, tex) def create_texture2D(cupy_array) -> texture.TextureObject: + # ch = texture.ChannelFormatDescriptor(8, 0, 0, 0, + # cp.cuda.runtime.cudaChannelFormatKindSigned) + + # res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeLinear, arr=cupy_array, chDesc=ch) + + # tex_desc = texture.TextureDescriptor(None, cp.cuda.runtime.cudaFilterModePoint, + # cp.cuda.runtime.cudaReadModeElementType) + + # return texture.TextureObject(res, tex_desc) + width, height, depth = (cupy_array.shape[0], cupy_array.shape[1], 0) dim = 3 if depth != 0 else 2 if height != 0 else 1 - shape = (depth, height, width) if dim == 3 else \ - (height, width) if dim == 2 else \ - (width,) + # shape = (depth, height, width) if dim == 3 else \ + # (height, width) if dim == 2 else \ + # (width,) tex_data = cp.transpose(cupy_array) ch = texture.ChannelFormatDescriptor(8, 0, 0, 0, @@ -1064,8 +1145,11 @@ def create_texture2D(cupy_array) -> texture.TextureObject: #tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) #tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - tex_seqs2_original_indices = create_texture1D(seqs2_original_indices_blocks) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + tex_seqs2_original_indices = create_texture1D(cp.asarray(seqs2_original_indices_blocks.astype(int))) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + d_seqs_mat1_transposed = cp.transpose(d_seqs_mat1).copy() + d_seqs_mat2_transposed = cp.transpose(d_seqs_mat2).copy() + print("d_seqs_mat1_transposed.shape:", d_seqs_mat1_transposed.shape) start_compile = time.time() hamming_kernel.compile() end_compile = time.time() @@ -1079,8 +1163,8 @@ def create_texture2D(cupy_array) -> texture.TextureObject: hamming_kernel( (blocks_per_grid,), (threads_per_block,), ( - d_seqs_mat1, # tex_seqs_mat1, - d_seqs_mat2, #tex_seqs_mat2, + d_seqs_mat1_transposed, # , #d_seqs_mat1, # tex_seqs_mat1, + d_seqs_mat2_transposed, #tex_seqs_mat2, #tex_seqs_mat1, tex_seqs_mat2, d_seqs_L1, #tex_seqs_L1, @@ -1142,7 +1226,7 @@ def create_texture2D(cupy_array) -> texture.TextureObject: return res, time_taken block_width = 4096 - n_blocks = 2 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) From 972836f1eb48f5a34424ba90f79cd3158a85a8e0 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sat, 19 Oct 2024 11:35:34 +0200 Subject: [PATCH 78/94] run 1 million cells with only global memory --- src/scirpy/ir_dist/metrics.py | 115 ++++++---------------------------- 1 file changed, 18 insertions(+), 97 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index c681735dd..011201e77 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -878,11 +878,11 @@ def _gpu_hamming_mat( const char* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, const char* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, //cudaTextureObject_t tex_mat1, - cudaTextureObject_t tex_mat2, + //cudaTextureObject_t tex_mat2, const int* __restrict__ seqs_L1, //cudaTextureObject_t tex_L1, // - cudaTextureObject_t tex_L2, //const int* seqs_L2, + const int* seqs_L2,//cudaTextureObject_t tex_L2, //const int* seqs_L2, const int* __restrict__ seqs_original_indices, //cudaTextureObject_t seqs_original_indices, - cudaTextureObject_t seqs2_original_indices, //const int* seqs2_original_indices, + const int* seqs2_original_indices, //cudaTextureObject_t seqs2_original_indices, //const int* seqs2_original_indices, const int cutoff, int* __restrict__ data, int* __restrict__ indices, @@ -898,106 +898,27 @@ def _gpu_hamming_mat( ) { int row = blockDim.x * blockIdx.x + threadIdx.x; if (row < seqs_mat1_rows) { - int seqs_original_index = seqs_original_indices[row]; //tex1D(seqs_original_indices, row); - int seq1_len = seqs_L1[row]; // tex1D(tex_L1, row); + int seqs_original_index = seqs_original_indices[row]; + int seq1_len = seqs_L1[row]; int row_end_index = 0; - /* - int num_threads_per_block = 256; - __shared__ char seq1[32*256]; - __shared__ char seq1_2[32*256]; - const char4* seqs_mat1_pointer = reinterpret_cast(seqs_mat1); - char4* seq1_2_pointer = reinterpret_cast(seq1_2); - */ - /* - for(int i = 0; i<32/4; i++){ - seq1[(i*4+0) * 256 + threadIdx.x] = seqs_mat1[(i*4+0) * seqs_mat1_cols + row];//seqs_mat1[row * seqs_mat1_cols + i];//tex2D(tex_mat1, row, i); - seq1[(i*4+1) * 256 + threadIdx.x] = seqs_mat1[(i*4+1) * seqs_mat1_cols + row]; - seq1[(i*4+2) * 256 + threadIdx.x] = seqs_mat1[(i*4+2) * seqs_mat1_cols + row]; - seq1[(i*4+3) * 256 + threadIdx.x] = seqs_mat1[(i*4+3) * seqs_mat1_cols + row]; - - }*/ - - /* - for(int i = 0; i<32/4; i++){ - seq1[(i*4+0)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+0) * seqs_mat1_rows + row]; - seq1[(i*4+1)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+1) * seqs_mat1_rows + row]; - seq1[(i*4+2)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+2) * seqs_mat1_rows + row]; - seq1[(i*4+3)* num_threads_per_block + threadIdx.x] = seqs_mat1[(i*4+3) * seqs_mat1_rows + row]; - - - seq1[i * 4 * num_threads_per_block + threadIdx.x*4 + 0] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 0]; - seq1[i * 4 * num_threads_per_block + threadIdx.x*4 + 1] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 1]; - seq1[i * 4 * num_threads_per_block + threadIdx.x*4 + 2] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 2]; - seq1[i * 4 *num_threads_per_block + threadIdx.x*4 + 3] = seqs_mat1[i * 4 * seqs_mat1_rows + row*4 + 3]; - - if(threadIdx.x == 255 && row == 255 && i<8){ - printf("***index1 mat: %d\n***", i * 4 * seqs_mat1_rows + row*4 + 0); - printf("***index1: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 0); - printf("***index2: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 1); - printf("***index3: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 2); - printf("***index4: %d\n***", i * 4 * num_threads_per_block + threadIdx.x*4 + 3); - - } - - seq1_2_pointer[i * num_threads_per_block + threadIdx.x] = seqs_mat1_pointer[i * seqs_mat1_rows + row]; - __syncthreads(); - if(threadIdx.x == 1 && row == 1 && i<8){ - printf("***value1: %d\n***", seqs_mat1[(i*4+0) * seqs_mat1_rows + row]); - printf("***value2: %d\n***", seqs_mat1[(i*4+1) * seqs_mat1_rows + row]); - printf("***value3: %d\n***", seqs_mat1[(i*4+2) * seqs_mat1_rows + row]); - printf("***value4: %d\n***", seqs_mat1[(i*4+3) * seqs_mat1_rows + row]); - printf("..\n"); - char4 input = seqs_mat1_pointer[i * 388 + row]; - printf("***values pointer: (%d, %d, %d, %d)***\n", input.x, input.y, input.z, input.w); - printf("-----\n"); - //printf("***index: %d***", i * num_threads_per_block + threadIdx.x); - //printf("***value: (%d, %d, %d, %d)***", input.x, input.y, input.z, input.w); - //printf("***row: %d, i: %d, value: %d\n***",row, i , seqs_mat1[i * seqs_mat1_rows + row]); - } - }*/ for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { - int seq2_len = tex1Dfetch(tex_L2, col); // seqs_L2[col]; + int seq2_len = seqs_L2[col];//tex1Dfetch(tex_L2, col); // seqs_L2[col]; int distance = 1; if (seq1_len == seq2_len) { - for (int i = 0; i < seq1_len; i++) { - /* - int val1 = seqs_mat1[row, i];//tex2D(seqs_mat1, row, i); - int val2 = seqs_mat2[col, i];//tex2D(seqs_mat2, col, i); - if(val1 != val2) { - distance++; - } - */ - /* - int val1 = __ldg(&seqs_mat1[row * seqs_mat1_cols + i]); - int val2 = __ldg(&seqs_mat2[col * seqs_mat2_cols + i]); - if( val1 != val2) { - distance++; - } - */ - - /* - char tex_val1 = /seq1[i * 256 + threadIdx.x];//seq1[i];//tex2D(tex_mat1, row, i); - char tex_val2 = tex2D(tex_mat2, col, i); - if( tex_val1 != tex_val2) { - distance++; - } - */ - char tex_val1 = seqs_mat1[i*seqs_mat1_rows+row];//tex2D(tex_mat1,row,i);//tex1Dfetch(tex_mat1, i * seqs_mat1_rows + row);;//seqs_mat1[i * seqs_mat1_rows + row]; //tex2D(tex_mat1, row, i);//seq1[i * num_threads_per_block + threadIdx.x];//seq1[i];//tex2D(tex_mat1, row, i); - char tex_val2 = seqs_mat2[i*seqs_mat2_rows+col];//tex2D(tex_mat2,col,i);//tex1Dfetch(tex_mat2, col * seqs_mat1_cols + i); - /*char4 tex_val3 = seq1_2[i * num_threads_per_block + threadIdx.x]; - if(tex_val1 != tex_val3.x && tex_val1 != tex_val3.y && tex_val1 != tex_val3.z && tex_val1 != tex_val3.w){ - printf("***unequal!!**"); - }*/ + for (int i = 0; i < seq1_len; i++) { + char tex_val1 = seqs_mat1[i*seqs_mat1_rows+row]; + char tex_val2 = seqs_mat2[i*seqs_mat2_rows+col]; + if( tex_val1 != tex_val2) { distance++; } } if (distance <= cutoff + 1) { - int seqs2_original_index = tex1Dfetch(seqs2_original_indices, col);//seqs2_original_indices[col]; + int seqs2_original_index = seqs2_original_indices[col];//tex1Dfetch(seqs2_original_indices, col); data[seqs_original_index * data_cols + row_end_index] = distance; indices[seqs_original_index * indices_cols + row_end_index] = seqs2_original_index; row_end_index++; @@ -1008,7 +929,7 @@ def _gpu_hamming_mat( row_element_counts[seqs_original_index] = row_end_index; } } - ''', 'hamming_kernel', options=('--maxrregcount=256', '--ptxas-options=-v', '-lineinfo')) + ''', 'hamming_kernel', options=('--maxrregcount=256',))#, '--ptxas-options=-v', '-lineinfo')) # @cuda.jit # def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): @@ -1141,11 +1062,11 @@ def create_texture2D(cupy_array) -> texture.TextureObject: return texture.TextureObject(res, tex) #tex_seqs_mat1 = create_texture2D(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - tex_seqs_mat2 = create_texture2D(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + #tex_seqs_mat2 = create_texture2D(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) #tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + #tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) #tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - tex_seqs2_original_indices = create_texture1D(cp.asarray(seqs2_original_indices_blocks.astype(int))) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) + #tex_seqs2_original_indices = create_texture1D(cp.asarray(seqs2_original_indices_blocks.astype(int))) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) d_seqs_mat1_transposed = cp.transpose(d_seqs_mat1).copy() d_seqs_mat2_transposed = cp.transpose(d_seqs_mat2).copy() @@ -1166,11 +1087,11 @@ def create_texture2D(cupy_array) -> texture.TextureObject: d_seqs_mat1_transposed, # , #d_seqs_mat1, # tex_seqs_mat1, d_seqs_mat2_transposed, #tex_seqs_mat2, #tex_seqs_mat1, - tex_seqs_mat2, + #tex_seqs_mat2, d_seqs_L1, #tex_seqs_L1, - tex_seqs_L2, # d_seqs_L2, + d_seqs_L2, #tex_seqs_L2, # d_seqs_L2, seqs_original_indices, # tex_seqs_original_indices, - tex_seqs2_original_indices, #seqs2_original_indices, + seqs2_original_indices_blocks, #tex_seqs2_original_indices, #seqs2_original_indices, self.cutoff, d_data_matrix, d_indices_matrix, From 8d0c2e41e9b7c95179cde47540836d33f326277c Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sat, 19 Oct 2024 12:32:19 +0200 Subject: [PATCH 79/94] refactoring and time measurements --- src/scirpy/ir_dist/metrics.py | 176 +++++++--------------------------- 1 file changed, 33 insertions(+), 143 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 011201e77..4e31488ff 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -815,10 +815,7 @@ def _gpu_hamming_mat( not implemented for the GPU hamming calculator yet. """ - print("seqs unsorted: ", seqs[0:20]) start_sorting = time.time() - # seqs = np.array(sorted(seqs, key=len)) - # seqs2 = np.array(sorted(seqs2, key=len)) seqs_lengths = np.vectorize(len)(seqs) seqs_original_indices = np.argsort(seqs_lengths) @@ -830,59 +827,31 @@ def _gpu_hamming_mat( end_sorting = time.time() print("sorting time taken: ", end_sorting-start_sorting) - print("seqs sorted: ", seqs[0:20]) + start_preparation = time.time() seqs_original_indices = cp.asarray(seqs_original_indices.astype(int), dtype=cp.int_) seqs2_original_indices = cp.asarray(seqs2_original_indices.astype(int), dtype=cp.int_) is_symmetric = False - print("is_symmetric: ", is_symmetric) unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) + print(f"max_seq_len: {max_seq_len}") - print(f"***max_seq_len: {max_seq_len}") - - print("seqs2 +++", seqs2[0:10]) - print("unique_characters +++", unique_characters) - print("seqs +++", seqs[0:10]) - seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=32) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=32) - - print("seqs +++", seqs_mat1[0]) - - # @cuda.jit - # def hamming_kernel( - # seqs_mat1, seqs_mat2, seqs_L1, seqs_L2, cutoff, data, indices, row_element_counts, block_offset - # ): - # row = cuda.grid(1) - # if row < seqs_mat1.shape[0]: - # row_end_index = 0 - # seq1_len = seqs_L1[row] - # for col in range(seqs_mat2.shape[0]): - # if (not is_symmetric) or ((col + block_offset) >= row): - # seq2_len = seqs_L2[col] - # distance = 1 - # if seq1_len == seq2_len: - # for i in range(0, seq1_len): - # distance += seqs_mat1[row, i] != seqs_mat2[col, i] - # if distance <= cutoff + 1: - # data[row, row_end_index] = distance - # indices[row, row_end_index] = col - # row_end_index += 1 - # row_element_counts[row] = row_end_index + seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + end_preparation = time.time() + print("preparation time taken: ", end_preparation-start_preparation) hamming_kernel = cp.RawKernel(r''' extern "C" __global__ __launch_bounds__(256) void hamming_kernel( - const char* __restrict__ seqs_mat1, //cudaTextureObject_t seqs_mat1, - const char* __restrict__ seqs_mat2, //cudaTextureObject_t seqs_mat2, - //cudaTextureObject_t tex_mat1, - //cudaTextureObject_t tex_mat2, - const int* __restrict__ seqs_L1, //cudaTextureObject_t tex_L1, // - const int* seqs_L2,//cudaTextureObject_t tex_L2, //const int* seqs_L2, - const int* __restrict__ seqs_original_indices, //cudaTextureObject_t seqs_original_indices, - const int* seqs2_original_indices, //cudaTextureObject_t seqs2_original_indices, //const int* seqs2_original_indices, + const char* __restrict__ seqs_mat1, + const char* __restrict__ seqs_mat2, + const int* __restrict__ seqs_L1, + const int* seqs_L2, + const int* __restrict__ seqs_original_indices, + const int* seqs2_original_indices, const int cutoff, int* __restrict__ data, int* __restrict__ indices, @@ -914,8 +883,7 @@ def _gpu_hamming_mat( if( tex_val1 != tex_val2) { distance++; - } - + } } if (distance <= cutoff + 1) { int seqs2_original_index = seqs2_original_indices[col];//tex1Dfetch(seqs2_original_indices, col); @@ -931,18 +899,6 @@ def _gpu_hamming_mat( } ''', 'hamming_kernel', options=('--maxrregcount=256',))#, '--ptxas-options=-v', '-lineinfo')) - # @cuda.jit - # def create_csr_kernel(data, indices, data_matrix, indices_matrix, indptr): - # row, col = cuda.grid(2) - # if row < data_matrix.shape[0] and col < data_matrix.shape[1]: - # row_start = indptr[row] - # row_end = indptr[row + 1] - # row_end_index = row_end - row_start - # data_index = row_start + col - # if (data_index < data.shape[0]) and (col < row_end_index): - # data[data_index] = data_matrix[row, col] - # indices[data_index] = indices_matrix[row, col] - create_csr_kernel = cp.RawKernel(r''' extern "C" __global__ void create_csr_kernel( @@ -969,6 +925,7 @@ def _gpu_hamming_mat( def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset): + create_input_matrices_start = time.time() # Transfer data to GPU (CuPy automatically places arrays on GPU) d_seqs_mat1 = cp.asarray(seqs_mat1.astype(np.int8)) d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(np.int8)) @@ -991,86 +948,14 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori d_data_matrix_cols = result_len d_indices_matrix_cols = result_len - from cupy.cuda import texture - - def create_texture1D(cupy_array) -> texture.TextureObject: - ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, - cp.cuda.runtime.cudaChannelFormatKindSigned) - - res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeLinear, arr=cupy_array, chDesc=ch) - - tex_desc = texture.TextureDescriptor(None, cp.cuda.runtime.cudaFilterModePoint, - cp.cuda.runtime.cudaReadModeElementType) - - return texture.TextureObject(res, tex_desc) - - - # width, height, depth = (cupy_array.shape[0], 0, 0) - # # dim = 3 if depth != 0 else 2 if height != 0 else 1 - - # # shape = (depth, height, width) if dim == 3 else \ - # # (height, width) if dim == 2 else \ - # # (width,) - - # # tex_data = cupy_array - # ch = texture.ChannelFormatDescriptor(32, 0, 0, 0, - # cp.cuda.runtime.cudaChannelFormatKindSigned) - - # # print(f"{tex_data.shape}, {width}, {height}") - # # arr = texture.CUDAarray(ch, width) - - # # arr.copy_from(tex_data) - # # res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) - # res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeLinear, arr=cupy_array, chDesc=ch) - # # address_mode = (cp.cuda.runtime.cudaAddressModeClamp, - # # ) - # tex = texture.TextureDescriptor(None, cp.cuda.runtime.cudaFilterModePoint, - # cp.cuda.runtime.cudaReadModeElementType) - # return texture.TextureObject(res, tex) - - def create_texture2D(cupy_array) -> texture.TextureObject: - # ch = texture.ChannelFormatDescriptor(8, 0, 0, 0, - # cp.cuda.runtime.cudaChannelFormatKindSigned) - - # res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeLinear, arr=cupy_array, chDesc=ch) - - # tex_desc = texture.TextureDescriptor(None, cp.cuda.runtime.cudaFilterModePoint, - # cp.cuda.runtime.cudaReadModeElementType) - - # return texture.TextureObject(res, tex_desc) - - width, height, depth = (cupy_array.shape[0], cupy_array.shape[1], 0) - dim = 3 if depth != 0 else 2 if height != 0 else 1 - - # shape = (depth, height, width) if dim == 3 else \ - # (height, width) if dim == 2 else \ - # (width,) - - tex_data = cp.transpose(cupy_array) - ch = texture.ChannelFormatDescriptor(8, 0, 0, 0, - cp.cuda.runtime.cudaChannelFormatKindSigned) - - print(f"{tex_data.shape}, {width}, {height}") - arr = texture.CUDAarray(ch, width, height) - - arr.copy_from(tex_data) - res = texture.ResourceDescriptor(cp.cuda.runtime.cudaResourceTypeArray, cuArr=arr) - address_mode = (cp.cuda.runtime.cudaAddressModeClamp, - ) - tex = texture.TextureDescriptor(address_mode, cp.cuda.runtime.cudaFilterModePoint, - cp.cuda.runtime.cudaReadModeElementType) - return texture.TextureObject(res, tex) - - #tex_seqs_mat1 = create_texture2D(d_seqs_mat1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - #tex_seqs_mat2 = create_texture2D(d_seqs_mat2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) - #tex_seqs_L1 = create_texture1D(d_seqs_L1) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - #tex_seqs_L2 = create_texture1D(d_seqs_L2) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) - #tex_seqs_original_indices = create_texture1D(seqs_original_indices) #cp.cuda.texture.TextureObject(d_seqs_mat1, channel_desc, tex_desc) - #tex_seqs2_original_indices = create_texture1D(cp.asarray(seqs2_original_indices_blocks.astype(int))) #cp.cuda.texture.TextureObject(d_seqs_mat2, channel_desc, tex_desc) - d_seqs_mat1_transposed = cp.transpose(d_seqs_mat1).copy() d_seqs_mat2_transposed = cp.transpose(d_seqs_mat2).copy() - print("d_seqs_mat1_transposed.shape:", d_seqs_mat1_transposed.shape) + + cp.cuda.Device().synchronize() + create_input_matrices_end = time.time() + print("sorting time taken: ", create_input_matrices_end-create_input_matrices_start) + + start_compile = time.time() hamming_kernel.compile() end_compile = time.time() @@ -1084,14 +969,12 @@ def create_texture2D(cupy_array) -> texture.TextureObject: hamming_kernel( (blocks_per_grid,), (threads_per_block,), ( - d_seqs_mat1_transposed, # , #d_seqs_mat1, # tex_seqs_mat1, - d_seqs_mat2_transposed, #tex_seqs_mat2, - #tex_seqs_mat1, - #tex_seqs_mat2, - d_seqs_L1, #tex_seqs_L1, - d_seqs_L2, #tex_seqs_L2, # d_seqs_L2, - seqs_original_indices, # tex_seqs_original_indices, - seqs2_original_indices_blocks, #tex_seqs2_original_indices, #seqs2_original_indices, + d_seqs_mat1_transposed, + d_seqs_mat2_transposed, + d_seqs_L1, + d_seqs_L2, + seqs_original_indices, + seqs2_original_indices_blocks, self.cutoff, d_data_matrix, d_indices_matrix, @@ -1113,6 +996,8 @@ def create_texture2D(cupy_array) -> texture.TextureObject: time_taken = (end_kernel-start_kernel) print("hamming kernel time taken: ", time_taken) + start_create_csr = time.time() + row_element_counts = d_row_element_counts.get() indptr = np.zeros(seqs_mat1.shape[0] + 1, dtype=np.int_) @@ -1144,6 +1029,11 @@ def create_texture2D(cupy_array) -> texture.TextureObject: indices = d_indices.get() res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2_block.shape[0])) + cp.cuda.Device().synchronize() + + end_create_csr = time.time() + print("end_create_csr time taken: ", end_create_csr-start_create_csr) + return res, time_taken block_width = 4096 From 6bc496cd775a9695bc560d3e5e2fff865b6e2c87 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Sat, 19 Oct 2024 13:53:07 +0200 Subject: [PATCH 80/94] optimized seqs2mat --- src/scirpy/ir_dist/metrics.py | 44 ++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 4e31488ff..e44527d60 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -815,6 +815,8 @@ def _gpu_hamming_mat( not implemented for the GPU hamming calculator yet. """ + start_gpu_hamming_mat = time.time() + start_sorting = time.time() seqs_lengths = np.vectorize(len)(seqs) @@ -829,17 +831,33 @@ def _gpu_hamming_mat( print("sorting time taken: ", end_sorting-start_sorting) start_preparation = time.time() - seqs_original_indices = cp.asarray(seqs_original_indices.astype(int), dtype=cp.int_) - seqs2_original_indices = cp.asarray(seqs2_original_indices.astype(int), dtype=cp.int_) + + seqs_original_indices = cp.asarray(seqs_original_indices, dtype=cp.int_) + seqs2_original_indices = cp.asarray(seqs2_original_indices, dtype=cp.int_) is_symmetric = False - unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) + #unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) print(f"max_seq_len: {max_seq_len}") - seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) - seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None + ) -> tuple[np.ndarray, np.ndarray]: + if max_len is None: + max_len = np.max([len(s) for s in seqs]) + mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) + L = np.zeros(len(seqs), dtype=np.int8) + for i, seq in enumerate(seqs): + mat[i][0:len(seq)] = np.frombuffer(seq.encode('ascii'), dtype=np.uint8) + L[i] = len(seq) + return mat, L + + # seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) + # seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) + + seqs_mat1, seqs_L1 = _seqs2mat_fast(seqs, max_len=max_seq_len) + seqs_mat2, seqs_L2 = _seqs2mat_fast(seqs2, max_len=max_seq_len) + end_preparation = time.time() print("preparation time taken: ", end_preparation-start_preparation) @@ -953,7 +971,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori cp.cuda.Device().synchronize() create_input_matrices_end = time.time() - print("sorting time taken: ", create_input_matrices_end-create_input_matrices_start) + print("create_input_matrices time taken: ", create_input_matrices_end-create_input_matrices_start) start_compile = time.time() @@ -1036,6 +1054,8 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori return res, time_taken + start_split_blocks = time.time() + block_width = 4096 n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 @@ -1046,12 +1066,18 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori block_offset = start_column time_sum = 0 + + end_split_blocks = time.time() + print("split_blocks time taken: ", end_split_blocks-start_split_blocks) + for i in range(0, n_blocks): result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], seqs2_original_indices_blocks[i], block_offset) time_sum += time_taken block_offset += seqs_mat2_blocks[i].shape[0] print("time_sum: ", time_sum) + start_stack_matrix = time.time() + data_blocks = [] indices_blocks = [] indptr = np.zeros(result_blocks[0].shape[0] + 1) @@ -1081,6 +1107,12 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori print("max row element count: ", np.max(row_element_counts_gpu)) + end_stack_matrix = time.time() + print("stack matrix time taken: ", end_stack_matrix-start_stack_matrix) + + end_gpu_hamming_mat = time.time() + print("gpu_hamming_mat time taken: ", end_gpu_hamming_mat-start_gpu_hamming_mat) + return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) _metric_mat = _gpu_hamming_mat From 2d4756f1133ca91a8a7877b974131c579ba5c943 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 21 Oct 2024 20:25:02 +0200 Subject: [PATCH 81/94] increased result matrix stacking speed --- src/scirpy/ir_dist/metrics.py | 62 ++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index e44527d60..37c5fad3c 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -950,7 +950,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) - result_len = 200 + result_len = 1100 # Create output arrays (on GPU) using CuPy d_data_matrix = cp.zeros((seqs_mat1.shape[0], result_len), dtype=cp.int_) @@ -1025,6 +1025,10 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori n_elements = indptr[-1] print("***n_elements: ", n_elements) + row_max_len = np.max(row_element_counts) + print("***row max len of block: ", row_max_len) + assert row_max_len<=result_len, f""""ERROR: The chosen result block width is too small to hold all result values of the current block. + Chosen width: {result_len}, Necessary width: {row_max_len}""" data = np.zeros(n_elements, dtype=np.int_) d_data = cp.zeros_like(data) @@ -1046,7 +1050,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori indptr = d_indptr.get() indices = d_indices.get() - res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2_block.shape[0])) + res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0]))#, seqs_mat2_block.shape[0])) #, seqs_mat2.shape[0])) # cp.cuda.Device().synchronize() end_create_csr = time.time() @@ -1057,7 +1061,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori start_split_blocks = time.time() block_width = 4096 - n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 16 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) @@ -1078,26 +1082,38 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori start_stack_matrix = time.time() - data_blocks = [] - indices_blocks = [] - indptr = np.zeros(result_blocks[0].shape[0] + 1) - for result in result_blocks: - indptr+=result.indptr - size_counter = 0 - for row in range(result_blocks[0].shape[0]): - for result in result_blocks: - start = result.indptr[row] - end = result.indptr[row+1] - data = result.data[start:end] - indices = result.indices[start:end] - data_blocks.append(data) - indices_blocks.append(indices) - size_counter += len(data) - - data = np.concatenate(data_blocks) - indices = np.concatenate(indices_blocks) - - result_sparse = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0])) + # data_blocks = [] + # indices_blocks = [] + + # start_indptr = time.time() + # indptr = sum(result.indptr for result in result_blocks) + # end_indptr = time.time() + # print("create indptr time: ", end_indptr-start_indptr) + + # num_rows = result_blocks[0].shape[0] + + # size_counter = 0 + # for row_id in range(num_rows): + # for i, result in enumerate(result_blocks): + # start = result.indptr[row_id] + # end = result.indptr[row_id+1] + # data = result.data[start:end] + # indices = result.indices[start:end] + # data_blocks.append(data) + # indices_blocks.append(indices) + # size_counter += len(data) + + # start_concat= time.time() + # data = np.concatenate(data_blocks) + # indices = np.concatenate(indices_blocks) + # end_concat = time.time() + # print("concatination time: ", end_concat-start_concat) + + # result_sparse = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0])) + + result_sparse = result_blocks[0] + for i in range(1,len(result_blocks)): + result_sparse += result_blocks[i] size_in_bytes = result_sparse.data.nbytes + result_sparse.indices.nbytes + result_sparse.indptr.nbytes size_in_gb = size_in_bytes / (1024 ** 3) From c2a290f05bf52d75b0f99a65188a9766e3869485 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 21 Oct 2024 20:44:14 +0200 Subject: [PATCH 82/94] changed data dtype to int8 --- src/scirpy/ir_dist/metrics.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 37c5fad3c..03ced0d1a 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -871,7 +871,7 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None const int* __restrict__ seqs_original_indices, const int* seqs2_original_indices, const int cutoff, - int* __restrict__ data, + char* __restrict__ data, int* __restrict__ indices, int* __restrict__ row_element_counts, const int block_offset, @@ -892,7 +892,7 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { int seq2_len = seqs_L2[col];//tex1Dfetch(tex_L2, col); // seqs_L2[col]; - int distance = 1; + char distance = 1; if (seq1_len == seq2_len) { for (int i = 0; i < seq1_len; i++) { @@ -921,7 +921,7 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None extern "C" __global__ void create_csr_kernel( int* data, int* indices, - int* data_matrix, int* indices_matrix, + char* data_matrix, int* indices_matrix, int* indptr, int data_matrix_rows, int data_matrix_cols, int data_rows, int indices_matrix_cols ) { int row = blockDim.x * blockIdx.x + threadIdx.x; @@ -953,8 +953,8 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori result_len = 1100 # Create output arrays (on GPU) using CuPy - d_data_matrix = cp.zeros((seqs_mat1.shape[0], result_len), dtype=cp.int_) - d_indices_matrix = cp.zeros((seqs_mat1.shape[0], result_len), dtype=cp.int_) + d_data_matrix = cp.empty((seqs_mat1.shape[0], result_len), dtype=cp.int8) + d_indices_matrix = cp.empty((seqs_mat1.shape[0], result_len), dtype=cp.int_) d_row_element_counts = cp.zeros(seqs_mat1.shape[0], dtype=cp.int_) # Configure the grid and block sizes @@ -1061,7 +1061,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori start_split_blocks = time.time() block_width = 4096 - n_blocks = 16 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 20 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) From 459c2a2b903bc99e5d687104ee3d68495a085e85 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Mon, 21 Oct 2024 20:48:58 +0200 Subject: [PATCH 83/94] scaled to 1 million cells --- src/scirpy/ir_dist/metrics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 03ced0d1a..56ed06fc3 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -957,6 +957,8 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori d_indices_matrix = cp.empty((seqs_mat1.shape[0], result_len), dtype=cp.int_) d_row_element_counts = cp.zeros(seqs_mat1.shape[0], dtype=cp.int_) + print("d_indices_matrix.dtype: ", d_indices_matrix.dtype) + # Configure the grid and block sizes threads_per_block = 256 blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block @@ -1061,7 +1063,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori start_split_blocks = time.time() block_width = 4096 - n_blocks = 20 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) From f54dc7ed88bada80180a9af24df8f7352b28288b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 21 Oct 2024 18:51:43 +0000 Subject: [PATCH 84/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 136 ++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 55 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 56ed06fc3..38ad95d1a 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -1,8 +1,11 @@ import abc import itertools +import time import warnings from collections.abc import Sequence +# from numba import cuda +import cupy as cp import joblib import matplotlib.pyplot as plt import numba as nb @@ -10,16 +13,11 @@ import scipy.sparse import scipy.spatial from Levenshtein import distance as levenshtein_dist -# from numba import cuda -import cupy as cp from scanpy import logging from scipy.sparse import coo_matrix, csr_matrix from scirpy.util import _doc_params, _get_usable_cpus, _parallelize_with_joblib, deprecated -import time -import math - _doc_params_parallel_distance_calculator = """\ n_jobs Number of jobs to use for the pairwise distance calculation, passed to @@ -814,9 +812,8 @@ def _gpu_hamming_mat( Always returns a numpy array containing None because the computation of the minimum distance per row is not implemented for the GPU hamming calculator yet. """ - start_gpu_hamming_mat = time.time() - + start_sorting = time.time() seqs_lengths = np.vectorize(len)(seqs) @@ -828,7 +825,7 @@ def _gpu_hamming_mat( seqs2 = seqs2[seqs2_original_indices] end_sorting = time.time() - print("sorting time taken: ", end_sorting-start_sorting) + print("sorting time taken: ", end_sorting - start_sorting) start_preparation = time.time() @@ -836,22 +833,21 @@ def _gpu_hamming_mat( seqs2_original_indices = cp.asarray(seqs2_original_indices, dtype=cp.int_) is_symmetric = False - - #unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) + + # unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) print(f"max_seq_len: {max_seq_len}") - def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None - ) -> tuple[np.ndarray, np.ndarray]: + def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None) -> tuple[np.ndarray, np.ndarray]: if max_len is None: max_len = np.max([len(s) for s in seqs]) mat = -1 * np.ones((len(seqs), max_len), dtype=np.int8) L = np.zeros(len(seqs), dtype=np.int8) for i, seq in enumerate(seqs): - mat[i][0:len(seq)] = np.frombuffer(seq.encode('ascii'), dtype=np.uint8) + mat[i][0 : len(seq)] = np.frombuffer(seq.encode("ascii"), dtype=np.uint8) L[i] = len(seq) return mat, L - + # seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) # seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) @@ -859,9 +855,10 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None seqs_mat2, seqs_L2 = _seqs2mat_fast(seqs2, max_len=max_seq_len) end_preparation = time.time() - print("preparation time taken: ", end_preparation-start_preparation) + print("preparation time taken: ", end_preparation - start_preparation) - hamming_kernel = cp.RawKernel(r''' + hamming_kernel = cp.RawKernel( + r""" extern "C" __global__ __launch_bounds__(256) void hamming_kernel( const char* __restrict__ seqs_mat1, @@ -880,28 +877,28 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None const int seqs_mat1_cols, const int seqs_mat2_cols, const int data_cols, - const int indices_cols, + const int indices_cols, const bool is_symmetric ) { int row = blockDim.x * blockIdx.x + threadIdx.x; if (row < seqs_mat1_rows) { - int seqs_original_index = seqs_original_indices[row]; + int seqs_original_index = seqs_original_indices[row]; int seq1_len = seqs_L1[row]; int row_end_index = 0; - + for (int col = 0; col < seqs_mat2_rows; col++) { if ((! is_symmetric ) || (col + block_offset) >= row) { int seq2_len = seqs_L2[col];//tex1Dfetch(tex_L2, col); // seqs_L2[col]; - char distance = 1; - + char distance = 1; + if (seq1_len == seq2_len) { - for (int i = 0; i < seq1_len; i++) { + for (int i = 0; i < seq1_len; i++) { char tex_val1 = seqs_mat1[i*seqs_mat1_rows+row]; char tex_val2 = seqs_mat2[i*seqs_mat2_rows+col]; if( tex_val1 != tex_val2) { distance++; - } + } } if (distance <= cutoff + 1) { int seqs2_original_index = seqs2_original_indices[col];//tex1Dfetch(seqs2_original_indices, col); @@ -912,12 +909,16 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None } } } - row_element_counts[seqs_original_index] = row_end_index; + row_element_counts[seqs_original_index] = row_end_index; } } - ''', 'hamming_kernel', options=('--maxrregcount=256',))#, '--ptxas-options=-v', '-lineinfo')) + """, + "hamming_kernel", + options=("--maxrregcount=256",), + ) # , '--ptxas-options=-v', '-lineinfo')) - create_csr_kernel = cp.RawKernel(r''' + create_csr_kernel = cp.RawKernel( + r""" extern "C" __global__ void create_csr_kernel( int* data, int* indices, @@ -939,17 +940,20 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None } } } - ''', 'create_csr_kernel') + """, + "create_csr_kernel", + ) - def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset): - + def calc_block_gpu( + seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset + ): create_input_matrices_start = time.time() # Transfer data to GPU (CuPy automatically places arrays on GPU) d_seqs_mat1 = cp.asarray(seqs_mat1.astype(np.int8)) d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(np.int8)) d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) - + result_len = 1100 # Create output arrays (on GPU) using CuPy @@ -970,16 +974,15 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori d_seqs_mat1_transposed = cp.transpose(d_seqs_mat1).copy() d_seqs_mat2_transposed = cp.transpose(d_seqs_mat2).copy() - + cp.cuda.Device().synchronize() create_input_matrices_end = time.time() - print("create_input_matrices time taken: ", create_input_matrices_end-create_input_matrices_start) - + print("create_input_matrices time taken: ", create_input_matrices_end - create_input_matrices_start) start_compile = time.time() hamming_kernel.compile() end_compile = time.time() - print("compile time taken: ", end_compile-start_compile) + print("compile time taken: ", end_compile - start_compile) cp.get_default_memory_pool().free_all_blocks() cp.cuda.Device().synchronize() @@ -987,11 +990,12 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori start_kernel = time.time() hamming_kernel( - (blocks_per_grid,), (threads_per_block,), + (blocks_per_grid,), + (threads_per_block,), ( d_seqs_mat1_transposed, d_seqs_mat2_transposed, - d_seqs_L1, + d_seqs_L1, d_seqs_L2, seqs_original_indices, seqs2_original_indices_blocks, @@ -1007,13 +1011,13 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori d_data_matrix_cols, d_indices_matrix_cols, is_symmetric, - ) + ), ) - + cp.cuda.Device().synchronize() end_kernel = time.time() - time_taken = (end_kernel-start_kernel) + time_taken = end_kernel - start_kernel print("hamming kernel time taken: ", time_taken) start_create_csr = time.time() @@ -1029,7 +1033,9 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori print("***n_elements: ", n_elements) row_max_len = np.max(row_element_counts) print("***row max len of block: ", row_max_len) - assert row_max_len<=result_len, f""""ERROR: The chosen result block width is too small to hold all result values of the current block. + assert ( + row_max_len <= result_len + ), f""""ERROR: The chosen result block width is too small to hold all result values of the current block. Chosen width: {result_len}, Necessary width: {row_max_len}""" data = np.zeros(n_elements, dtype=np.int_) @@ -1042,28 +1048,41 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori blocks_per_grid_x = (d_data_matrix.shape[0] + threads_per_block[0] - 1) // threads_per_block[0] blocks_per_grid_y = (d_data_matrix.shape[1] + threads_per_block[1] - 1) // threads_per_block[1] blocks_per_grid = (blocks_per_grid_x, blocks_per_grid_y) - + create_csr_kernel( - (blocks_per_grid_x, blocks_per_grid_y), threads_per_block, - (d_data, d_indices, d_data_matrix, d_indices_matrix, d_indptr, d_data_matrix.shape[0], d_data_matrix.shape[1],d_data.shape[0], d_indices_matrix.shape[1]) + (blocks_per_grid_x, blocks_per_grid_y), + threads_per_block, + ( + d_data, + d_indices, + d_data_matrix, + d_indices_matrix, + d_indptr, + d_data_matrix.shape[0], + d_data_matrix.shape[1], + d_data.shape[0], + d_indices_matrix.shape[1], + ), ) data = d_data.get() indptr = d_indptr.get() indices = d_indices.get() - res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0]))#, seqs_mat2_block.shape[0])) #, seqs_mat2.shape[0])) # + res = csr_matrix( + (data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0]) + ) # , seqs_mat2_block.shape[0])) #, seqs_mat2.shape[0])) # cp.cuda.Device().synchronize() - + end_create_csr = time.time() - print("end_create_csr time taken: ", end_create_csr-start_create_csr) + print("end_create_csr time taken: ", end_create_csr - start_create_csr) return res, time_taken start_split_blocks = time.time() block_width = 4096 - n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) @@ -1074,16 +1093,23 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori time_sum = 0 end_split_blocks = time.time() - print("split_blocks time taken: ", end_split_blocks-start_split_blocks) + print("split_blocks time taken: ", end_split_blocks - start_split_blocks) for i in range(0, n_blocks): - result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], seqs2_original_indices_blocks[i], block_offset) + result_blocks[i], time_taken = calc_block_gpu( + seqs_mat1, + seqs_mat2_blocks[i], + seqs_L1, + seqs_L2_blocks[i], + seqs2_original_indices_blocks[i], + block_offset, + ) time_sum += time_taken block_offset += seqs_mat2_blocks[i].shape[0] print("time_sum: ", time_sum) start_stack_matrix = time.time() - + # data_blocks = [] # indices_blocks = [] @@ -1114,11 +1140,11 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori # result_sparse = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0])) result_sparse = result_blocks[0] - for i in range(1,len(result_blocks)): + for i in range(1, len(result_blocks)): result_sparse += result_blocks[i] size_in_bytes = result_sparse.data.nbytes + result_sparse.indices.nbytes + result_sparse.indptr.nbytes - size_in_gb = size_in_bytes / (1024 ** 3) + size_in_gb = size_in_bytes / (1024**3) print(f"Size of the CSR matrix: {size_in_gb:.6f} GB") row_element_counts_gpu = np.diff(result_sparse.indptr) @@ -1126,10 +1152,10 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori print("max row element count: ", np.max(row_element_counts_gpu)) end_stack_matrix = time.time() - print("stack matrix time taken: ", end_stack_matrix-start_stack_matrix) + print("stack matrix time taken: ", end_stack_matrix - start_stack_matrix) end_gpu_hamming_mat = time.time() - print("gpu_hamming_mat time taken: ", end_gpu_hamming_mat-start_gpu_hamming_mat) + print("gpu_hamming_mat time taken: ", end_gpu_hamming_mat - start_gpu_hamming_mat) return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) @@ -1694,4 +1720,4 @@ def _num_different_characters(self, s1, s2, len_diff): for c in shorter: if c in longer: longer = longer.replace(c, "", 1) - return len(longer) - len_diff \ No newline at end of file + return len(longer) - len_diff From 38f1fead3e44685d186103207ae0f92e3eddab14 Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 14 Nov 2024 16:38:38 +0100 Subject: [PATCH 85/94] sort indices of result csr matrix --- src/scirpy/ir_dist/metrics.py | 5 +++++ src/scirpy/tests/test_ir_dist_metrics.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 56ed06fc3..58b8d760f 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -1123,6 +1123,11 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori row_element_counts_gpu = np.diff(result_sparse.indptr) + start_sort_indices = time.time() + result_sparse.sort_indices() + end_sort_indices = time.time() + print("sorting indices of csr matrix time taken: ", end_sort_indices-start_sort_indices) + print("max row element count: ", np.max(row_element_counts_gpu)) end_stack_matrix = time.time() diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index a2d108154..6cda3c7c6 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -756,7 +756,7 @@ def test_gpu_hamming_reference(): gpu_hamming_calculator = GPUHammingDistanceCalculator(cutoff=2) res = gpu_hamming_calculator.calc_dist_mat(seqs, seqs) - # assert np.array_equal(res.data, reference_result.data) - # assert np.array_equal(res.indices, reference_result.indices) - # assert np.array_equal(res.indptr, reference_result.indptr) + assert np.array_equal(res.data, reference_result.data) + assert np.array_equal(res.indices, reference_result.indices) + assert np.array_equal(res.indptr, reference_result.indptr) assert np.array_equal(res.todense(), reference_result.todense()) From c646651c00ace65c4a4ccbc7d50c17b8c46d0b5d Mon Sep 17 00:00:00 2001 From: felixpetschko Date: Thu, 14 Nov 2024 17:04:50 +0100 Subject: [PATCH 86/94] refactoring --- src/scirpy/ir_dist/metrics.py | 78 ++++++++++++----------------------- 1 file changed, 26 insertions(+), 52 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 58b8d760f..7989ec436 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -828,7 +828,7 @@ def _gpu_hamming_mat( seqs2 = seqs2[seqs2_original_indices] end_sorting = time.time() - print("sorting time taken: ", end_sorting-start_sorting) + print("sorting seqs by length time taken: ", end_sorting-start_sorting) start_preparation = time.time() @@ -837,7 +837,6 @@ def _gpu_hamming_mat( is_symmetric = False - #unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) max_seq_len = max(len(s) for s in (*seqs, *seqs2)) print(f"max_seq_len: {max_seq_len}") @@ -852,6 +851,8 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None L[i] = len(seq) return mat, L + # If not ascii compatible, the following can be used: + # unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) # seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) # seqs_mat2, seqs_L2 = _seqs2mat(seqs2, alphabet=unique_characters, max_len=max_seq_len) @@ -944,29 +945,27 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset): create_input_matrices_start = time.time() - # Transfer data to GPU (CuPy automatically places arrays on GPU) + d_seqs_mat1 = cp.asarray(seqs_mat1.astype(np.int8)) d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(np.int8)) d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) - result_len = 1100 + #Due to performance reasons and since we expect the result matrix to be very sparse, we + #set a maximum result width for the current block + max_block_width = 1100 - # Create output arrays (on GPU) using CuPy - d_data_matrix = cp.empty((seqs_mat1.shape[0], result_len), dtype=cp.int8) - d_indices_matrix = cp.empty((seqs_mat1.shape[0], result_len), dtype=cp.int_) + d_data_matrix = cp.empty((seqs_mat1.shape[0], max_block_width), dtype=cp.int8) + d_indices_matrix = cp.empty((seqs_mat1.shape[0], max_block_width), dtype=cp.int_) d_row_element_counts = cp.zeros(seqs_mat1.shape[0], dtype=cp.int_) - print("d_indices_matrix.dtype: ", d_indices_matrix.dtype) - - # Configure the grid and block sizes threads_per_block = 256 blocks_per_grid = (seqs_mat1.shape[0] + (threads_per_block - 1)) // threads_per_block seqs_mat1_rows, seqs_mat1_cols = seqs_mat1.shape seqs_mat2_rows, seqs_mat2_cols = seqs_mat2_block.shape - d_data_matrix_cols = result_len - d_indices_matrix_cols = result_len + d_data_matrix_cols = max_block_width + d_indices_matrix_cols = max_block_width d_seqs_mat1_transposed = cp.transpose(d_seqs_mat1).copy() d_seqs_mat2_transposed = cp.transpose(d_seqs_mat2).copy() @@ -975,12 +974,12 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori create_input_matrices_end = time.time() print("create_input_matrices time taken: ", create_input_matrices_end-create_input_matrices_start) - start_compile = time.time() hamming_kernel.compile() end_compile = time.time() print("compile time taken: ", end_compile-start_compile) + #For performance testing, we free all memory blocks and synchronize with the device cp.get_default_memory_pool().free_all_blocks() cp.cuda.Device().synchronize() @@ -1026,11 +1025,12 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori n_elements = indptr[-1] - print("***n_elements: ", n_elements) + print("n_elements: ", n_elements) row_max_len = np.max(row_element_counts) - print("***row max len of block: ", row_max_len) - assert row_max_len<=result_len, f""""ERROR: The chosen result block width is too small to hold all result values of the current block. - Chosen width: {result_len}, Necessary width: {row_max_len}""" + print("row max len of block: ", row_max_len) + + assert row_max_len<=max_block_width, f""""ERROR: The chosen result block width is too small to hold all result values of the current block. + Chosen width: {max_block_width}, Necessary width: {row_max_len}""" data = np.zeros(n_elements, dtype=np.int_) d_data = cp.zeros_like(data) @@ -1052,7 +1052,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori indptr = d_indptr.get() indices = d_indices.get() - res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0]))#, seqs_mat2_block.shape[0])) #, seqs_mat2.shape[0])) # + res = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0])) cp.cuda.Device().synchronize() end_create_csr = time.time() @@ -1062,8 +1062,10 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori start_split_blocks = time.time() + #Set the number of blocks for the calculation. A higher number can be more memory friendly, whereas + #a lower number can improve the performance. block_width = 4096 - n_blocks = 10 # seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # or use: seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) @@ -1071,47 +1073,18 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori result_blocks = [None] * n_blocks block_offset = start_column - time_sum = 0 + time_sum_blocks = 0 end_split_blocks = time.time() print("split_blocks time taken: ", end_split_blocks-start_split_blocks) for i in range(0, n_blocks): result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], seqs2_original_indices_blocks[i], block_offset) - time_sum += time_taken + time_sum_blocks += time_taken block_offset += seqs_mat2_blocks[i].shape[0] - print("time_sum: ", time_sum) + print("calculate blocks time taken: ", time_sum_blocks) start_stack_matrix = time.time() - - # data_blocks = [] - # indices_blocks = [] - - # start_indptr = time.time() - # indptr = sum(result.indptr for result in result_blocks) - # end_indptr = time.time() - # print("create indptr time: ", end_indptr-start_indptr) - - # num_rows = result_blocks[0].shape[0] - - # size_counter = 0 - # for row_id in range(num_rows): - # for i, result in enumerate(result_blocks): - # start = result.indptr[row_id] - # end = result.indptr[row_id+1] - # data = result.data[start:end] - # indices = result.indices[start:end] - # data_blocks.append(data) - # indices_blocks.append(indices) - # size_counter += len(data) - - # start_concat= time.time() - # data = np.concatenate(data_blocks) - # indices = np.concatenate(indices_blocks) - # end_concat = time.time() - # print("concatination time: ", end_concat-start_concat) - - # result_sparse = csr_matrix((data, indices, indptr), shape=(seqs_mat1.shape[0], seqs_mat2.shape[0])) result_sparse = result_blocks[0] for i in range(1,len(result_blocks)): @@ -1128,7 +1101,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori end_sort_indices = time.time() print("sorting indices of csr matrix time taken: ", end_sort_indices-start_sort_indices) - print("max row element count: ", np.max(row_element_counts_gpu)) + print("final result max row element count: ", np.max(row_element_counts_gpu)) end_stack_matrix = time.time() print("stack matrix time taken: ", end_stack_matrix-start_stack_matrix) @@ -1136,6 +1109,7 @@ def calc_block_gpu(seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_ori end_gpu_hamming_mat = time.time() print("gpu_hamming_mat time taken: ", end_gpu_hamming_mat-start_gpu_hamming_mat) + #Returns the results in a way that fits the current interface, could be improved later return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) _metric_mat = _gpu_hamming_mat From f6668c495e891ff7c5bda7e5af69d1c34976ac9d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:54:13 +0000 Subject: [PATCH 87/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 39 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 5faf7ded2..1c9eea939 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -825,7 +825,7 @@ def _gpu_hamming_mat( seqs2 = seqs2[seqs2_original_indices] end_sorting = time.time() - print("sorting seqs by length time taken: ", end_sorting-start_sorting) + print("sorting seqs by length time taken: ", end_sorting - start_sorting) start_preparation = time.time() @@ -833,7 +833,7 @@ def _gpu_hamming_mat( seqs2_original_indices = cp.asarray(seqs2_original_indices, dtype=cp.int_) is_symmetric = False - + max_seq_len = max(len(s) for s in (*seqs, *seqs2)) print(f"max_seq_len: {max_seq_len}") @@ -846,7 +846,7 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None) -> tuple[np. mat[i][0 : len(seq)] = np.frombuffer(seq.encode("ascii"), dtype=np.uint8) L[i] = len(seq) return mat, L - + # If not ascii compatible, the following can be used: # unique_characters = "".join(sorted({char for string in (*seqs, *seqs2) for char in string})) # seqs_mat1, seqs_L1 = _seqs2mat(seqs, alphabet=unique_characters, max_len=max_seq_len) @@ -954,9 +954,9 @@ def calc_block_gpu( d_seqs_mat2 = cp.asarray(seqs_mat2_block.astype(np.int8)) d_seqs_L1 = cp.asarray(seqs_L1_block.astype(int)) d_seqs_L2 = cp.asarray(seqs_L2.astype(int)) - - #Due to performance reasons and since we expect the result matrix to be very sparse, we - #set a maximum result width for the current block + + # Due to performance reasons and since we expect the result matrix to be very sparse, we + # set a maximum result width for the current block max_block_width = 1100 d_data_matrix = cp.empty((seqs_mat1.shape[0], max_block_width), dtype=cp.int8) @@ -976,14 +976,14 @@ def calc_block_gpu( cp.cuda.Device().synchronize() create_input_matrices_end = time.time() - print("create_input_matrices time taken: ", create_input_matrices_end-create_input_matrices_start) + print("create_input_matrices time taken: ", create_input_matrices_end - create_input_matrices_start) start_compile = time.time() hamming_kernel.compile() end_compile = time.time() print("compile time taken: ", end_compile - start_compile) - #For performance testing, we free all memory blocks and synchronize with the device + # For performance testing, we free all memory blocks and synchronize with the device cp.get_default_memory_pool().free_all_blocks() cp.cuda.Device().synchronize() @@ -1034,7 +1034,9 @@ def calc_block_gpu( row_max_len = np.max(row_element_counts) print("row max len of block: ", row_max_len) - assert row_max_len<=max_block_width, f""""ERROR: The chosen result block width is too small to hold all result values of the current block. + assert ( + row_max_len <= max_block_width + ), f""""ERROR: The chosen result block width is too small to hold all result values of the current block. Chosen width: {max_block_width}, Necessary width: {row_max_len}""" data = np.zeros(n_elements, dtype=np.int_) @@ -1078,10 +1080,10 @@ def calc_block_gpu( start_split_blocks = time.time() - #Set the number of blocks for the calculation. A higher number can be more memory friendly, whereas - #a lower number can improve the performance. + # Set the number of blocks for the calculation. A higher number can be more memory friendly, whereas + # a lower number can improve the performance. block_width = 4096 - n_blocks = 10 # or use: seqs_mat2.shape[0] // block_width + 1 + n_blocks = 10 # or use: seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) seqs_L2_blocks = np.array_split(seqs_L2, n_blocks) @@ -1095,7 +1097,14 @@ def calc_block_gpu( print("split_blocks time taken: ", end_split_blocks - start_split_blocks) for i in range(0, n_blocks): - result_blocks[i], time_taken = calc_block_gpu(seqs_mat1, seqs_mat2_blocks[i], seqs_L1, seqs_L2_blocks[i], seqs2_original_indices_blocks[i], block_offset) + result_blocks[i], time_taken = calc_block_gpu( + seqs_mat1, + seqs_mat2_blocks[i], + seqs_L1, + seqs_L2_blocks[i], + seqs2_original_indices_blocks[i], + block_offset, + ) time_sum_blocks += time_taken block_offset += seqs_mat2_blocks[i].shape[0] print("calculate blocks time taken: ", time_sum_blocks) @@ -1115,7 +1124,7 @@ def calc_block_gpu( start_sort_indices = time.time() result_sparse.sort_indices() end_sort_indices = time.time() - print("sorting indices of csr matrix time taken: ", end_sort_indices-start_sort_indices) + print("sorting indices of csr matrix time taken: ", end_sort_indices - start_sort_indices) print("final result max row element count: ", np.max(row_element_counts_gpu)) @@ -1125,7 +1134,7 @@ def calc_block_gpu( end_gpu_hamming_mat = time.time() print("gpu_hamming_mat time taken: ", end_gpu_hamming_mat - start_gpu_hamming_mat) - #Returns the results in a way that fits the current interface, could be improved later + # Returns the results in a way that fits the current interface, could be improved later return [result_sparse.data], [result_sparse.indices], row_element_counts_gpu, np.array([None]) _metric_mat = _gpu_hamming_mat From f7a4a03d162bdbc57b3395105a41db2356610f2e Mon Sep 17 00:00:00 2001 From: Intron7 Date: Thu, 5 Dec 2024 15:54:04 +0100 Subject: [PATCH 88/94] remove test from ci --- .github/workflows/test.yaml | 2 +- pyproject.toml | 3 ++- src/scirpy/tests/test_ir_dist_metrics.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 76ee1c6b8..dd1dcdecc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -59,7 +59,7 @@ jobs: PLATFORM: ${{ matrix.os }} DISPLAY: :42 run: | - coverage run -m pytest -v --color=yes + coverage run -m pytest -v --color=yes -m "not gpu" - name: Report coverage run: | coverage report diff --git a/pyproject.toml b/pyproject.toml index fa27fcd8a..435cf027c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,8 @@ xfail_strict = true # ] markers = [ "conda: marks a subset of tests to be ran on the Bioconda CI.", - "extra: marks tests that require extra dependencies." + "extra: marks tests that require extra dependencies.", + "gpu: mark test to run on GPU", ] minversion = 6.0 norecursedirs = [ '.*', 'build', 'dist', '*.egg', 'data', '__pycache__'] diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index 6cda3c7c6..c12d8768b 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -745,7 +745,7 @@ def test_tcrdist_histogram_not_implemented(): seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) _ = tcrdist_calculator.calc_dist_mat(seqs, seqs) - +@pytest.mark.gpu def test_gpu_hamming_reference(): # test hamming distance against reference implementation from . import TESTDATA From 9e021fd86c323c7141b117fcc5dd57367c287e94 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:54:21 +0000 Subject: [PATCH 89/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/tests/test_ir_dist_metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scirpy/tests/test_ir_dist_metrics.py b/src/scirpy/tests/test_ir_dist_metrics.py index c12d8768b..d6154bfc1 100644 --- a/src/scirpy/tests/test_ir_dist_metrics.py +++ b/src/scirpy/tests/test_ir_dist_metrics.py @@ -745,6 +745,7 @@ def test_tcrdist_histogram_not_implemented(): seqs = np.array(["AAAA", "AA", "AABB", "ABA"]) _ = tcrdist_calculator.calc_dist_mat(seqs, seqs) + @pytest.mark.gpu def test_gpu_hamming_reference(): # test hamming distance against reference implementation From 44d7b342e1360e9470df4d6e81478fc2666e5b41 Mon Sep 17 00:00:00 2001 From: Intron7 Date: Thu, 5 Dec 2024 16:00:52 +0100 Subject: [PATCH 90/94] move cupy import to func --- src/scirpy/ir_dist/metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 1c9eea939..4d72d2bc9 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -5,7 +5,6 @@ from collections.abc import Sequence # from numba import cuda -import cupy as cp import joblib import matplotlib.pyplot as plt import numba as nb @@ -812,6 +811,7 @@ def _gpu_hamming_mat( Always returns a numpy array containing None because the computation of the minimum distance per row is not implemented for the GPU hamming calculator yet. """ + import cupy as cp start_gpu_hamming_mat = time.time() start_sorting = time.time() @@ -948,6 +948,7 @@ def _seqs2mat_fast(seqs: Sequence[str], max_len: None | int = None) -> tuple[np. def calc_block_gpu( seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset ): + import cupy as cp create_input_matrices_start = time.time() d_seqs_mat1 = cp.asarray(seqs_mat1.astype(np.int8)) From 8315e2fcd706bd3d4657c8f61ffbb56996238523 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 5 Dec 2024 15:01:50 +0000 Subject: [PATCH 91/94] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/scirpy/ir_dist/metrics.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 4d72d2bc9..6a4288ae8 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -812,6 +812,7 @@ def _gpu_hamming_mat( not implemented for the GPU hamming calculator yet. """ import cupy as cp + start_gpu_hamming_mat = time.time() start_sorting = time.time() @@ -949,6 +950,7 @@ def calc_block_gpu( seqs_mat1, seqs_mat2_block, seqs_L1_block, seqs_L2, seqs2_original_indices_blocks, block_offset ): import cupy as cp + create_input_matrices_start = time.time() d_seqs_mat1 = cp.asarray(seqs_mat1.astype(np.int8)) From 8044b281d492a66f15ac0533a40282c80e6b5112 Mon Sep 17 00:00:00 2001 From: Intron7 Date: Thu, 5 Dec 2024 16:17:03 +0100 Subject: [PATCH 92/94] add GPU test --- .cirun.yml | 11 ++++++ .github/workflows/test-gpu.yaml | 65 +++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 4 +- pyproject.toml | 3 ++ src/scirpy/ir_dist/metrics.py | 2 +- 5 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 .cirun.yml create mode 100644 .github/workflows/test-gpu.yaml diff --git a/.cirun.yml b/.cirun.yml new file mode 100644 index 000000000..263d0faaa --- /dev/null +++ b/.cirun.yml @@ -0,0 +1,11 @@ +runners: + - name: aws-gpu-runner + cloud: aws + instance_type: g4dn.xlarge + machine_image: ami-067a4ba2816407ee9 + region: eu-north-1 + preemptible: + - true + - false + labels: + - cirun-aws-gpu diff --git a/.github/workflows/test-gpu.yaml b/.github/workflows/test-gpu.yaml new file mode 100644 index 000000000..bba741219 --- /dev/null +++ b/.github/workflows/test-gpu.yaml @@ -0,0 +1,65 @@ +name: GPU-CI + +on: + push: + branches: [main] + pull_request: + types: + - labeled + - opened + - synchronize + +# Cancel the job if new commits are pushed +# https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: flying-sheep/check@v1 + with: + success: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'run-gpu-ci') }} + test: + name: GPU Tests + needs: check + runs-on: "cirun-aws-gpu--${{ github.run_id }}" + timeout-minutes: 30 + + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Nvidia SMI sanity check + run: nvidia-smi + + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install UV + uses: hynek/setup-cached-uv@v2 + with: + cache-dependency-path: pyproject.toml + + - name: Install scirpy + run: uv pip install --system -e ".[dev,test,rpack,dandelion,diversity,parasail,cupy]" + - name: Pip list + run: pip list + + - name: Run test + run: pytest -m gpu + + - name: Remove 'run-gpu-ci' Label + if: always() + uses: actions-ecosystem/action-remove-labels@v1 + with: + labels: "run-gpu-ci" + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 147357eae..aa8c07f13 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,8 +2,8 @@ fail_fast: false default_language_version: python: python3 default_stages: - - commit - - push + - pre-commit + - pre-push minimum_pre_commit_version: 2.16.0 repos: - repo: https://github.com/pre-commit/mirrors-prettier diff --git a/pyproject.toml b/pyproject.toml index 435cf027c..fbda64b96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,9 @@ test = [ 'coverage', 'black', ] +cupy = [ + 'cupy-cuda12x', +] dandelion = [ 'sc-dandelion>=0.3.5', ] diff --git a/src/scirpy/ir_dist/metrics.py b/src/scirpy/ir_dist/metrics.py index 6a4288ae8..3903c886d 100644 --- a/src/scirpy/ir_dist/metrics.py +++ b/src/scirpy/ir_dist/metrics.py @@ -1085,7 +1085,7 @@ def calc_block_gpu( # Set the number of blocks for the calculation. A higher number can be more memory friendly, whereas # a lower number can improve the performance. - block_width = 4096 + # block_width = 4096 n_blocks = 10 # or use: seqs_mat2.shape[0] // block_width + 1 seqs_mat2_blocks = np.array_split(seqs_mat2, n_blocks) From d638cfa2cd9c6ab779264d7de5dd347c9d67c678 Mon Sep 17 00:00:00 2001 From: Intron7 Date: Thu, 5 Dec 2024 16:28:55 +0100 Subject: [PATCH 93/94] rename --- .cirun.yml => .cirun.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .cirun.yml => .cirun.yaml (100%) diff --git a/.cirun.yml b/.cirun.yaml similarity index 100% rename from .cirun.yml rename to .cirun.yaml From a644fefad355483f834a168cd2f147f0cdf3bbfc Mon Sep 17 00:00:00 2001 From: Severin Dicks <37635888+Intron7@users.noreply.github.com> Date: Fri, 6 Dec 2024 18:24:06 +0100 Subject: [PATCH 94/94] Rename .cirun.yaml to .cirun.yml --- .cirun.yaml => .cirun.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .cirun.yaml => .cirun.yml (100%) diff --git a/.cirun.yaml b/.cirun.yml similarity index 100% rename from .cirun.yaml rename to .cirun.yml