Source code for finaletoolkit.frag._delfi_merge_bins

"""
Merge uniform DELFI bins (100kb by default) into larger windows (5Mb by
default), per chromosome arm.
"""
from __future__ import annotations

from typing import NamedTuple

import numpy as np
import pandas as pd

from finaletoolkit.utils.logging import get_logger

logger = get_logger(__name__)

__all__ = ["delfi_merge_bins"]


class _BinLayout(NamedTuple):
    """Result of :func:`_resolve_bin_size`."""

    bin_size: int
    inclusive: bool
    keep: np.ndarray  # boolean mask over the input rows


def _resolve_bin_size(bins: pd.DataFrame, window_size: int) -> _BinLayout:
    """Infer and validate a uniform bin size that evenly divides ``window_size``.

    Both BED-style half-open bins (``stop - start == size``) and end-inclusive
    bins (``stop - start == size - 1`` with each bin starting at the previous
    stop + 1, as in the Cristiano et al. HiC compartment bins) are accepted.
    The last bin of a contig may be shorter than the rest (as produced by e.g.
    ``bedtools makewindows``); such bins are flagged for removal in ``keep``.

    Raises
    ------
    ValueError
        If ``window_size`` is not positive, the bins are empty or not uniform,
        the coordinate convention is mixed, or the bin size does not evenly
        divide ``window_size``.
    """
    if window_size <= 0:
        raise ValueError(f"window_size must be positive, got {window_size}.")
    if bins.shape[0] == 0:
        raise ValueError("Cannot determine bin size: no bins were provided.")

    contigs = bins["contig"].to_numpy()
    starts = bins["start"].to_numpy(dtype=np.int64)
    stops = bins["stop"].to_numpy(dtype=np.int64)
    widths = stops - starts

    # The bin with the greatest start on each contig is its last bin.
    is_last = np.zeros(bins.shape[0], dtype=bool)
    is_last[pd.Series(starts).groupby(contigs).idxmax().to_numpy()] = True

    def _describe(i: int) -> str:
        return f"{contigs[i]}:{starts[i]}-{stops[i]} (width {widths[i]})"

    # Every bin but the last of each contig must share one width. If every
    # contig has a single bin, there is nothing to compare against but each
    # other, so all bins are held to the same width.
    reference = widths[~is_last] if (~is_last).any() else widths
    width = int(reference[0])
    if width <= 0:
        raise ValueError(f"Bin {_describe(0)} has non-positive width.")
    mismatched = np.flatnonzero(
        (widths != width) & ~(is_last & (widths < width))
    )
    if mismatched.size > 0:
        examples = ", ".join(_describe(i) for i in mismatched[:3])
        raise ValueError(
            "Bins must all be the same size to merge them, but found "
            f"{mismatched.size} bin(s) whose width differs from {width}: "
            f"{examples}. Only the last bin of a contig may be shorter. Use "
            "uniform bins or pass merge_bins=False (--no-merge-bins)."
        )
    keep = widths == width

    # Infer the coordinate convention from the gap between adjacent bins.
    order = np.lexsort((starts, contigs))
    same_contig = contigs[order][1:] == contigs[order][:-1]
    gaps = (starts[order][1:] - stops[order][:-1])[same_contig]
    half_open = bool((gaps == 0).any())
    inclusive = bool((gaps == 1).any())
    if half_open and inclusive:
        raise ValueError(
            "Bins mix half-open (next start == previous stop) and "
            "end-inclusive (next start == previous stop + 1) coordinates."
        )
    bin_size = width + 1 if inclusive else width
    logger.info(
        f"Detected {bin_size} bp "
        f"{'end-inclusive' if inclusive else 'half-open'} bins "
        f"(stop - start = {width})."
    )

    dropped = np.flatnonzero(~keep)
    if dropped.size > 0:
        examples = ", ".join(_describe(i) for i in dropped[:3])
        logger.warning(
            f"Dropping {dropped.size} short bin(s) at contig ends: {examples}"
            f"{', ...' if dropped.size > 3 else ''}."
        )

    if window_size < bin_size or window_size % bin_size != 0:
        raise ValueError(
            f"Merge window size ({window_size} bp) must be a positive "
            f"multiple of the bin size ({bin_size} bp). Choose a window size "
            f"such as {max(1, window_size // bin_size) * bin_size} bp."
        )
    return _BinLayout(bin_size, inclusive, keep)


def _aggregate_chunk(
    chunk: pd.DataFrame, contig: str, arm: str, include_corrected: bool
) -> tuple:
    """Aggregate one chunk of bins into a single merged-bin record."""
    record = [
        contig,
        chunk["start"].min(),
        chunk["stop"].max(),
        arm,
        chunk["short"].sum(),
        chunk["long"].sum(),
        chunk["gc"].mean(),
        chunk["num_frags"].sum(),
        chunk["ratio"].mean(),
    ]
    if include_corrected:
        record.extend(
            [
                chunk["short_corrected"].sum(),
                chunk["long_corrected"].sum(),
                chunk["num_frags_corrected"].sum(),
                chunk["ratio_corrected"].mean(),
            ]
        )
    return tuple(record)


def _fixed_span_windows(
    arm_bins: pd.DataFrame,
    contig: str,
    arm: str,
    forward: bool,
    window_size: int,
    bin_size: int,
    include_corrected: bool,
) -> list[tuple]:
    """Aggregate one arm's bins into windows exactly ``window_size`` wide.

    Windows tile the arm from its first bin (``forward``) or back from its
    last bin, and each bin goes to the window containing its start. Windows
    keep their full span even when bins inside them were removed; windows
    left with no bins, and the partial window at the far end, are dropped.
    """
    starts = arm_bins["start"].to_numpy(dtype=np.int64)
    # End-inclusive bins have stop = start + bin_size - 1; report windows in
    # the same convention.
    stop_offset = bin_size - int(arm_bins["stop"].iloc[0] - starts[0])
    if forward:
        anchor = int(starts.min())
        index = (starts - anchor) // window_size
        n_windows = (int(starts.max()) + bin_size - anchor) // window_size
    else:
        anchor = int(starts.max()) + bin_size
        index = (anchor - (starts + bin_size)) // window_size
        n_windows = (anchor - int(starts.min())) // window_size

    windows = []
    for k in np.unique(index[index < n_windows]):
        record = list(
            _aggregate_chunk(
                arm_bins.loc[index == k], contig, arm, include_corrected
            )
        )
        window_start = (
            anchor + k * window_size if forward
            else anchor - (k + 1) * window_size
        )
        record[1] = window_start
        record[2] = window_start + window_size - stop_offset
        windows.append(tuple(record))
    windows.sort(key=lambda record: record[1])
    return windows


[docs] def delfi_merge_bins( hundred_kb_bins: pd.DataFrame, gc_corrected: bool = True, verbose: bool = False, *, window_size: int = 5000000, bin_size: int | None = None, strict_window_size: bool = False, ) -> pd.DataFrame: """Merge uniform DELFI bins into non-overlapping windows per arm. By default each window holds ``window_size // bin_size`` consecutive bins (50 x 100kb = 5Mb by default). As in the original DELFI scripts, bins are grouped by count, so a window may span more than ``window_size`` bp where bins were removed (e.g. for overlapping a gap). With ``strict_window_size=True``, windows instead span exactly ``window_size`` bp and hold however many bins remain inside them. p-arms are aggregated 5'->3'; q-arms are aggregated 3'->5' and then reversed. Partial windows at arm ends are dropped. Without gap annotations (``arm`` equal to ``contig``), each contig is aggregated 5'->3' as a whole. Parameters ---------- hundred_kb_bins : pandas.DataFrame Uniform bins with ``contig`` and ``arm`` columns and DELFI feature columns. When ``gc_corrected`` is ``True`` the ``*_corrected`` columns must be present. gc_corrected : bool, optional Whether the input carries GC-corrected columns to aggregate (default ``True``). Unlike the original implementation this flag is honored, so merging works with GC correction disabled. verbose : bool, optional Unused; kept for signature compatibility. window_size : int, optional Merged window size in bp (default 5,000,000). Must be a multiple of the bin size. bin_size : int, optional Size of the input bins in bp. If omitted, it is inferred from the ``start``/``stop`` columns, accepting half-open or end-inclusive coordinates; short bins at contig ends are dropped with a warning. strict_window_size : bool, optional Merge into windows exactly ``window_size`` bp wide instead of a fixed number of bins (default ``False``, matching the original DELFI scripts). Windows where bins were removed then aggregate fewer bins, so their summed counts are lower; windows with no remaining bins are omitted. Bins are assigned to windows by their start coordinate. Returns ------- pandas.DataFrame The merged bins, with the same columns as the input (minus any ``index`` column). Raises ------ ValueError If the bins are not uniform or the bin size does not evenly divide ``window_size``. """ if bin_size is None: layout = _resolve_bin_size(hundred_kb_bins, window_size) hundred_kb_bins = hundred_kb_bins.loc[layout.keep] bin_size = layout.bin_size elif bin_size <= 0 or window_size < bin_size or window_size % bin_size != 0: raise ValueError( f"Merge window size ({window_size} bp) must be a positive " f"multiple of the bin size ({bin_size} bp)." ) bins_per_window = window_size // bin_size merged_bins: list[tuple] = [] for arm in hundred_kb_bins["arm"].unique(): arm_bins = hundred_kb_bins[hundred_kb_bins["arm"] == arm].reset_index() # delfi labels bins with their contig when no gap file is given. whole_contig = (arm_bins["contig"] == arm).all() # Arm labels drop the "chr" prefix (e.g. "1p"); keep that output. contig = arm if whole_contig else arm[:-1] if strict_window_size and (whole_contig or "p" in arm or "q" in arm): merged_bins.extend( _fixed_span_windows( arm_bins, contig, arm, whole_contig or "p" in arm, window_size, bin_size, gc_corrected, ) ) elif whole_contig or "p" in arm: for i in range(0, arm_bins.shape[0], bins_per_window): chunk = arm_bins.loc[i : i + bins_per_window - 1, :] if chunk.shape[0] < bins_per_window: continue merged_bins.append( _aggregate_chunk(chunk, contig, arm, gc_corrected) ) elif "q" in arm: reversed_bins: list[tuple] = [] # Stop at -1 (not 0) so the chunk ending at row 0 is reachable # when bins_per_window == 1. for i in range(arm_bins.shape[0] - 1, -1, -bins_per_window): chunk = arm_bins.loc[i - (bins_per_window - 1) : i, :] if chunk.shape[0] < bins_per_window: continue reversed_bins.append( _aggregate_chunk(chunk, contig, arm, gc_corrected) ) merged_bins.extend(reversed(reversed_bins)) del arm_bins return pd.DataFrame( merged_bins, columns=hundred_kb_bins.columns[hundred_kb_bins.columns != "index"], )