Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cumsum #251

Closed
wants to merge 69 commits into from
Closed
Show file tree
Hide file tree
Changes from 43 commits
Commits
Show all changes
69 commits
Select commit Hold shift + click to select a range
d45b494
Turn on cumulative sum
Illviljan Jul 7, 2023
29747f0
Add basic test
Illviljan Jul 7, 2023
4b6e5d6
Minimal changes to get numpy_groupies running
Illviljan Jul 7, 2023
b5d19e0
Merge remote-tracking branch 'upstream/main' into cumsum
Illviljan Jul 7, 2023
228f2cb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 7, 2023
6041749
Update test_core.py
Illviljan Jul 7, 2023
b965658
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 7, 2023
78faf3f
Update core.py
Illviljan Jul 7, 2023
9715440
reduce diffs
Illviljan Jul 7, 2023
734ef95
reduce diffs
Illviljan Jul 7, 2023
423184a
Copy some reduce to accumulate
Illviljan Jul 7, 2023
d40d23d
add accumulate as reduction_type
Illviljan Jul 7, 2023
9684ac2
Update aggregations.py
Illviljan Jul 7, 2023
f41e50e
add groupby_accumulate and groupby_aggregate
Illviljan Jul 7, 2023
d32ae4c
Update test_core.py
Illviljan Jul 7, 2023
d369d50
reduction_type to kind
Illviljan Jul 7, 2023
d0c29b0
chunk=None
Illviljan Jul 7, 2023
08bf818
Trim unused paths
Illviljan Jul 8, 2023
d8aa5f2
cumulative bool check
Illviljan Jul 8, 2023
3e30f35
rename
Illviljan Jul 8, 2023
4b21e89
Add cumprod, renaming
Illviljan Jul 8, 2023
1848b02
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 8, 2023
24466de
Update core.py
Illviljan Jul 8, 2023
07645b0
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 8, 2023
82b99b3
align typing
Illviljan Jul 8, 2023
9d50792
cumprod not supported in npg from numpy
Illviljan Jul 8, 2023
1dfa1bd
Merge remote-tracking branch 'upstream/main' into cumsum
Illviljan Jul 8, 2023
69fbad0
cleanup
Illviljan Jul 8, 2023
6da99df
Update aggregations.py
Illviljan Jul 8, 2023
25d36eb
missed engine arg
Illviljan Jul 8, 2023
03d57a4
Fix typo
Illviljan Jul 8, 2023
4dc5b09
Apply suggestions from code review
Illviljan Jul 9, 2023
b1f3a40
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 9, 2023
bb3c8e7
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 10, 2023
1bcd277
trim
Illviljan Jul 10, 2023
29b6f3f
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 10, 2023
cedbc4d
Remove a bunch of not needed code
Illviljan Jul 10, 2023
44c4646
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 10, 2023
ffe5f82
Update core.py
Illviljan Jul 10, 2023
2e27ca5
More trimming
Illviljan Jul 10, 2023
37e5b95
Update core.py
Illviljan Jul 10, 2023
9a4ce23
Update core.py
Illviljan Jul 10, 2023
4d6c8a4
Update core.py
Illviljan Jul 10, 2023
c209100
Update flox/core.py
Illviljan Jul 10, 2023
0b01fa0
Apply suggestions from code review
Illviljan Jul 10, 2023
0deeb11
trim
Illviljan Jul 10, 2023
1a373cb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 10, 2023
6b4a3ae
consistent naming
Illviljan Jul 10, 2023
e2f7772
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 10, 2023
12ee35a
Update core.py
Illviljan Jul 10, 2023
fac4634
Apply suggestions from code review
Illviljan Jul 10, 2023
62f659e
Update core.py
Illviljan Jul 10, 2023
b05e7b3
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 10, 2023
7637313
Update core.py
Illviljan Jul 10, 2023
ace3b77
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 10, 2023
91e6884
Have to send a fill_value to npg
Illviljan Jul 10, 2023
92a08c4
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jul 10, 2023
a1e6af7
Add asv benchmark
Illviljan Jul 10, 2023
9406598
Update cumulate.py
Illviljan Jul 10, 2023
54d6ace
Update core.py
Illviljan Jul 11, 2023
bc1a81d
Merge remote-tracking branch 'upstream/main' into cumsum
Illviljan Jun 8, 2024
4e0d93e
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jun 8, 2024
9da1246
Update core.py
Illviljan Jun 8, 2024
1332065
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jun 8, 2024
c0e9642
Update core.py
Illviljan Jun 8, 2024
afb390b
Update core.py
Illviljan Jun 8, 2024
33d2403
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jun 8, 2024
c44d506
Update core.py
Illviljan Jun 8, 2024
5561079
Merge branch 'cumsum' of https://github.com/Illviljan/flox into cumsum
Illviljan Jun 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion flox/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@
"""Top-level module for flox ."""
from . import cache
from .aggregations import Aggregation # noqa
from .core import groupby_reduce, rechunk_for_blockwise, rechunk_for_cohorts # noqa
from .core import (
groupby_accumulate,
groupby_aggregate,
groupby_reduce,
rechunk_for_blockwise,
rechunk_for_cohorts,
) # noqa


def _get_version():
Expand Down
40 changes: 30 additions & 10 deletions flox/aggregations.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import copy
import warnings
from functools import partial
from typing import TYPE_CHECKING, Any, Callable, TypedDict
from typing import TYPE_CHECKING, Any, Callable, Literal, TypedDict

import numpy as np
import numpy_groupies as npg
Expand All @@ -14,12 +14,13 @@

if TYPE_CHECKING:
FuncTuple = tuple[Callable | str, ...]
T_Kind = Literal["reduce", "argreduce", "cumulate"]


def _is_arg_reduction(func: str | Aggregation) -> bool:
if isinstance(func, str) and func in ["argmin", "argmax", "nanargmax", "nanargmin"]:
return True
if isinstance(func, Aggregation) and func.reduction_type == "argreduce":
if isinstance(func, Aggregation) and func.kind == "argreduce":
return True
return False

Expand Down Expand Up @@ -130,7 +131,7 @@ def __init__(
final_fill_value=dtypes.NA,
dtypes=None,
final_dtype: DTypeLike | None = None,
reduction_type="reduce",
kind: T_Kind = "reduce",
):
"""
Blueprint for computing grouped aggregations.
Expand Down Expand Up @@ -159,7 +160,7 @@ def __init__(
final result.
preprocess : callable
For dask inputs only. Preprocess inputs before ``chunk`` stage.
reduction_type : {"reduce", "argreduce"}
kind : {"reduce", "argreduce"}
Type of reduction.
fill_value : number or tuple(number), optional
Value to use when a group has no members. If single value will be converted
Expand All @@ -178,7 +179,7 @@ def __init__(
# preprocess before blockwise
self.preprocess = preprocess
# Use "chunk_reduce" or "chunk_argreduce"
self.reduction_type = reduction_type
self.kind = kind
self.numpy: FuncTuple = (numpy,) if numpy else (self.name,)
# initialize blockwise reduction
self.chunk: FuncTuple = _atleast_1d(chunk)
Expand Down Expand Up @@ -222,7 +223,7 @@ def __dask_tokenize__(self):
Aggregation,
self.name,
self.preprocess,
self.reduction_type,
self.kind,
self.numpy,
self.chunk,
self.combine,
Expand Down Expand Up @@ -394,7 +395,7 @@ def _pick_second(*x):
preprocess=argreduce_preprocess,
chunk=("max", "argmax"), # order is important
combine=("max", "argmax"),
reduction_type="argreduce",
kind="argreduce",
fill_value=(dtypes.NINF, 0),
final_fill_value=-1,
finalize=_pick_second,
Expand All @@ -407,7 +408,7 @@ def _pick_second(*x):
preprocess=argreduce_preprocess,
chunk=("min", "argmin"), # order is important
combine=("min", "argmin"),
reduction_type="argreduce",
kind="argreduce",
fill_value=(dtypes.INF, 0),
final_fill_value=-1,
finalize=_pick_second,
Expand All @@ -420,7 +421,7 @@ def _pick_second(*x):
preprocess=argreduce_preprocess,
chunk=("nanmax", "nanargmax"), # order is important
combine=("max", "argmax"),
reduction_type="argreduce",
kind="argreduce",
fill_value=(dtypes.NINF, 0),
final_fill_value=-1,
finalize=_pick_second,
Expand All @@ -433,7 +434,7 @@ def _pick_second(*x):
preprocess=argreduce_preprocess,
chunk=("nanmin", "nanargmin"), # order is important
combine=("min", "argmin"),
reduction_type="argreduce",
kind="argreduce",
fill_value=(dtypes.INF, 0),
final_fill_value=-1,
finalize=_pick_second,
Expand Down Expand Up @@ -470,7 +471,23 @@ def _pick_second(*x):
# median = Aggregation("median", chunk=None, combine=None, fill_value=None)
# nanmedian = Aggregation("nanmedian", chunk=None, combine=None, fill_value=None)

# Cumulatives:
cumsum_ = Aggregation(
Illviljan marked this conversation as resolved.
Show resolved Hide resolved
"cumsum",
chunk=None,
combine="sum",
kind="cumulate",
)
cumprod_ = Aggregation(
"cumprod",
chunk=None,
combine="prod",
kind="cumulate",
)
Illviljan marked this conversation as resolved.
Show resolved Hide resolved


aggregations = {
# Reductions:
"any": any_,
"all": all_,
"count": count,
Expand All @@ -496,6 +513,9 @@ def _pick_second(*x):
"nanfirst": nanfirst,
"last": last,
"nanlast": nanlast,
# Cumulatives:
"cumsum": cumsum_,
"cumprod": cumprod_,
Illviljan marked this conversation as resolved.
Show resolved Hide resolved
}


Expand Down
Loading