From a3ae414048f65883758c99745a1ffa2a0795d792 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 22 Feb 2023 09:04:59 +0100 Subject: [PATCH 1/3] Make sigma_clip_ng the default --- pyFAI/azimuthalIntegrator.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pyFAI/azimuthalIntegrator.py b/pyFAI/azimuthalIntegrator.py index 1e0fa3670..233e7039d 100644 --- a/pyFAI/azimuthalIntegrator.py +++ b/pyFAI/azimuthalIntegrator.py @@ -30,7 +30,7 @@ __contact__ = "Jerome.Kieffer@ESRF.eu" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "06/01/2023" +__date__ = "22/02/2023" __status__ = "stable" __docformat__ = 'restructuredtext' @@ -3326,10 +3326,7 @@ def sigma_clip_ng(self, data, result._set_error_model(error_model) return result - @deprecated(reason="will be replaced by `sigma_clip_ng` in version 0.23.0. Please use either `_sigma_clip_legacy` for full compatibility or upgrade your code to accomodate the new API", - replacement="sigma_clip_ng", since_version="0.21.0", only_once=True, skip_backtrace_count=1, deprecated_since="0.22.0") - def sigma_clip(self, *args, **kwargs): - return self._sigma_clip_legacy(*args, **kwargs) + sigma_clip = sigma_clip_ng def separate(self, data, npt_rad=1024, npt_azim=512, unit="2th_deg", method="splitpixel", percentile=50, mask=None, restore_mask=True): From d65528399ea26dfc7c70bb2fe6b5bd90571f92c2 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 22 Feb 2023 09:30:41 +0100 Subject: [PATCH 2/3] Make safe and other options of sigma_clip_ng available in legacy --- pyFAI/azimuthalIntegrator.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pyFAI/azimuthalIntegrator.py b/pyFAI/azimuthalIntegrator.py index 233e7039d..e3cc85262 100644 --- a/pyFAI/azimuthalIntegrator.py +++ b/pyFAI/azimuthalIntegrator.py @@ -2899,21 +2899,21 @@ def medfilt1d(self, data, npt_rad=1024, npt_azim=512, return result def sigma_clip_legacy(self, data, npt_rad=1024, npt_azim=512, - correctSolidAngle=True, polarization_factor=None, - radial_range=None, azimuth_range=None, - dark=None, flat=None, - method="splitpixel", unit=units.Q, - thres=3, max_iter=5, dummy=None, delta_dummy=None, - mask=None, normalization_factor=1.0, metadata=None): + correctSolidAngle=True, polarization_factor=None, + radial_range=None, azimuth_range=None, + dark=None, flat=None, + method=("full", "histogram", "cython"), unit=units.Q, + thres=3, max_iter=5, dummy=None, delta_dummy=None, + mask=None, normalization_factor=1.0, metadata=None, + safe=True, **kwargs): """Perform first a 2D integration and then an iterative sigma-clipping filter along each row. See the doc of scipy.stats.sigmaclip for the options `thres` and `max_iter`. :param data: input image as numpy array - :param npt_rad: number of radial points + :param npt_rad: number of radial points (alias: npt) :param npt_azim: number of azimuthal points - :param bool correctSolidAngle: correct for solid angle of each pixel - if True + :param bool correctSolidAngle: correct for solid angle of each pixel when set :param float polarization_factor: polarization factor between -1 (vertical) and +1 (horizontal). @@ -2928,17 +2928,22 @@ def sigma_clip_legacy(self, data, npt_rad=1024, npt_azim=512, :param ndarray flat: flat field image :param unit: unit to be used for integration :param method: pathway for integration and sort - :param thres: cut-off for n*sigma: discard any values with `|I-|/sigma > thres`. + :param thres: cut-off for n*sigma: discard any values with `|I-| > thres*σ`. The threshold can be a 2-tuple with sigma_low and sigma_high. :param max_iter: maximum number of iterations :param mask: masked out pixels array :param float normalization_factor: Value of a normalization monitor :param metadata: any other metadata, :type metadata: JSON serializable dict + :param safe: unset to save some checks on sparse matrix shape/content. :return: Integrate1D-like result Nota: The initial 2D-integration requires pixel splitting """ + #compatibility layer with sigma_clip_ng + if "npt" in kwargs: + npt_rad = kwargs["npt"] + # We use NaN as dummies if dummy is None: dummy = numpy.NaN @@ -2966,7 +2971,7 @@ def sigma_clip_legacy(self, data, npt_rad=1024, npt_azim=512, dummy=dummy, delta_dummy=delta_dummy, correctSolidAngle=correctSolidAngle, polarization_factor=polarization_factor, - normalization_factor=normalization_factor) + normalization_factor=normalization_factor,safe=safe) image = res2d.intensity if (method.impl_lower == "opencl"): if (method.algo_lower == "csr") and \ @@ -3079,7 +3084,7 @@ def sigma_clip_ng(self, data, Keep only pixels with intensty: - ``|I - | < thres * std(I)`` + ``|I - | < thres * σ(I)`` This enforces a symmetric, bell-shaped distibution (i.e. gaussian-like) and is very good at extracting background or amorphous isotropic scattering out of Bragg peaks. From f9a3b8ab1c8c091aa3525909f8d7a220d1ce20f5 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 22 Feb 2023 09:36:30 +0100 Subject: [PATCH 3/3] Non regression test for #1825 --- pyFAI/test/test_worker.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pyFAI/test/test_worker.py b/pyFAI/test/test_worker.py index 63e1b8fd5..e9d0bacac 100644 --- a/pyFAI/test/test_worker.py +++ b/pyFAI/test/test_worker.py @@ -32,7 +32,7 @@ __contact__ = "valentin.valls@esrf.fr" __license__ = "MIT" __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France" -__date__ = "26/11/2022" +__date__ = "22/02/2023" import unittest import logging @@ -307,8 +307,20 @@ def test_sigma_clip(self): extra_options={"thres":2, "error_model":"azimuthal"}, integrator_name="sigma_clip_ng", shapeOut=(1, 100)) + self.assertEqual(worker.error_model, "azimuthal", "error model is OK") + img = numpy.random.random(ai.detector.shape) + worker(img) + def test_sigma_clip_legacy(self): + "Non regression test for #1825" + ai = AzimuthalIntegrator.sload({"detector": "Pilatus100k", "wavelength":1e-10}) + worker = Worker(azimuthalIntegrator=ai, + extra_options={"thres":2}, + integrator_name="sigma_clip_legacy", + shapeOut=(1, 100)) + img = numpy.random.random(ai.detector.shape) + worker(img) class TestWorkerConfig(unittest.TestCase):