CI: bump upper Python version to 3.11 (and address numerous deprecati…

…on warnings from dependencies) (#1235)

.github/workflows/readme_snippets.yml

@@ -22,6 +22,7 @@ jobs:
           python-version: 3.9
       - run: pip install -e .[tests]
       - run: pip install pytest-codeblocks pytest
+      - run: pip install "pyparsing<3.0.0"  # https://github.com/matplotlib/matplotlib/issues/25204
       - run: python -c "import pytest_codeblocks; code=pytest_codeblocks.extract_from_file('README.md'); f=open('readme.py', 'w'); f.writelines(block.code for block in code if block.syntax=='Python'); f.close()"
       - run: python -We readme.py
       - run: sed -i 's/CPU/GPU/g' readme.py

.github/workflows/tests+artifacts+pypi.yml

@@ -124,7 +124,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.8", "3.11"]
         test-suite: ["unit_tests", "smoke_tests/no_env", "smoke_tests/box", "smoke_tests/parcel", "smoke_tests/kinematic_1d", "smoke_tests/kinematic_2d", "tutorials_tests"]
         exclude:
           - test-suite: "devops_tests"
@@ -178,7 +178,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-22.04, macos-12, windows-latest]
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.8", "3.11"]
         test-suite: [ "aqueous_chemistry", "freezing", "condensation", "coagulation", "breakup", "multi-process", "isotopes"]
       fail-fast: false
     runs-on: ${{ matrix.platform }}
@@ -242,7 +242,7 @@ jobs:
           fetch-depth: 0  # https://github.com/pypa/setuptools_scm/issues/480
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - run: pip install twine build
 
@@ -262,7 +262,7 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.8", "3.10"]
+        python-version: ["3.8", "3.11"]
     runs-on: ${{ matrix.platform }}
     needs: [package]
     steps:

examples/PySDM_examples/Shipway_and_Hill_2012/settings.py

@@ -1,9 +1,9 @@
 from typing import Iterable
 
 import numpy as np
+from numdifftools import Derivative
 from scipy.integrate import solve_ivp
 from scipy.interpolate import interp1d
-from scipy.misc import derivative
 
 from PySDM import Formulae
 from PySDM.dynamics import condensation
@@ -106,9 +106,9 @@ def drhod_dz(z_above_reservoir, rhod):
             water_vapour_mixing_ratio = self.water_vapour_mixing_ratio(
                 z_above_reservoir
             )
-            d_water_vapour_mixing_ratio__dz = derivative(
-                self.water_vapour_mixing_ratio, z_above_reservoir
-            )
+            d_water_vapour_mixing_ratio__dz = Derivative(
+                self.water_vapour_mixing_ratio
+            )(z_above_reservoir)
             T = self.formulae.state_variable_triplet.T(
                 rhod[0], self.thd(z_above_reservoir)
             )

examples/PySDM_examples/deJong_Mackay_et_al_2023/fig_9.ipynb

@@ -13,8 +13,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "#### based on Fig. 10 from Straub et al. 2010 (J. Atmos. Sci. 67)  \"_Numerical Investigation of Collision-Induced Breakup of Raindrops, Part II: Parameterizations of Coalescence Efficiencies and Fragment Size Distributions_\"   \n",
-    "DOI: 10.1175/2009JAS3175.1"
+    "#### based on Fig. 10 from [Straub et al. 2010 (J. Atmos. Sci. 67)  \"_Numerical Investigation of Collision-Induced Breakup of Raindrops, Part II: Parameterizations of Coalescence Efficiencies and Fragment Size Distributions_\"](https://doi.org/10.1175/2009JAS3175.1)"
    ]
   },
   {
@@ -50,23 +49,29 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "CI = 'CI' in os.environ\n",
     "run_sims = True\n",
-    "n_sds = [2**6, 2**10, 2**14] if 'CI' not in os.environ else [32]\n",
-    "steps = [0, 7200] \n",
-    "nruns = 10 if 'CI' not in os.environ else 1\n",
+    "parameterization = 'Straub2010'\n",
+    "dt = 1 * si.s\n",
+    "n_sds = (2**6, 2**10, 2**14) if not CI else (32,)\n",
+    "steps = (0, 7200) \n",
+    "nruns = 10 if not CI else 1\n",
+    "\n",
+    "colors = {32: 'red', 2**6: 'blue', 2**10: 'orange', 2**14: 'green'}\n",
     "\n",
-    "(fig, ax) = pyplot.subplots(nrows=1, ncols=2, figsize=(10,4))\n",
-    "(fig2, ax2) = pyplot.subplots(nrows=1, ncols=2, figsize=(10,4))      \n",
+    "kwargs = {'nrows': 1, 'ncols': 2, 'figsize': (10, 4)}\n",
+    "fig, ax = pyplot.subplots(**kwargs)\n",
+    "fig2, ax2 = pyplot.subplots(**kwargs)      \n",
     "\n",
     "# Obtain data\n",
-    "parameterization = 'Straub2010'\n",
     "for n_sd in n_sds:\n",
+    "    lbl = f\"{n_sd} SDs\"\n",
     "    if run_sims:\n",
     "        t1 = time.time()\n",
-    "        run_to_steady_state(parameterization, n_sd, steps, nruns, dt=1*si.s)\n",
+    "        run_to_steady_state(parameterization, n_sd, steps, nruns, dt=dt)\n",
     "        t2 = time.time()\n",
-    "        print('ran '+parameterization+' for '+str(n_sd)+' superdroplets in '+str(t2 - t1)+' sec')\n",
-    "    data_filename = 'steadystate_' + parameterization + '_' + str(n_sd) + 'sd.pkl'\n",
+    "        print(f'ran {parameterization} for {n_sd} superdroplets in {t2 - t1} sec')\n",
+    "    data_filename = f'steadystate_{parameterization}_{n_sd}sd.pkl'\n",
     "    with open(data_filename, 'rb') as handle:\n",
     "        (x, y_ensemble, y2_ensemble, rates) = pkl.load(handle)\n",
     "    \n",
@@ -80,57 +85,61 @@
     "    y2_std = np.nanstd(y2_ensemble, axis=0)\n",
     "\n",
     "    # Plotting\n",
-    "    cmap = matplotlib.cm.get_cmap('viridis')\n",
+    "    cmap = matplotlib.colormaps['viridis']\n",
     "    dr = np.diff(x) * si.um\n",
     "    dr = np.concatenate([dr, [dr[-1]]])\n",
     "\n",
-    "    legend_entries = []\n",
-    "\n",
-    "    color = next(ax[0]._get_lines.prop_cycler)['color'] # pylint: disable=protected-access\n",
-    "    for (j, step) in enumerate(steps):  # pylint: disable=protected-access\n",
+    "    for (j, step) in enumerate(steps):\n",
+    "        idx = 1 if step != 0 else 0\n",
+    "        label = lbl if lbl not in pyplot.gca().get_legend_handles_labels()[1] else ''\n",
+    "        ax[idx].step(\n",
+    "            2*x*si.mm,\n",
+    "            y2_mean[j]/2/dr * si.mm,\n",
+    "            linestyle='-',\n",
+    "            color=colors[n_sd],\n",
+    "            label=label\n",
+    "        )\n",
     "        if step != 0:\n",
-    "            lbl = str(n_sd) + ' SDs'\n",
-    "            ax[1].step(2*x*si.mm, y2_mean[j]/2/dr * si.mm,linestyle='-',color=color,\n",
-    "                label= lbl if lbl not in pyplot.gca().get_legend_handles_labels()[1] else '')\n",
-    "            ax[1].fill_between(2*x*si.mm, y2_mean[j]/2/dr * si.mm - y2_std[j]/2/dr * si.mm, \n",
-    "                               y2_mean[j]/2/dr * si.mm + y2_std[j]/2/dr * si.mm, color=color, alpha=0.2)\n",
-    "            \n",
-    "            ax2[1].step(2*x*si.mm, y_mean[j],linestyle='-',color=color,\n",
-    "                label= lbl if lbl not in pyplot.gca().get_legend_handles_labels()[1] else '')\n",
-    "            ax2[1].fill_between(2*x*si.mm, y_mean[j] - y_std[j], \n",
-    "                    y_mean[j] + y_std[j], color=color, alpha=0.2)\n",
-    "        else:\n",
-    "            lbl = str(n_sd) + ' SDs'\n",
-    "            ax[0].step(2*x*si.mm, y2_mean[j]/2/dr * si.mm,linestyle='-',color=color,\n",
-    "                    label= lbl if lbl not in pyplot.gca().get_legend_handles_labels()[1] else '')\n",
-    "\n",
-    "            ax2[0].step(2*x*si.mm, y_mean[j],linestyle='-',color=color,\n",
-    "                label= lbl if lbl not in pyplot.gca().get_legend_handles_labels()[1] else '')\n",
+    "            ax[idx].fill_between(\n",
+    "                2*x*si.mm,\n",
+    "                y2_mean[j]/2/dr * si.mm - y2_std[j]/2/dr * si.mm, \n",
+    "                y2_mean[j]/2/dr * si.mm + y2_std[j]/2/dr * si.mm,\n",
+    "                color=colors[n_sd],\n",
+    "                alpha=0.2\n",
+    "            )            \n",
+    "        ax2[idx].step(\n",
+    "            2*x*si.mm,\n",
+    "            y_mean[j],\n",
+    "            linestyle='-',\n",
+    "            color=colors[n_sd],\n",
+    "            label=label\n",
+    "        )\n",
+    "        if step != 0:\n",
+    "            ax2[idx].fill_between(\n",
+    "                2*x*si.mm,\n",
+    "                y_mean[j] - y_std[j], \n",
+    "                y_mean[j] + y_std[j],\n",
+    "                color=colors[n_sd],\n",
+    "                alpha=0.2\n",
+    "            )\n",
     "            \n",
     "# Reference data\n",
-    "ax[0].plot(straub_x_init/si.mm, straub_y_init, color='k', linestyle='--', label='Reference')\n",
-    "ax[1].plot(straub_x/si.mm, np.power(10, straub_log_y) ,color='k',linestyle='--', label='Reference')\n",
-    "\n",
-    "ax2[0].plot(straub_x_init/si.mm, straub_dvdlnr_init, color='k', linestyle='--', label='Reference')\n",
-    "ax2[1].plot(straub_x/si.mm, straub_dvdlnr_ss / si.mm ,color='k',linestyle='--', label='Reference')\n",
-    "\n",
-    "ax[0].set_yscale(\"log\")\n",
-    "ax[1].set_yscale(\"log\")\n",
-    "ax[0].set_xlim([0.0, 7.0])\n",
-    "ax[1].set_xlim([0.0, 7.0])\n",
-    "ax[0].set_ylim([4.0, 2e4])\n",
-    "ax[1].set_ylim([4.0, 2e4])\n",
+    "kwargs = {\"label\": \"Reference\", \"linestyle\": \"--\", \"color\": \"k\"}\n",
+    "ax[0].plot(straub_x_init/si.mm, straub_y_init, **kwargs)\n",
+    "ax[1].plot(straub_x/si.mm, np.power(10, straub_log_y), **kwargs)\n",
+    "ax2[0].plot(straub_x_init/si.mm, straub_dvdlnr_init, **kwargs)\n",
+    "ax2[1].plot(straub_x/si.mm, straub_dvdlnr_ss / si.mm, **kwargs)\n",
     "\n",
-    "ax2[0].set_xscale(\"log\")\n",
-    "ax2[1].set_xscale(\"log\")\n",
-    "ax2[0].set_xlim([0.2, 8.0])\n",
-    "ax2[1].set_xlim([0.2, 8.0])\n",
-    "ax2[0].set_ylim([0.0, 2e-5])\n",
-    "ax2[1].set_ylim([0.0, 2e-5])\n",
-    "ax[0].set_xlabel(\"particle diameter (mm)\")\n",
-    "ax[1].set_xlabel(\"particle diameter (mm)\")\n",
-    "ax2[0].set_xlabel(\"particle diameter (mm)\")\n",
-    "ax2[1].set_xlabel(\"particle diameter (mm)\")\n",
+    "for _ax in (ax[0], ax[1]):\n",
+    "    _ax.set_yscale(\"log\")\n",
+    "    _ax.set_xlim([0.0, 7.0])\n",
+    "    _ax.set_ylim([4.0, 2e4])\n",
+    "for _ax in (ax2[0], ax2[1]):\n",
+    "    _ax.set_xscale(\"log\")\n",
+    "    _ax.set_xlim([0.2, 8.0])\n",
+    "    _ax.set_ylim([0.0, 2e-5])\n",
+    "for _ax in (ax[0], ax[1], ax2[0], ax2[1]):\n",
+    "    _ax.set_xlabel(\"particle diameter (mm)\")\n",
     "\n",
     "ax[0].set_ylabel(\"N(v) (m$^{-3}$ mm$^{-1})$\")\n",
     "ax2[0].set_ylabel(\"dv/dlnr\")\n",
@@ -152,7 +161,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "edjPySDM",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -166,7 +175,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.4 (default, Apr  9 2021, 09:32:38) \n[Clang 10.0.0 ]"
+   "version": "3.9.2"
   },
   "vscode": {
    "interpreter": {
@@ -175,5 +184,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 2
+ "nbformat_minor": 4
 }

examples/setup.py

@@ -26,11 +26,11 @@ def get_long_description():
         "PyMPDATA" + ">=1.0.15" if CI else "",
         "open-atmos-jupyter-utils",
         "pystrict",
-        "matplotlib" + "<3.8.0" if CI else "",  # TODO #1142
+        "matplotlib",
         "joblib",
         "ipywidgets",
         "seaborn",
-        "ghapi",
+        "numdifftools",
     ]
     + (["pyvinecopulib", "vtk"] if platform.architecture()[0] != "32bit" else []),
     extras_require={

setup.py

@@ -2,6 +2,7 @@
 the magick behind "pip install PySDM"
 """
 import os
+import platform
 
 from setuptools import find_packages, setup
 
@@ -13,6 +14,7 @@ def get_long_description():
 
 
 CI = "CI" in os.environ
+_32bit = platform.architecture()[0] == "32bit"
 
 setup(
     name="PySDM",
@@ -23,19 +25,18 @@ def get_long_description():
     install_requires=[
         "ThrustRTC==0.3.20",
         "CURandRTC" + ("==0.1.6" if CI else ">=0.1.2"),
-        "numba" + ("==0.56.4" if CI else ">=0.51.2"),
-        "numpy" + ("==1.21.6" if CI else ""),
-        "Pint" + ("==0.17" if CI else ""),
+        "numba" + ("==0.58.1" if CI and not _32bit else ">=0.51.2"),
+        "numpy" + ("==1.24.4" if CI else ""),
+        "Pint" + ("==0.21.1" if CI else ""),
         "chempy" + ("==0.8.3" if CI else ""),
-        "scipy" + ("==1.7.3" if CI else ""),
+        "scipy" + ("==1.10.1" if CI and not _32bit else ""),
         "pyevtk" + ("==1.2.0" if CI else ""),
     ],
     extras_require={
         "tests": [
-            "matplotlib" + ("==3.5.3" if CI else ""),
+            "matplotlib",
             "jupyter-core<5.0.0",
             "ipywidgets!=8.0.3",
-            "ghapi",
             "pytest",
             "pytest-timeout",
             "PyPartMC==1.0.1",

tests/smoke_tests/box/dejong_and_mackay_et_al_2023/test_fig_6.py

@@ -62,7 +62,7 @@ def test_fig_6_reduced_resolution(backend_class, plot=False):
         pyplot.step(
             data_x[lbl],
             data_y[lbl][0] * settings.rho,
-            color=matplotlib.cm.get_cmap("viridis")(i / len(EC_VALS)),
+            color=matplotlib.colormaps["viridis"](i / len(EC_VALS)),
             label=lbl if lbl not in pyplot.gca().get_legend_handles_labels()[1] else "",
         )
 

tests/smoke_tests/box/dejong_and_mackay_et_al_2023/test_fig_7.py

@@ -10,7 +10,7 @@
 from PySDM.dynamics.collisions.coalescence_efficiencies import ConstEc
 from PySDM.physics import si
 
-CMAP = matplotlib.cm.get_cmap("viridis")
+CMAP = matplotlib.colormaps["viridis"]
 N_SD = 2**12
 DT = 1 * si.s
 

tests/smoke_tests/box/dejong_and_mackay_et_al_2023/test_fig_8.py

@@ -35,7 +35,7 @@ def test_fig_8(backend_class, plot=False):
     data_x, data_y = res.x, res.y
 
     # plot
-    cmap = matplotlib.cm.get_cmap("viridis")
+    cmap = matplotlib.colormaps["viridis"]
     for j, step in enumerate(steps):
         if j == 0:
             kwargs = {"color": "k", "linestyle": "--", "label": "initial"}

tests/smoke_tests/kinematic_2d/arabas_et_al_2015/test_export.py

@@ -10,7 +10,7 @@
 from PySDM_examples.Szumowski_et_al_1998.storage import Storage
 from PySDM_examples.utils import DummyController
 from PySDM_examples.utils.widgets import IntSlider
-from scipy.io import netcdf
+from scipy.io import netcdf_file
 
 from PySDM import Formulae
 from PySDM.backends import CPU
@@ -78,9 +78,7 @@ def test_export_with_gui_settings():
         vtk_exporter.write_pvd()
 
         # Assert
-        versions = netcdf.netcdf_file(  # pylint: disable=no-member
-            file.absolute_path
-        ).versions
+        versions = netcdf_file(file.absolute_path).versions  # pylint: disable=no-member
         assert "PyMPDATA" in str(versions)
 
         filenames_list = os.listdir(os.path.join(tempdir, "output"))