Speed up CI conda solve

.github/workflows/CI_WEIS.yml

@@ -8,6 +8,11 @@ jobs:
   build:
     name: Build (${{ matrix.os }} Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 150
+
+    # Keeps output from being lost when a step is killed by a signal (exit 143).
+    env:
+      PYTHONUNBUFFERED: "1"
 
     defaults:
         run:
@@ -21,17 +26,34 @@ jobs:
 
     steps:
       - uses: actions/checkout@v5
-
+
+      # All conda packages must go in ONE solve: adding petsc4py/mpi4py/
+      # pyoptsparse to an already-solved env made conda backtrack for 80+ min.
+      # openmpi is pinned so the MPI stack doesn't vary per Python version.
+      # netcdf4 is held at 1.7.3 because 1.7.4 build 110+ is Cython 3.3.0
+      # limited-API and clashes with petsc4py under mpiexec ("Shared Cython
+      # type ... has the wrong size"). Unpin when conda-forge rebuilds them
+      # consistently, or when windIO drops its unused netCDF4 import.
+      - name: Build CI environment file
+        run: |
+            cp environment.yml environment_ci.yml
+            if [[ "$RUNNER_OS" == "Linux" ]]; then
+                printf -- '  - petsc4py\n  - mpi4py\n  - openmpi\n  - pyoptsparse\n  - libglu\n' >> environment_ci.yml
+                printf -- '  - netcdf4=1.7.3\n' >> environment_ci.yml
+            fi
+            printf -- '  - pytest\n  - pytest-cov\n  - coveralls\n  - treon\n' >> environment_ci.yml
+            cat environment_ci.yml
+
       - name: Install conda/mamba
-        uses: conda-incubator/setup-miniconda@v3
+        uses: conda-incubator/setup-miniconda@v4
         # https://github.com/marketplace/actions/setup-miniconda
         with:
           # To use mamba, uncomment here, comment out the miniforge line
           # mamba-version: "*"
           miniforge-version: "latest"
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
-          environment-file: environment.yml
+          environment-file: environment_ci.yml
           activate-environment: test
           auto-activate-base: false
 
@@ -62,24 +84,18 @@ jobs:
       #     pip install https://github.com/NLRWindSystems/WISDEM/archive/develop.zip
       # Don't delete after this
 
-      # Note- could add mpich through conda, but takes forever to resolve the environment in testing
+      # MPI, petsc4py, pyoptsparse and the test packages now come from
+      # environment_ci.yml above; only pip-only packages are left here.
       - name: Add dependencies ubuntu specific
         if: contains( matrix.os, 'ubuntu')
         run: |
-            sudo apt install -y libglu1-mesa mpich
-            conda install -y petsc4py mpi4py pyoptsparse
             pip install pygmsh==7.1.17
-            pip install https://github.com/LHEEA/meshmagick/archive/master.zip  
-            which -a mpiexec
-            which -a mpirun
-
-      - name: Add test packages
-        run: |
-            conda install -y pytest pytest-cov coveralls treon
-
+            pip install https://github.com/LHEEA/meshmagick/archive/master.zip
+
       - name: Show custom environment
         run: |
             conda list
+            which -a mpiexec mpirun || true
             printenv | sort
 
       # Install WEIS
@@ -182,7 +198,7 @@ jobs:
 
       # Run coveralls
       - name: Run coveralls
-        if: contains( matrix.os, 'ubuntu') && contains( matrix.python-version, '3.11')
+        if: contains( matrix.os, 'ubuntu') && matrix.python-version == '3.12'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         #uses: coverallsapp/github-action@v2

README.md

@@ -62,7 +62,7 @@ The installation instructions below use the environment name, "weis-env," but an
         git checkout branch_name                         # (Only if you want to switch branches, say "develop")
         conda env create --name weis-env -f environment.yml
         conda activate weis-env                          # (if this does not work, try source activate weis-env)
-        conda install -y pyoptsparse     # (Mac / Linux only, sometimes Windows users may need to install mpi4py)
+        conda install -y petsc4py mpi4py pyoptsparse netcdf4=1.7.3     # (Mac / Linux only, sometimes Windows users may need to install mpi4py)
 
 2. If you are NOT on Kestrel, install the software
 
@@ -73,6 +73,8 @@ The installation instructions below use the environment name, "weis-env," but an
         module load intel-oneapi-compilers intel-oneapi-mpi intel-oneapi-mkl
         pip install --no-deps -e . -v
 
+The conda environment includes the visualization tools. If you install WEIS with pip alone, the visualization app, MOO dashboard and input GUI need the `viz` extra: `pip install -e ".[viz]"`. Other extras are `opt` (pyOptSparse), `test` and `docs`.
+
 **NOTE:** To use WEIS again after installation is complete, you will always need to activate the conda environment first with `conda activate weis-env` (or `source activate weis-env`). On Kestrel, make sure to reload the necessary modules
 
 For Windows users, we recommend installing `git` and the `m264` packages in separate environments as some of the libraries appear to conflict such that WISDEM cannot be successfully built from source.  The `git` package is best installed in the `base` environment.

docs/how_weis_moo_viz_works.rst

@@ -24,12 +24,13 @@ The dashboard enables users to:
 Installation
 ------------
 
-The MOO Dashboard is included with WEIS. Ensure you have the required dependencies:
+The MOO Dashboard is included with WEIS, and its dependencies are part of the
+conda environment in ``environment.yml``. If you installed WEIS with pip alone,
+install the ``viz`` extra from the WEIS directory:
 
 .. code-block:: bash
 
-   conda activate weis-env
-   pip install dash plotly pandas numpy pyyaml dash-bootstrap-components
+   pip install -e ".[viz]"
 
 Running the Dashboard
 ---------------------

docs/how_weis_viz_works.rst

@@ -40,6 +40,8 @@ We created a bash script which installs all of the related libraries with a sing
 
 Please check if the installation of weis, conda virtual environment, openfast, rosco, wisdem and raft are successful by running a few example cases.
 
+The app's dependencies are part of the conda environment in ``environment.yml``. If you installed WEIS with pip alone, add them with ``pip install -e ".[viz]"`` from the WEIS directory.
+
 **The following steps (4 and 5) apply to running the visualization application from an HPC host or locally.**
 
 4. Generate visualization input yaml file

docs/installation.rst

@@ -57,15 +57,44 @@ Standard Installation (Linux, macOS, Windows)
 2. Add final packages and install the software:
    ::
 
-      conda install -y petsc4py mpi4py pyoptsparse     # (Mac / Linux only, sometimes Windows users may need to install mpi4py)
+      conda install -y petsc4py mpi4py pyoptsparse netcdf4=1.7.3     # (Mac / Linux only, sometimes Windows users may need to install mpi4py)
       pip install -e .
 
+   ``netcdf4`` is held at 1.7.3 because newer builds clash with ``petsc4py``
+   when both are imported under ``mpiexec``. Only needed if you run WEIS in
+   parallel.
+
 3. If you want to use the more advanced meshing capability for BEM modeling, install the following after you install WEIS:
    ::
 
-      pip install pygmsh==7.1.17 
+      pip install pygmsh==7.1.17
       pip install https://github.com/LHEEA/meshmagick/archive/refs/tags/3.4.zip
-      pip install trimesh                              # (Only if you want to use the internal function to plot the mesh)
+
+Optional Dependencies (pip extras)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The conda environment in ``environment.yml`` installs everything, including the
+visualization tools. A plain ``pip install`` of WEIS installs only what is needed
+to run WEIS; optional features are grouped into extras:
+
+=========  ==================================================================================
+Extra      Contents
+=========  ==================================================================================
+``viz``    Visualization app (``weis_viz``), MOO dashboard, input GUI and mesh rendering
+           (dash, dash-vtk, dearpygui, pyvista, trimesh, ...)
+``opt``    pyOptSparse optimizers
+``test``   pytest and coverage tools
+``docs``   Sphinx and extensions for building this documentation
+=========  ==================================================================================
+
+Install one or more extras with, for example:
+::
+
+   pip install -e ".[viz]"
+   pip install -e ".[viz,test]"
+
+OpenFAST itself is only available from conda-forge, so the conda environment
+above is still the recommended way to install WEIS.
 
 Installation on Kestrel (DOE HPC System)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

environment.yml

@@ -2,38 +2,45 @@ channels:
   - conda-forge
   - defaults
 
+# Keep in sync with pyproject.toml: the first block matches its dependencies
+# and the viz block matches its [viz] extra. Differences are only conda-only
+# packages (openfast, pyopenfast, pip, numpydoc for docs) and PyPI-only
+# packages (under pip: below).
 dependencies:
-  - bs4
-  - dash
-  - dash-bootstrap-components
-  - dash-mantine-components
+  - control
   - dill
   - jsonmerge
+  - marmot-agents>=0.2.5
   - mat4py
   - moorpy==1.2.1
   - nlopt
-  - numpydoc
   - numpy
+  - numpydoc
   - openfast==4.2
   - openfast-io==4.2
-  - pyopenfast==4.2
+  - openmdao
+  - openpyxl
   - openraft>=2.0.3
-  - openmdao~=3.44.0
   - osqp
   - pcrunch>=2.1.5
   - pip
   - pyhams>=1.3
-  - pyvista
+  - pyopenfast==4.2
   #- pyoptsparse
-  - pyvista
   - rosco>=2.10.1
+  - scipy
+  - smt
+  - wisdem>=4.2.7  # numpy>=2.5 thrust-shaving fix
+  # viz: visualization app, MOO dashboard, input GUI and mesh rendering
+  - beautifulsoup4
+  - dash
+  - dash-bootstrap-components
+  - dash-mantine-components
+  - pyvista
   - trimesh
-  - wisdem~=4.2.5 # tight coupling to WISDEM's OM output names; unpin when actively developing
   - pip:
-    - control
-    - dash-vtk
-    - dearpygui
-    - smt
+    - dash-vtk  # viz
+    - dearpygui  # viz
     # - git+https://github.com/NLRWindSystems/RAFT.git@dev
 # Needs to be done outside of environment file:
 #  - m2w64-toolchain # [win]

pyproject.toml

@@ -42,21 +42,27 @@ classifiers = [  # Optional
   "Programming Language :: Python :: 3 :: Only",
 ]
 
+# Keep in sync with environment.yml (openfast and pyopenfast are conda-only).
 dependencies = [
+  "control",
+  "dill",
+  "jsonmerge",
   "marmot-agents>=0.2.5",
   "mat4py",
-  "moorpy",
+  "moorpy==1.2.1",
   "nlopt",
   "numpy",
+  "openfast-io==4.2",
+  "openmdao",
   "openpyxl",
-  "openraft",
+  "openraft>=2.0.3",
   "osqp",
   "pcrunch>=2.1.5",
-  "pyhams",
+  "pyhams>=1.3",
   "rosco>=2.10.1",
   "scipy",
   "smt",
-  "wisdem>4.0.4",
+  "wisdem>=4.2.7",  # numpy>=2.5 thrust-shaving fix
 ]
 
 # List additional groups of dependencies here (e.g. development
@@ -70,7 +76,18 @@ dependencies = [
 [project.optional-dependencies] # Optional
 test = ["coveralls", "pytest", "pytest-cov"]
 opt = ["pyoptsparse"]
-docs = ["sphinx",
+# Visualization app (weis_viz), MOO dashboard, input GUI and mesh rendering
+viz = ["beautifulsoup4",
+	"dash",
+	"dash-bootstrap-components",
+	"dash-mantine-components",
+	"dash-vtk",
+	"dearpygui",
+	"pyvista",
+	"trimesh",
+]
+docs = ["numpydoc",
+	"sphinx",
 	"sphinxcontrib-bibtex",
 	"sphinx_rtd_theme>=1.3",
 	"sphinxcontrib-applehelp",

weis/inputs/gui.py

@@ -485,7 +485,7 @@ def run():
         dpg.start_dearpygui()
         dpg.destroy_context()
     else:
-        print("GUI is not available")
+        print("GUI is not available: install dearpygui, e.g. pip install weis[viz]")
 
 if __name__ == "__main__":
     run()