Reporting via jupyter

.github/workflows/python-app.yml

@@ -28,7 +28,7 @@ jobs:
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --select=E9,F63,F7,F82 --ignore=F824 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with unittest

.gitignore

@@ -140,3 +140,10 @@ cython_debug/
 .DS_Store
 .vscode/
 .idea/
+reports/exported/
+
+naturerecorder.db
+naturerecorder_dev.db
+
+clean.sh
+clean.bat

README.md

@@ -31,6 +31,9 @@ Nature Recorder is an application for recording wildlife sightings, maintaining
   - Conservation schemes, acting as containers for a set of rating values
   - Conservation status ratings, values for the conservation status within a scheme
   - Species conservation status ratings, status ratings for a species with effective start and end dates
+- Reports
+  - Simple reports built into the application
+  - Flexible reporting using Jupyter Notebooks, outside the application
 
 # Getting Started
 

docker/Dockerfile

@@ -1,11 +1,12 @@
 FROM python:3.10-slim-bullseye AS runtime
 
-COPY naturerecorderpy-1.13.0.0 /opt/naturerecorderpy
+COPY naturerecorderpy-1.14.0.0 /opt/naturerecorderpy
+
 WORKDIR /opt/naturerecorderpy
 
 RUN apt-get update -y
 RUN pip install -r requirements.txt
-RUN pip install nature_recorder-1.13.0-py3-none-any.whl
+RUN pip install nature_recorder-1.14.0-py3-none-any.whl
 
 ENV NATURE_RECORDER_DATA_FOLDER=/var/opt/naturerecorderpy
 ENV NATURE_RECORDER_DB=/var/opt/naturerecorderpy/naturerecorder.db

reports/README.md

@@ -0,0 +1,49 @@
+# Nature Recorder Reporting
+
+This folder contains Jupyter notebooks and supporting files for reporting on wildlife sightings stored in the Nature Recorder SQLite database. This provides more flexible reporting than the built-in reports in the application itself.
+
+The following reports are currently available:
+
+| Notebook | Report Type |
+| --- | --- |
+| heatmap.ipynb | Heatmap of number of sightings of each species in a category at a location during a specified year | 
+
+## Setting Up the Reporting Environment
+
+The reports have been written and tested using [Visual Studio Code](https://code.visualstudio.com/download) and the Jupyter extension from Microsoft using a Python virtual environment with the requirements listed in requirements.txt installed as the kernel for running the notebooks.
+
+### Set Environment Variables
+
+The following environment variable to be set *before* running code and opening the notebook:
+
+``` bash
+export NATURE_RECORDER_DB=/path/to/naturerecorder.db
+```
+
+Or, in PowerShell:
+
+```powershell
+$env: NATURE_RECORDER_DB = C:\path\to\naturerecorder.db
+```
+
+### Build the Virtual Environment
+
+To build the virtual environment, run the following command:
+
+```bash
+./make_venv.sh
+```
+
+Or, in PowerShell:
+
+```powershell
+.\make_venv.bat
+```
+
+## Running a Report in Visual Studio Code
+
+- Open the Jupyter notebook for the report of interest
+- If using Visual Studio Code, select the Python virtual environment as the kernel for running the notebook
+- Review the instructions at the top of the report and make any required changes to e.g. reporting parameters
+- Click on "Run All" to run the report and export the results
+- Exported results are written to a folder named "exported" within the reports folder

reports/make_venv.bat

@@ -0,0 +1,19 @@
+@ECHO OFF
+
+REM Deactivate and remove the old virtual environment, if present
+ECHO Removing existing Virtual Environment, if present ...
+deactivate > nul 2>&1
+RMDIR /S /Q venv
+
+REM Create a new environment and activate it
+ECHO Creating new Virtual Environment ...
+python -m venv venv
+CALL venv\Scripts\activate.bat
+
+REM Make sure pip is up to date
+python -m pip install --upgrade pip
+
+REM Install the requirements
+python -m pip install -r requirements.txt
+
+ECHO ON

reports/make_venv.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -f
+
+# Deactivate and remove the old virtual environment, if present
+echo "Removing existing Virtual Environment, if present ..."
+deactivate 2> /dev/null || true
+rm -fr venv
+
+# Create a new environment and activate it
+echo "Creating new Virtual Environment ..."
+python -m venv venv
+. venv/bin/activate
+
+# Make sure pip is up to date
+pip install --upgrade pip
+
+# Install the requirements
+pip install -r requirements.txt

reports/requirements.txt

@@ -0,0 +1,109 @@
+anyio==4.9.0
+appnope==0.1.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==3.0.0
+async-lru==2.0.5
+attrs==25.3.0
+babel==2.17.0
+beautifulsoup4==4.13.4
+bleach==6.2.0
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+comm==0.2.2
+contourpy==1.3.2
+cycler==0.12.1
+debugpy==1.8.14
+decorator==5.2.1
+defusedxml==0.7.1
+executing==2.2.0
+fastjsonschema==2.21.1
+fonttools==4.57.0
+fqdn==1.5.1
+h11==0.14.0
+httpcore==1.0.8
+httpx==0.28.1
+idna==3.10
+ipykernel==6.29.5
+ipython==9.1.0
+ipython_pygments_lexers==1.1.1
+ipywidgets==8.1.6
+isoduration==20.11.0
+jedi==0.19.2
+Jinja2==3.1.6
+json5==0.12.0
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter==1.1.1
+jupyter-console==6.6.3
+jupyter-events==0.12.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.15.0
+jupyter_server_terminals==0.5.3
+jupyterlab==4.4.0
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+jupyterlab_widgets==3.0.14
+kiwisolver==1.4.8
+MarkupSafe==3.0.2
+matplotlib==3.10.1
+matplotlib-inline==0.1.7
+mistune==3.1.3
+nbclient==0.10.2
+nbconvert==7.16.6
+nbformat==5.10.4
+nest-asyncio==1.6.0
+notebook==7.4.0
+notebook_shim==0.2.4
+numpy==2.2.4
+overrides==7.7.0
+packaging==24.2
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pexpect==4.9.0
+pillow==11.2.1
+platformdirs==4.3.7
+prometheus_client==0.21.1
+prompt_toolkit==3.0.51
+psutil==7.0.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pycparser==2.22
+Pygments==2.19.1
+pyparsing==3.2.3
+python-dateutil==2.9.0.post0
+python-json-logger==3.3.0
+pytz==2025.2
+PyYAML==6.0.2
+pyzmq==26.4.0
+referencing==0.36.2
+requests==2.32.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.24.0
+seaborn==0.13.2
+Send2Trash==1.8.3
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.6
+stack-data==0.6.3
+terminado==0.18.1
+tinycss2==1.4.0
+tornado==6.4.2
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20241206
+typing_extensions==4.13.2
+tzdata==2025.2
+uri-template==1.3.0
+urllib3==2.4.0
+wcwidth==0.2.13
+webcolors==24.11.1
+webencodings==0.5.1
+websocket-client==1.8.0
+widgetsnbextension==4.0.14

reports/sql/sightings.sql

@@ -0,0 +1,9 @@
+SELECT l.Name AS 'Location', sp.Name AS 'Species', sp.Name AS 'Category', DATE( s.Date ) AS 'Date'
+FROM SIGHTINGS s
+INNER JOIN SPECIES sp ON sp.Id = s.SpeciesId
+INNER JOIN CATEGORIES c ON c.Id = sp.CategoryId
+INNER JOIN LOCATIONS l ON l.Id = s.LocationId
+WHERE l.Name = '$LOCATION'
+AND s.Date LIKE '$YEAR-%'
+AND sp.Name LIKE '%$SPECIES%'
+AND c.Name = "$CATEGORY";

setup.py

@@ -31,7 +31,7 @@ def find_package_files(directory, remove_root):
 
 setuptools.setup(
     name="nature_recorder",
-    version="1.13.0",
+    version="1.14.0",
     description="Wildlife sightings database",
     packages=setuptools.find_packages("src"),
     include_package_data=True,