Berickson/sync upstream 2025 10 06

.github/workflows/lint.yml

@@ -5,40 +5,31 @@ on: [push]
 jobs:
   lint-python:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install pycodestyle
-    - name: Analysing the code with pycodestyle
-      run: |
-        pycodestyle --ignore=E501,W503 $(git ls-files 'learning_observer/*.py' 'modules/*.py')
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install Make
+        run: sudo apt-get install make
+
+      - name: Lint files
+        run: make linting-python
   lint-node:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-      - name: Set up Node.js
-        uses: actions/setup-node@v2
+        uses: actions/checkout@v3
+      - name: Set up Node
+        uses: actions/setup-node@v4
         with:
-          node-version: '16.x'
-      - name: Install dependencies
-        run: |
-          npm install
-
-      - name: List unused CSS
-        run: npm run find-unused-css
+          node-version: '22.x'
 
-      - name: Lint CSS
-        run: npm run lint:css
+      - name: Install Make
+        run: sudo apt-get install make
 
-      - name: Lint JS
-        run: npm run lint:js
+      - name: Lint files
+        run: make linting-node

.github/workflows/test.yml

@@ -0,0 +1,51 @@
+name: Test packages
+
+on: [push]
+
+jobs:
+  test-packages:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        package: ['learning_observer/', 'modules/writing_observer/']
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Install Make
+        run: sudo apt-get install make
+
+      - name: Get list of changed files
+        id: changes
+        run: |
+          git fetch origin master
+          git diff --name-only origin/master HEAD > changed_files.txt
+
+      - name: Check if package has changes
+        id: package_check
+        run: |
+          if grep -qE "^${{ matrix.package }}" changed_files.txt; then
+            echo "run_tests=true" >> $GITHUB_ENV
+          else
+            echo "run_tests=false" >> $GITHUB_ENV
+          fi
+
+      - name: Skip tests if no changes
+        if: env.run_tests == 'false'
+        run: echo "Skipping tests for ${{ matrix.package }} as there are no changes."
+
+      - name: Install the base Learning Observer
+        if: env.run_tests == 'true'
+        run: make install
+
+      - name: Install the package with pip
+        if: env.run_tests == 'true'
+        run: pip install -e ${{ matrix.package }}
+
+      - name: Run tests
+        if: env.run_tests == 'true'
+        run: make test PKG=${{ matrix.package }}

.gitignore

@@ -37,4 +37,6 @@ LanguageTool-5.4
 package-lock.json
 learning_observer/learning_observer/static_data/google/
 learning_observer/learning_observer/static_data/admins.yaml
-.ipynb_checkpoints/
+.ipynb_checkpoints/
+.eggs/
+.next/

Makefile

@@ -1,16 +1,40 @@
-PACKAGES ?= wo,awe
+# TODO rename these packages to something else
+PACKAGES ?= wo
+
+help:
+	@echo "Available commands:"
+	@echo ""
+	@echo "  run                          Run the learning_observer Python application."
+	@echo "	 install-pre-commit-hook      Install the pre-commit git hook."
+	@echo "  install                      Install the learning_observer package in development mode."
+	@echo "  install-dev                  Install dev dependencies (requires additional setup)."
+	@echo "  install-packages             Install specific packages: [${PACKAGES}]."
+	@echo "  test                         Run tests for the specified package (PKG=<package>)."
+	@echo "  linting-setup                Setup linting tools and dependencies."
+	@echo "  linting-python               Lint Python files using pycodestyle and pylint."
+	@echo "  linting-node                 Lint Node files (JS, CSS, and unused CSS detection)."
+	@echo "  linting                      Perform all linting tasks (Python and Node)."
+	@echo "  build-wo-chrome-extension    Build the writing-process extension."
+	@echo "  build-python-distribution    Build a distribution for the specified package (PKG=<package>)."
+	@echo ""
+	@echo "Note: All commands are executed in the current shell environment."
+	@echo "      Ensure your virtual environment is activated if desired, as installs and actions"
+	@echo "      will occur in the environment where the 'make' command is run."
+	@echo ""
+	@echo "Use 'make <command>' to execute a command. For example: make run"
 
 run:
 	# If you haven't done so yet, run: make install
 	# we need to make sure we are on the virtual env when we do this
 	cd learning_observer && python learning_observer
 
-venv:
-	# This is unnecessary since LO installs requirements on install.
-	# pip install --no-cache-dir -r requirements.txt
+# Install commands
+install-pre-commit-hook:
+	# Adding pre-commit.sh to Git hooks
+	cp scripts/hooks/pre-commit.sh .git/hooks/pre-commit
+	chmod +x .git/hooks/pre-commit
 
-# install commands
-install: venv
+install: install-pre-commit-hook
 	# The following only works with specified packages
 	# we need to install learning_observer in dev mode to
 	# more easily pass in specific files we need, such as creds
@@ -25,52 +49,28 @@ install: venv
 	@LODRC_CURRENT=$$(curl -s https://raw.githubusercontent.com/ETS-Next-Gen/lo_assets/main/lo_dash_react_components/lo_dash_react_components-current.tar.gz); \
 	pip install https://raw.githubusercontent.com/ETS-Next-Gen/lo_assets/main/lo_dash_react_components/$${LODRC_CURRENT}
 
-install-dev: venv
+install-dev:
 	# TODO create a dev requirements file
 	pip install --no-cache-dir -e learning_observer/[${PACKAGES}]
 	. ${HOME}/.nvm/nvm.sh && nvm use && pip install -v -e modules/lo_dash_react_components/
 
-install-packages: venv
+install-packages:
 	pip install -e learning_observer/[${PACKAGES}]
 
-	# Just a little bit of dependency hell...
-
-	# The AWE Components are built using a specific version of
-	# `spacy`. This requires an out-of-date `typing-extensions`
-	# package. There are few other dependecies that require a
-	# newer version. As far as I can tell, upgrading this package
-	# does not effect the functionality we receive from the AWE
-	# components.
-	# TODO remove this extra step after AWE Component's `spacy`
-	# is no longer version locked.
-	# This is no longer an issue, but we will leave until all
-	# dependecies can be resolved in the appropriate locations.
-	# pip install -U typing-extensions
-
-	# On Python3.11 with tensorflow, we get some odd errors
-	# regarding compatibility with `protobuf`. Some installation
-	# files are missing from the protobuf binary on pip.
-	# Using the `--no-binary` option includes all files.
-	pip uninstall -y protobuf
-	pip install --no-binary=protobuf protobuf==4.25
-
-# testing commands
+# Testing commands
 test:
-	# this is where we run doctests
-	pytest modules/wo_highlight_dashboard
+	@if [ -z "$(PKG)" ]; then echo "No module specified, please try again with \"make test PKG=path/to/module\""; exit 1; fi
+	./test.sh $(PKG)
 
 # Linting commands
-linting-setup:
-	# Setting up linting related packages
-	pip install pycodestyle pylint
-	npm install
-
 linting-python:
 	# Linting Python modules
+	pip install pycodestyle pylint
 	pycodestyle --ignore=E501,W503 $$(git ls-files 'learning_observer/*.py' 'modules/*.py')
 	pylint -d W0613,W0511,C0301,R0913,too-few-public-methods $$(git ls-files 'learning_observer/*.py' 'modules/*.py')
 
 linting-node:
+	npm install
 	# TODO each of these have lots of errors and block
 	# the next item from running
 	# Starting to lint Node modules
@@ -84,8 +84,27 @@ linting-node:
 linting: linting-setup linting-python linting-node
 	# Finished linting
 
-build-writing-ext:
+# Build commands
+build-wo-chrome-extension:
 	# Installing LO Event
 	cd modules/lo_event && npm install & npm link lo_event
 	# Building extension
 	cd extension/writing-process && npm install && npm run build
+
+build-python-distribution:
+	# Building distribution for package
+	pip install build
+	# Switching to package directory
+	cd $(PKG) && python -m build
+
+# TODO we may want to have a separate command for uploading to testpypi
+upload-python-package-to-pypi: build-python-distribution
+	pip install twine
+	# TODO we currently only upload to testpypi
+	# TODO we need to include `TWINE_USERNAME=__token__`
+	# and `TWINE_PASSWORD={ourTwineToken}` to authenticate
+	#
+	# TODO We have not fully tested the following commands.
+	# Try out the following steps and fix any bugs so the
+	# Makefile can do it automatically.
+	# cd $(PKG) && twine upload -r testpypi dist/*

VERSION

@@ -0,0 +1 @@
+0.1.0+2025.10.01T21.16.15.146Z.8b205d0c.master

autodocs/.gitignore

@@ -1,3 +1,4 @@
 _build/
 generated/
 apidocs/
+module_readmes/

autodocs/api.rst

@@ -2,5 +2,6 @@ API
 ===
 
 .. toctree::
+   :maxdepth: 4
 
    apidocs/index

autodocs/concepts.rst

@@ -0,0 +1,49 @@
+Concepts
+=============
+
+Explanations of key ideas, principles, and background knowledge.
+Follow this recommended sequence to build context before diving into
+implementation details:
+
+- :doc:`History <docs/concepts/history>` - establishes the background and
+  problem space the project is addressing.
+- :doc:`System Design <docs/concepts/system_design>` - explains how the product
+  strategy and user needs translate into an overall system approach.
+- :doc:`Architecture <docs/concepts/architecture>` - outlines the concrete
+  architecture that implements the system design.
+- :doc:`Technologies <docs/concepts/technologies>` - surveys the primary tools
+  and platforms we rely on to realize the architecture.
+- :doc:`System Settings <docs/concepts/system_settings>` - describes how the
+  system loads global and cascading settings.
+- :doc:`Events <docs/concepts/events>` - introduces the event model that drives
+  data flowing through the system.
+- :doc:`Reducers <docs/concepts/reducers>` - details how incoming events are
+  aggregated into the state our experiences depend on.
+- :doc:`Communication Protocol <docs/concepts/communication_protocol>` - discusses how
+  the system queries data from reducers for dashboards.
+- :doc:`Student Identity Mapping <docs/concepts/student_identity_mapping>` - explain
+  how learners information is mapped across integrations.
+- :doc:`Scaling <docs/concepts/scaling>` - covers strategies for growing the
+  system once the fundamentals are in place.
+- :doc:`Auth <docs/concepts/auth>` - describes authentication considerations
+  that secure access to the system.
+- :doc:`Privacy <docs/concepts/privacy>` - documents how we protect learner data
+  and comply with privacy expectations.
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+   :titlesonly:
+
+   docs/concepts/history
+   docs/concepts/system_design
+   docs/concepts/architecture
+   docs/concepts/technologies
+   docs/concepts/system_settings
+   docs/concepts/events
+   docs/concepts/reducers
+   docs/concepts/communication_protocol
+   docs/concepts/student_identity_mapping
+   docs/concepts/scaling
+   docs/concepts/auth
+   docs/concepts/privacy