From ee9f7367c442c83c843dcb31c17847dbe08f8e14 Mon Sep 17 00:00:00 2001 From: Bryceson Laing Date: Thu, 2 Jul 2026 10:21:00 -0500 Subject: [PATCH 1/5] add new `events_queue_reindex_touched_dois` lambda --- ...vents_queue_reindex_touched_dois_runner.py | 42 +++++++++++++++++++ .../requirements.txt | 1 + 2 files changed, 43 insertions(+) create mode 100644 lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py create mode 100644 lambdas/events_queue_reindex_touched_dois/requirements.txt diff --git a/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py b/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py new file mode 100644 index 0000000..293fb8d --- /dev/null +++ b/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py @@ -0,0 +1,42 @@ +""" +This lambda function is triggered by a CloudWatch event rule on a schedule. +It add a message to the queue with the current date, to be processed by Events. +Events will call the `reindex_touched_dois` method to reindex the DOIs touched today. +""" + +import os +import json +import datetime +import boto3 + +# Lambda handler +def lambda_handler(event, context): + """Lambda process handler""" + print("Starting reindex_touched_dois_runner lambda function") + + # Configuration + queue_name = os.getenv('QUEUE_NAME') + + # Calculate the date range for the reindex + today = datetime.date.today().strftime("%Y-%m-%d") + + message = { + 'Name': 'shoryuken_class', + 'Type': 'String', + 'Value': 'ReindexTouchedDoisWorker', + + 'date': today, + } + + # Queue a task for each repository + sqs = boto3.resource('sqs') + queue = sqs.get_queue_by_name(QueueName=queue_name) + queue.send_message(MessageBody=json.dumps(message)) + + return("Queued reindex for dois touched on {}".format(today)) + +if __name__ == '__main__': + # For local testing fake the arguments to lambda handler function + event = {} + context = [] + lambda_handler(event, context) diff --git a/lambdas/events_queue_reindex_touched_dois/requirements.txt b/lambdas/events_queue_reindex_touched_dois/requirements.txt new file mode 100644 index 0000000..30ddf82 --- /dev/null +++ b/lambdas/events_queue_reindex_touched_dois/requirements.txt @@ -0,0 +1 @@ +boto3 From 4b16fc15c161ee10586786c5b5a3f78b8c9dfaa1 Mon Sep 17 00:00:00 2001 From: Bryceson Laing Date: Thu, 2 Jul 2026 10:23:55 -0500 Subject: [PATCH 2/5] add docker-compose and bash script to test lambdas locally with datacite/datacite_localstack --- docker-compose.yml | 13 +++++++++++ run_lambda.sh | 57 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 docker-compose.yml create mode 100755 run_lambda.sh diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..97d9dae --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + lambda: + image: python:3.14.6-alpine + networks: + - localstack_network + volumes: + - ${LAMBDA_DIR}:/var/task + working_dir: /var/task + command: sh -c "pip install --quiet -r requirements.txt && python ${LAMBDA_PATH}" + +networks: + localstack_network: + external: true diff --git a/run_lambda.sh b/run_lambda.sh new file mode 100755 index 0000000..20bcd3f --- /dev/null +++ b/run_lambda.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + ./run_lambda.sh [KEY=VALUE]... +EOF +} + +[[ $# -ge 1 ]] || { usage; exit 1; } + +SCRIPT_PATH="$1" +shift + +LAMBDA_DIR="$(dirname "$(realpath "$SCRIPT_PATH")")" +LAMBDA_PATH="$(basename "$SCRIPT_PATH")" + +# Defaults for LocalStack use +: "${AWS_ACCESS_KEY_ID:=test}" +: "${AWS_SECRET_ACCESS_KEY:=test}" +: "${AWS_DEFAULT_REGION:=us-east-1}" +: "${AWS_ENDPOINT_URL:=http://localstack:4566}" + +export LAMBDA_DIR LAMBDA_PATH +export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_ENDPOINT_URL + +# Parse trailing KEY=VALUE args +env_args=() +for spec in "$@"; do + if [[ "$spec" != *=* ]]; then + echo "Error: invalid argument '$spec' (expected KEY=VALUE)" + usage + exit 1 + fi + + key="${spec%%=*}" + val="${spec#*=}" + + if [[ -z "$key" ]]; then + echo "Error: invalid assignment '$spec' (empty key)" + exit 1 + fi + + # Export so compose process sees it, and pass through to container + export "$key=$val" + env_args+=(-e "$key") +done + +# Always pass required AWS vars explicitly +docker compose run --rm \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -e AWS_DEFAULT_REGION \ + -e AWS_ENDPOINT_URL \ + "${env_args[@]}" \ + lambda From 17091ef5cf87fa71de33c9fa00f98aa7093f3bf6 Mon Sep 17 00:00:00 2001 From: Bryceson Laing Date: Tue, 7 Jul 2026 10:13:02 -0500 Subject: [PATCH 3/5] set the date as yesterday instead of today --- .../events_queue_reindex_touched_dois_runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py b/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py index 293fb8d..70ba9d4 100644 --- a/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py +++ b/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py @@ -18,14 +18,15 @@ def lambda_handler(event, context): queue_name = os.getenv('QUEUE_NAME') # Calculate the date range for the reindex - today = datetime.date.today().strftime("%Y-%m-%d") + yesterday = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=1) + yesterday = yesterday.strftime("%Y-%m-%d") message = { 'Name': 'shoryuken_class', 'Type': 'String', 'Value': 'ReindexTouchedDoisWorker', - 'date': today, + 'date': yesterday, } # Queue a task for each repository @@ -33,7 +34,7 @@ def lambda_handler(event, context): queue = sqs.get_queue_by_name(QueueName=queue_name) queue.send_message(MessageBody=json.dumps(message)) - return("Queued reindex for dois touched on {}".format(today)) + return("Queued reindex for dois touched on {}".format(yesterday)) if __name__ == '__main__': # For local testing fake the arguments to lambda handler function From 24b9b24324bd1c7eb67a8f5cf9b407182379f91c Mon Sep 17 00:00:00 2001 From: Bryceson Laing Date: Wed, 8 Jul 2026 13:48:53 -0500 Subject: [PATCH 4/5] add lambda to trigger daily events reindex to the build script --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index fd797b8..af0451c 100755 --- a/build.sh +++ b/build.sh @@ -29,3 +29,4 @@ packagePythonLambda 'check_links' packagePythonLambda 'analytics_worker' packagePythonLambda 'analytics_queue_reports' packagePythonLambda 'datafile_generator' +packagePythonLambda 'events_queue_reindex_touched_dois' From db474b6197806f6ec0975393ddc8bf90f6978e8c Mon Sep 17 00:00:00 2001 From: Bryceson Laing Date: Wed, 8 Jul 2026 14:28:46 -0500 Subject: [PATCH 5/5] simplify the `run_lambda` script --- .env.example | 6 ++++++ docker-compose.yml | 6 ++++++ run_lambda.sh | 47 ++++++---------------------------------------- 3 files changed, 18 insertions(+), 41 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ad39586 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test +AWS_DEFAULT_REGION=us-east-1 +AWS_ENDPOINT_URL=http://localstack:4566 + +QUEUE_NAME=development_events_reindex_daily diff --git a/docker-compose.yml b/docker-compose.yml index 97d9dae..5ae377f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,12 @@ services: lambda: image: python:3.14.6-alpine + environment: + - AWS_ACCESS_KEY_ID=test + - AWS_SECRET_ACCESS_KEY=test + - AWS_DEFAULT_REGION=us-east-1 + - AWS_ENDPOINT_URL=http://localstack:4566 + - QUEUE_NAME networks: - localstack_network volumes: diff --git a/run_lambda.sh b/run_lambda.sh index 20bcd3f..bfac52a 100755 --- a/run_lambda.sh +++ b/run_lambda.sh @@ -4,54 +4,19 @@ set -euo pipefail usage() { cat <<'EOF' Usage: - ./run_lambda.sh [KEY=VALUE]... + ./run_lambda.sh + +Additional environment variables need to be added to docker-compose.yml EOF } -[[ $# -ge 1 ]] || { usage; exit 1; } +[[ $# -eq 1 ]] || { usage; exit 1; } SCRIPT_PATH="$1" -shift +[[ -f "$SCRIPT_PATH" ]] || { echo "Error: file not found: $SCRIPT_PATH" >&2; exit 1; } LAMBDA_DIR="$(dirname "$(realpath "$SCRIPT_PATH")")" LAMBDA_PATH="$(basename "$SCRIPT_PATH")" - -# Defaults for LocalStack use -: "${AWS_ACCESS_KEY_ID:=test}" -: "${AWS_SECRET_ACCESS_KEY:=test}" -: "${AWS_DEFAULT_REGION:=us-east-1}" -: "${AWS_ENDPOINT_URL:=http://localstack:4566}" - export LAMBDA_DIR LAMBDA_PATH -export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_ENDPOINT_URL - -# Parse trailing KEY=VALUE args -env_args=() -for spec in "$@"; do - if [[ "$spec" != *=* ]]; then - echo "Error: invalid argument '$spec' (expected KEY=VALUE)" - usage - exit 1 - fi - - key="${spec%%=*}" - val="${spec#*=}" - - if [[ -z "$key" ]]; then - echo "Error: invalid assignment '$spec' (empty key)" - exit 1 - fi - - # Export so compose process sees it, and pass through to container - export "$key=$val" - env_args+=(-e "$key") -done -# Always pass required AWS vars explicitly -docker compose run --rm \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -e AWS_DEFAULT_REGION \ - -e AWS_ENDPOINT_URL \ - "${env_args[@]}" \ - lambda +docker compose run --rm lambda