From 61cd60cf70027eafb9a9d8929812e5d88f705c94 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Thu, 4 Jun 2026 14:57:47 +0200 Subject: [PATCH 01/16] Remove unused docker-compose files --- .github/workflows/run-test-suite.yml | 20 ++-- .github/workflows/tests.yml | 2 +- docker-compose-dev.yml | 163 --------------------------- docker-compose-geoserver-server.yml | 36 ------ docker-compose-test.yml | 150 +++--------------------- 5 files changed, 25 insertions(+), 346 deletions(-) delete mode 100644 docker-compose-dev.yml delete mode 100644 docker-compose-geoserver-server.yml diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index c1f32764a2f..8a402c56472 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,20 +28,20 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: docker compose --env-file .env_test -f docker-compose-test.yml up -d + run: docker compose -f docker-compose.yml -f docker-compose-test.yml up -d - name: Setup test databases run: | - docker compose --env-file .env_test -f docker-compose-test.yml exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" - docker compose --env-file .env_test -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_postgres - docker compose --env-file .env_test -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode - docker compose --env-file .env_test -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode_data - docker compose --env-file .env_test -f docker-compose-test.yml exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" - docker compose --env-file .env_test -f docker-compose-test.yml exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" + docker compose -f docker-compose.yml -f docker-compose-test.yml exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" + docker compose -f docker-compose.yml -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_postgres + docker compose -f docker-compose.yml -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode + docker compose -f docker-compose.yml -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode_data + docker compose -f docker-compose.yml -f docker-compose-test.yml exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" + docker compose -f docker-compose.yml -f docker-compose-test.yml exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" - name: Run ${{ inputs.suite_name }} run: | - docker compose --env-file .env_test -f docker-compose-test.yml exec -T django bash -s <<'BASH' + docker compose -f docker-compose.yml -f docker-compose-test.yml exec -T django bash -s <<'BASH' set -euo pipefail ${{ inputs.test_command }} BASH @@ -52,8 +52,8 @@ jobs: - name: Codecov run: | - docker compose --env-file .env_test -f docker-compose-test.yml exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" + docker compose -f docker-compose.yml -f docker-compose-test.yml exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" - name: Stop the stack if: always() - run: docker compose --env-file .env_test -f docker-compose-test.yml down -v + run: docker compose -f docker-compose.yml -f docker-compose-test.yml down -v diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 516d8ed8ac4..b6bdf456317 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: docker compose --env-file .env_test -f docker-compose-test.yml build --progress plain + run: docker compose -f docker-compose.yml -f docker-compose-test.yml build --progress plain - name: Save built Docker images run: | diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml deleted file mode 100644 index ab2398edafc..00000000000 --- a/docker-compose-dev.yml +++ /dev/null @@ -1,163 +0,0 @@ -version: '3.9' - -# Common Django template for GeoNode and Celery services below -x-common-django: - &default-common-django - image: geonode/geonode:local - build: - context: ./ - dockerfile: Dockerfile - restart: unless-stopped - env_file: - - .env - volumes: - - '.:/usr/src/geonode' - - statics:/mnt/volumes/statics - - geoserver-data-dir:/geoserver_data/data - - backup-restore:/backup_restore - - data:/data - - tmp:/tmp - depends_on: - db: - condition: service_healthy - -services: - - # Our custom django application. It includes Geonode. - django: - << : *default-common-django - container_name: django4${COMPOSE_PROJECT_NAME} - healthcheck: - test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://django:8000/" - start_period: 60s - interval: 60s - timeout: 10s - retries: 2 - environment: - - IS_CELERY=False - entrypoint: ["/usr/src/geonode/entrypoint.sh"] - command: "uwsgi --ini /usr/src/geonode/uwsgi.ini" - - # Celery worker that executes celery tasks created by Django. - celery: - << : *default-common-django - container_name: celery4${COMPOSE_PROJECT_NAME} - depends_on: - django: - condition: service_healthy - environment: - - IS_CELERY=True - entrypoint: ["/usr/src/geonode/entrypoint.sh"] - command: "celery-cmd" - - # Nginx is serving django static and media files and proxies to django and geonode - geonode: - image: geonode/nginx:1.31.0-latest - container_name: nginx4${COMPOSE_PROJECT_NAME} - env_file: - - .env - environment: - - RESOLVER=127.0.0.11 - ports: - - "${HTTP_PORT}:80" - - "${HTTPS_PORT}:443" - volumes: - - nginx-confd:/etc/nginx - - nginx-certificates:/geonode-certificates - - statics:/mnt/volumes/statics - restart: unless-stopped - - # Gets and installs letsencrypt certificates - letsencrypt: - image: geonode/letsencrypt:2.6.0-latest - container_name: letsencrypt4${COMPOSE_PROJECT_NAME} - env_file: - - .env - volumes: - - nginx-certificates:/geonode-certificates - restart: unless-stopped - - # Geoserver backend - geoserver: - image: geonode/geoserver:2.28.x-latest - container_name: geoserver4${COMPOSE_PROJECT_NAME} - healthcheck: - test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" - start_period: 60s - interval: 60s - timeout: 10s - retries: 2 - env_file: - - .env - ports: - - "8080:8080" - volumes: - - statics:/mnt/volumes/statics - - geoserver-data-dir:/geoserver_data/data - - backup-restore:/backup_restore - - data:/data - - tmp:/tmp - restart: unless-stopped - depends_on: - data-dir-conf: - condition: service_healthy - django: - condition: service_healthy - - data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest - container_name: gsconf4${COMPOSE_PROJECT_NAME} - entrypoint: sleep infinity - volumes: - - geoserver-data-dir:/geoserver_data/data - restart: unless-stopped - healthcheck: - test: "ls -A '/geoserver_data/data' | wc -l" - - # PostGIS database. - db: - # use geonode official postgis 15 image - image: geonode/postgis:15-3.5-latest - command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS}" - container_name: db4${COMPOSE_PROJECT_NAME} - env_file: - - .env - volumes: - - dbdata:/var/lib/postgresql/data - - dbbackups:/pg_backups - restart: unless-stopped - healthcheck: - test: "pg_isready -d postgres -U postgres" - # uncomment to enable remote connections to postgres - #ports: - # - "5432:5432" - - # Vanilla Redis service. This is needed by celery - redis: - image: redis:7-alpine - container_name: redis4${COMPOSE_PROJECT_NAME} - volumes: - - redisdata:/data - restart: unless-stopped - -volumes: - statics: - name: ${COMPOSE_PROJECT_NAME}-statics - nginx-confd: - name: ${COMPOSE_PROJECT_NAME}-nginxconfd - nginx-certificates: - name: ${COMPOSE_PROJECT_NAME}-nginxcerts - geoserver-data-dir: - name: ${COMPOSE_PROJECT_NAME}-gsdatadir - dbdata: - name: ${COMPOSE_PROJECT_NAME}-dbdata - dbbackups: - name: ${COMPOSE_PROJECT_NAME}-dbbackups - backup-restore: - name: ${COMPOSE_PROJECT_NAME}-backup-restore - data: - name: ${COMPOSE_PROJECT_NAME}-data - tmp: - name: ${COMPOSE_PROJECT_NAME}-tmp - redisdata: - name: ${COMPOSE_PROJECT_NAME}-redisdata diff --git a/docker-compose-geoserver-server.yml b/docker-compose-geoserver-server.yml deleted file mode 100644 index b0fa460421c..00000000000 --- a/docker-compose-geoserver-server.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: '2.2' -services: - - data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest - restart: on-failure - container_name: gsconf4${COMPOSE_PROJECT_NAME} - labels: - org.geonode.component: conf - org.geonode.instance.name: geonode - command: /bin/true - volumes: - - geoserver-data-dir:/geoserver_data/data - - geoserver: - image: geonode/geoserver:2.28.x-latest - restart: unless-stopped - container_name: geoserver4${COMPOSE_PROJECT_NAME} - stdin_open: true - # tty: true - labels: - org.geonode.component: geoserver - org.geonode.instance.name: geonode - depends_on: - - data-dir-conf - volumes: - - geoserver-data-dir:/geoserver_data/data - env_file: - - .env - ports: - - "${GEOSERVER_SERVER_PORT}:8080" - network_mode: "bridge" - -volumes: - geoserver-data-dir: - name: ${COMPOSE_PROJECT_NAME}-gsdatadir diff --git a/docker-compose-test.yml b/docker-compose-test.yml index c1d018a3d64..e94bc38e20f 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -1,154 +1,32 @@ version: '3.9' -# Common Django template for GeoNode and Celery services below -x-common-django: - &default-common-django - image: geonode/geonode:latest-ubuntu-24.04 - build: - context: ./ - dockerfile: Dockerfile - restart: unless-stopped - env_file: - - .env_test - volumes: - # - '.:/usr/src/geonode' - - statics:/mnt/volumes/statics - - geoserver-data-dir:/geoserver_data/data - - backup-restore:/backup_restore - - data:/data - - tmp:/tmp - depends_on: - db: - condition: service_healthy - services: # Our custom django application. It includes Geonode. django: - << : *default-common-django - container_name: django4${COMPOSE_PROJECT_NAME} - healthcheck: - test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://django:8000/" - start_period: 60s - interval: 60s - timeout: 10s - retries: 5 + env_file: + - .env_test environment: - IS_CELERY=False - entrypoint: ["/usr/src/geonode/entrypoint.sh"] - command: "uwsgi --ini /usr/src/geonode/uwsgi.ini" - # Nginx is serving django static and media files and proxies to django and geonode - geonode: - image: geonode/nginx:1.31.0-latest - container_name: nginx4${COMPOSE_PROJECT_NAME} + # Celery worker that executes celery tasks created by Django. + celery: env_file: - .env_test environment: - - RESOLVER=127.0.0.11 - ports: - - "${HTTP_PORT}:80" - - "${HTTPS_PORT}:443" - volumes: - - nginx-confd:/etc/nginx - - nginx-certificates:/geonode-certificates - - statics:/mnt/volumes/statics - restart: unless-stopped - - # memcached service - memcached: - image: memcached:alpine - container_name: memcached4${COMPOSE_PROJECT_NAME} - command: memcached ${MEMCACHED_OPTIONS} - restart: unless-stopped - healthcheck: - test: nc -z 127.0.0.1 11211 - interval: 30s - timeout: 30s - retries: 5 - start_period: 30s - - # # Gets and installs letsencrypt certificates - # letsencrypt: - # image: geonode/letsencrypt:2.6.0-latest - # container_name: letsencrypt4${COMPOSE_PROJECT_NAME} - # env_file: - # - .env_test - # volumes: - # - nginx-certificates:/geonode-certificates - # restart: unless-stopped + - IS_CELERY=True + # Nginx is serving django static and media files and proxies to django and geonode + geonode: + env_file: + - .env_test + # Gets and installs letsencrypt certificates + letsencrypt: + env_file: + - .env_test # Geoserver backend geoserver: - image: geonode/geoserver:2.28.x-latest - container_name: geoserver4${COMPOSE_PROJECT_NAME} - healthcheck: - test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" - start_period: 60s - interval: 60s - timeout: 10s - retries: 2 env_file: - .env_test ports: - - "8080:8080" - volumes: - - statics:/mnt/volumes/statics - - geoserver-data-dir:/geoserver_data/data - - backup-restore:/backup_restore - - data:/data - - tmp:/tmp - restart: unless-stopped - depends_on: - data-dir-conf: - condition: service_healthy - django: - condition: service_healthy - - data-dir-conf: - image: geonode/geoserver_data:2.28.x-latest - container_name: gsconf4${COMPOSE_PROJECT_NAME} - entrypoint: sleep infinity - volumes: - - geoserver-data-dir:/geoserver_data/data - restart: unless-stopped - healthcheck: - test: "ls -A '/geoserver_data/data' | wc -l" - - # PostGIS database. - db: - # use geonode official postgis 15 image - image: geonode/postgis:15-3.5-latest - command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS}" - container_name: db4${COMPOSE_PROJECT_NAME} - env_file: - - .env_test - volumes: - - dbdata:/var/lib/postgresql/data - - dbbackups:/pg_backups - restart: unless-stopped - healthcheck: - test: "pg_isready -d postgres -U postgres" - # uncomment to enable remote connections to postgres - #ports: - # - "5432:5432" - -volumes: - statics: - name: ${COMPOSE_PROJECT_NAME}-statics - nginx-confd: - name: ${COMPOSE_PROJECT_NAME}-nginxconfd - nginx-certificates: - name: ${COMPOSE_PROJECT_NAME}-nginxcerts - geoserver-data-dir: - name: ${COMPOSE_PROJECT_NAME}-gsdatadir - dbdata: - name: ${COMPOSE_PROJECT_NAME}-dbdata - dbbackups: - name: ${COMPOSE_PROJECT_NAME}-dbbackups - backup-restore: - name: ${COMPOSE_PROJECT_NAME}-backup-restore - data: - name: ${COMPOSE_PROJECT_NAME}-data - tmp: - name: ${COMPOSE_PROJECT_NAME}-tmp + - "8080:8080" \ No newline at end of file From a82c1861fdb72e1fcd4281bd1989e9ae4c8b1a9e Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Thu, 4 Jun 2026 15:23:58 +0200 Subject: [PATCH 02/16] Remove unused docker-compose files --- .env_dev | 213 -------------------------- .env_local | 214 --------------------------- .github/workflows/run-test-suite.yml | 20 +-- .github/workflows/tests.yml | 2 +- docker-compose-test.yml | 32 ---- docker-compose.yml | 10 +- 6 files changed, 16 insertions(+), 475 deletions(-) delete mode 100644 .env_dev delete mode 100644 .env_local delete mode 100644 docker-compose-test.yml diff --git a/.env_dev b/.env_dev deleted file mode 100644 index 9c9b7eaa176..00000000000 --- a/.env_dev +++ /dev/null @@ -1,213 +0,0 @@ -COMPOSE_PROJECT_NAME=geonode -# See https://github.com/containers/podman/issues/13889 -# DOCKER_BUILDKIT=0 -DOCKER_ENV=production -BACKUPS_VOLUME_DRIVER=local - -C_FORCE_ROOT=1 -FORCE_REINIT=false -INVOKE_LOG_STDOUT=true - -# LANGUAGE_CODE=it-it -# LANGUAGES=(('en-us','English'),('it-it','Italiano')) - -DJANGO_SETTINGS_MODULE=geonode.settings -GEONODE_INSTANCE_NAME=geonode - -# ################# -# backend -# ################# -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -GEONODE_DATABASE=geonode -GEONODE_DATABASE_USER=geonode -GEONODE_DATABASE_PASSWORD=geonode -GEONODE_GEODATABASE=geonode_data -GEONODE_GEODATABASE_USER=geonode -GEONODE_GEODATABASE_PASSWORD=geonode -GEONODE_DATABASE_SCHEMA=public -GEONODE_GEODATABASE_SCHEMA=public -DATABASE_HOST=localhost -DATABASE_PORT=5432 -DATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode -GEODATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode_data -GEONODE_DB_CONN_MAX_AGE=0 -GEONODE_DB_CONN_TOUT=5 -DEFAULT_BACKEND_DATASTORE=datastore -BROKER_URL=redis://localhost:6379/0 -CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler -ASYNC_SIGNALS=False - -# Harvesting Monitoring configuration -HARVESTING_MONITOR_ENABLED=False -HARVESTING_MONITOR_DELAY=60 - -SITEURL=http://localhost:8000/ - -ALLOWED_HOSTS="['django', '*']" - -# Data Uploader -TIME_ENABLED=True -MOSAIC_ENABLED=False - -# ################# -# nginx -# HTTPD Server -# ################# -GEONODE_LB_HOST_IP=django -GEONODE_LB_PORT=8000 -GEOSERVER_LB_HOST_IP=geoserver -GEOSERVER_LB_PORT=8080 -NGINX_BASE_URL=http://localhost - -# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) -# port where the server can be reached on HTTPS -HTTP_HOST=localhost -HTTPS_HOST= - -HTTP_PORT=8000 -HTTPS_PORT=443 - -# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work -# with an ip) and it must be reachable from the outside. This can be one of the following : -# disabled : we do not get a certificate at all (a placeholder certificate will be used) -# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) -# production : we get a normal certificate (default) -LETSENCRYPT_MODE=disabled -# LETSENCRYPT_MODE=staging -# LETSENCRYPT_MODE=production - -RESOLVER=127.0.0.11 - -# ################# -# geoserver -# ################# -GEOSERVER_WEB_UI_LOCATION=http://localhost:8080/geoserver/ -GEOSERVER_PUBLIC_LOCATION=http://localhost:8080/geoserver/ -GEOSERVER_LOCATION=http://localhost:8080/geoserver/ -GEOSERVER_ADMIN_USER=admin -GEOSERVER_ADMIN_PASSWORD=geoserver - -OGC_REQUEST_TIMEOUT=30 -OGC_REQUEST_MAX_RETRIES=1 -OGC_REQUEST_BACKOFF_FACTOR=0.3 -OGC_REQUEST_POOL_MAXSIZE=10 -OGC_REQUEST_POOL_CONNECTIONS=10 - -# Java Options & Memory -ENABLE_JSONP=true -outFormat=text/javascript -GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' - -# ################# -# Security -# ################# -# Admin Settings -# -# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time -# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, -# otherwise the invoke updateadmin task will be skipped and the current password already stored -# in DB will honored. - -ADMIN_USERNAME=admin -ADMIN_PASSWORD=admin -ADMIN_EMAIL=admin@localhost - -# EMAIL Notifications -EMAIL_ENABLE=False -DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend -DJANGO_EMAIL_HOST=localhost -DJANGO_EMAIL_PORT=25 -DJANGO_EMAIL_HOST_USER= -DJANGO_EMAIL_HOST_PASSWORD= -DJANGO_EMAIL_USE_TLS=False -DJANGO_EMAIL_USE_SSL=False -DEFAULT_FROM_EMAIL='GeoNode ' - -# Session/Access Control -LOCKDOWN_GEONODE=False -X_FRAME_OPTIONS="SAMEORIGIN" -SESSION_EXPIRED_CONTROL_ENABLED=True -DEFAULT_ANONYMOUS_PERMISSIONS=download -DEFAULT_REGISTERED_MEMBERS_PERMISSIONS=download - -CORS_ALLOW_ALL_ORIGINS=True -GEOSERVER_CORS_ENABLED=True -GEOSERVER_CORS_ALLOWED_ORIGINS=* -GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS -GEOSERVER_CORS_ALLOWED_HEADERS=* - -# Users Registration -ACCOUNT_OPEN_SIGNUP=True -ACCOUNT_SIGNUP_FIELDS="['email*', 'username*', 'password1*', 'password2*']" -ACCOUNT_APPROVAL_REQUIRED=False -ACCOUNT_CONFIRM_EMAIL_ON_GET=False -ACCOUNT_EMAIL_VERIFICATION=none -ACCOUNT_LOGIN_METHODS="{'email', 'username'}" -AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True -AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True - -# OAuth2 -OAUTH2_API_KEY= -OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV -OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 - -# GeoNode APIs -API_LOCKDOWN=False - -# ################# -# Production and -# Monitoring -# ################# -DEBUG=True - -SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' - -CACHE_BUSTING_STATIC_ENABLED=False - -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache -MEMCACHED_LOCATION=127.0.0.1:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 -PERMISSION_CACHE_EXPIRATION_TIME=604800 -# -# Options for memcached binary, e.g. -vvv to log all requests and cache hits -# -MEMCACHED_OPTIONS= - -MAX_DOCUMENT_SIZE=200 -CLIENT_RESULTS_LIMIT=5 -API_LIMIT_PER_PAGE=1000 - -# GIS Client -GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore -MAPBOX_ACCESS_TOKEN= -GOOGLE_API_KEY= - -# Other Options/Contribs -MODIFY_TOPICCATEGORY=True -AVATAR_GRAVATAR_SSL=True -EXIF_ENABLED=True -CREATE_LAYER=True -FAVORITE_ENABLED=True - -# Advanced Workflow -RESOURCE_PUBLISHING=False -ADMIN_MODERATE_UPLOADS=False - -# PostgreSQL -POSTGRESQL_MAX_CONNECTIONS=200 - -# Common containers restart policy -RESTART_POLICY_CONDITION="on-failure" -RESTART_POLICY_DELAY="5s" -RESTART_POLICY_MAX_ATTEMPTS="3" -RESTART_POLICY_WINDOW=120s - -DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 -UPSERT_CHUNK_SIZE= 100 -UPSERT_LIMIT_ERROR_LOG=100 - -# Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=False \ No newline at end of file diff --git a/.env_local b/.env_local deleted file mode 100644 index 61a65461676..00000000000 --- a/.env_local +++ /dev/null @@ -1,214 +0,0 @@ -COMPOSE_PROJECT_NAME=geonode -# See https://github.com/containers/podman/issues/13889 -# DOCKER_BUILDKIT=0 -DOCKER_ENV=production -BACKUPS_VOLUME_DRIVER=local - -C_FORCE_ROOT=1 -FORCE_REINIT=false -INVOKE_LOG_STDOUT=true - -# LANGUAGE_CODE=it-it -# LANGUAGES=(('en-us','English'),('it-it','Italiano')) - -DJANGO_SETTINGS_MODULE=geonode.settings -GEONODE_INSTANCE_NAME=geonode - -# ################# -# backend -# ################# -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -GEONODE_DATABASE=geonode -GEONODE_DATABASE_USER=geonode -GEONODE_DATABASE_PASSWORD=geonode -GEONODE_GEODATABASE=geonode_data -GEONODE_GEODATABASE_USER=geonode -GEONODE_GEODATABASE_PASSWORD=geonode -GEONODE_DATABASE_SCHEMA=public -GEONODE_GEODATABASE_SCHEMA=public -DATABASE_HOST=localhost -DATABASE_PORT=5432 -DATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode -GEODATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode_data -GEONODE_DB_CONN_MAX_AGE=0 -GEONODE_DB_CONN_TOUT=5 -DEFAULT_BACKEND_DATASTORE=datastore -BROKER_URL=redis://localhost:6379/0 -CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler -ASYNC_SIGNALS=False - -# Monitoring configuration -HARVESTING_MONITOR_ENABLED=False -HARVESTING_MONITOR_DELAY=60 - -SITEURL=http://localhost:8000/ - -ALLOWED_HOSTS="['django', '*']" - -# Data Uploader -TIME_ENABLED=True -MOSAIC_ENABLED=False - -# ################# -# nginx -# HTTPD Server -# ################# -GEONODE_LB_HOST_IP=django -GEONODE_LB_PORT=8000 -GEOSERVER_LB_HOST_IP=geoserver -GEOSERVER_LB_PORT=8080 -NGINX_BASE_URL=https://localhost - -# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) -# port where the server can be reached on HTTPS -HTTP_HOST=localhost -HTTPS_HOST= - -HTTP_PORT=80 -HTTPS_PORT=443 - -# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work -# with an ip) and it must be reachable from the outside. This can be one of the following : -# disabled : we do not get a certificate at all (a placeholder certificate will be used) -# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) -# production : we get a normal certificate (default) -LETSENCRYPT_MODE=disabled -# LETSENCRYPT_MODE=staging -# LETSENCRYPT_MODE=production - -RESOLVER=127.0.0.11 - -# ################# -# geoserver -# ################# -GEOSERVER_WEB_UI_LOCATION=http://localhost:8080/geoserver/ -GEOSERVER_PUBLIC_LOCATION=http://localhost:8080/geoserver/ -GEOSERVER_LOCATION=http://localhost:8080/geoserver/ -GEOSERVER_ADMIN_USER=admin -GEOSERVER_ADMIN_PASSWORD=geoserver - -OGC_REQUEST_TIMEOUT=30 -OGC_REQUEST_MAX_RETRIES=1 -OGC_REQUEST_BACKOFF_FACTOR=0.3 -OGC_REQUEST_POOL_MAXSIZE=10 -OGC_REQUEST_POOL_CONNECTIONS=10 - -# Java Options & Memory -ENABLE_JSONP=true -outFormat=text/javascript -GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' - -# ################# -# Security -# ################# -# Admin Settings -# -# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time -# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, -# otherwise the invoke updateadmin task will be skipped and the current password already stored -# in DB will honored. - -ADMIN_USERNAME=admin -ADMIN_PASSWORD=admin -ADMIN_EMAIL=admin@localhost - -# EMAIL Notifications -EMAIL_ENABLE=False -DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend -DJANGO_EMAIL_HOST=localhost -DJANGO_EMAIL_PORT=25 -DJANGO_EMAIL_HOST_USER= -DJANGO_EMAIL_HOST_PASSWORD= -DJANGO_EMAIL_USE_TLS=False -DJANGO_EMAIL_USE_SSL=False -DEFAULT_FROM_EMAIL='GeoNode ' - -# Session/Access Control -LOCKDOWN_GEONODE=False -X_FRAME_OPTIONS="SAMEORIGIN" -SESSION_EXPIRED_CONTROL_ENABLED=True -DEFAULT_ANONYMOUS_PERMISSIONS=download -DEFAULT_REGISTERED_MEMBERS_PERMISSIONS=download - -CORS_ALLOW_ALL_ORIGINS=True -GEOSERVER_CORS_ENABLED=True -GEOSERVER_CORS_ALLOWED_ORIGINS=* -GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS -GEOSERVER_CORS_ALLOWED_HEADERS=* - -# Users Registration -ACCOUNT_OPEN_SIGNUP=True -ACCOUNT_SIGNUP_FIELDS="['email*', 'username*', 'password1*', 'password2*']" -ACCOUNT_APPROVAL_REQUIRED=False -ACCOUNT_CONFIRM_EMAIL_ON_GET=False -ACCOUNT_EMAIL_VERIFICATION=none -ACCOUNT_LOGIN_METHODS="{'email', 'username'}" -AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True -AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True - -# OAuth2 -OAUTH2_API_KEY= -OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV -OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 - -# GeoNode APIs -API_LOCKDOWN=False - -# ################# -# Production and -# Monitoring -# ################# -DEBUG=False - -SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' - -# STATIC_ROOT=/mnt/volumes/statics/static/ -# MEDIA_ROOT=/mnt/volumes/statics/uploaded/ - -CACHE_BUSTING_STATIC_ENABLED=False - -MEMCACHED_ENABLED=False -MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache -MEMCACHED_LOCATION=127.0.0.1:11211 -MEMCACHED_LOCK_EXPIRE=3600 -MEMCACHED_LOCK_TIMEOUT=10 -PERMISSION_CACHE_EXPIRATION_TIME=604800 -# -# Options for memcached binary, e.g. -vvv to log all requests and cache hits -# -MEMCACHED_OPTIONS= - -MAX_DOCUMENT_SIZE=200 -CLIENT_RESULTS_LIMIT=5 -API_LIMIT_PER_PAGE=1000 - -# GIS Client -GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore -MAPBOX_ACCESS_TOKEN= -GOOGLE_API_KEY= - -# Other Options/Contribs -MODIFY_TOPICCATEGORY=True -AVATAR_GRAVATAR_SSL=True -EXIF_ENABLED=True -CREATE_LAYER=True -FAVORITE_ENABLED=True - -# Advanced Workflow -RESOURCE_PUBLISHING=False -ADMIN_MODERATE_UPLOADS=False - -# PostgreSQL -POSTGRESQL_MAX_CONNECTIONS=200 - -# Common containers restart policy -RESTART_POLICY_CONDITION="on-failure" -RESTART_POLICY_DELAY="5s" -RESTART_POLICY_MAX_ATTEMPTS="3" -RESTART_POLICY_WINDOW=120s - -DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 - -# Enable or not the XLSX / XLS upload -XLSX_UPLOAD_ENABLED=False diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 8a402c56472..8e6cc8f5b8b 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,20 +28,20 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: docker compose -f docker-compose.yml -f docker-compose-test.yml up -d + run: APP_ENV=_test docker compose up -d - name: Setup test databases run: | - docker compose -f docker-compose.yml -f docker-compose-test.yml exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" - docker compose -f docker-compose.yml -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_postgres - docker compose -f docker-compose.yml -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode - docker compose -f docker-compose.yml -f docker-compose-test.yml exec db createdb -U postgres -T postgres test_geonode_data - docker compose -f docker-compose.yml -f docker-compose-test.yml exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" - docker compose -f docker-compose.yml -f docker-compose-test.yml exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" + APP_ENV=_test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" + APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_postgres + APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode + APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode_data + APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" + APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" - name: Run ${{ inputs.suite_name }} run: | - docker compose -f docker-compose.yml -f docker-compose-test.yml exec -T django bash -s <<'BASH' + APP_ENV=_test docker compose exec -T django bash -s <<'BASH' set -euo pipefail ${{ inputs.test_command }} BASH @@ -52,8 +52,8 @@ jobs: - name: Codecov run: | - docker compose -f docker-compose.yml -f docker-compose-test.yml exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" + APP_ENV=_test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" - name: Stop the stack if: always() - run: docker compose -f docker-compose.yml -f docker-compose-test.yml down -v + run: APP_ENV=_test docker compose down -v diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b6bdf456317..1fbd737bee9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: docker compose -f docker-compose.yml -f docker-compose-test.yml build --progress plain + run: APP_ENV=_test docker compose build --progress plain - name: Save built Docker images run: | diff --git a/docker-compose-test.yml b/docker-compose-test.yml deleted file mode 100644 index e94bc38e20f..00000000000 --- a/docker-compose-test.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '3.9' - -services: - - # Our custom django application. It includes Geonode. - django: - env_file: - - .env_test - environment: - - IS_CELERY=False - - # Celery worker that executes celery tasks created by Django. - celery: - env_file: - - .env_test - environment: - - IS_CELERY=True - - # Nginx is serving django static and media files and proxies to django and geonode - geonode: - env_file: - - .env_test - # Gets and installs letsencrypt certificates - letsencrypt: - env_file: - - .env_test - # Geoserver backend - geoserver: - env_file: - - .env_test - ports: - - "8080:8080" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a0d35841bba..ad5418f9db1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ x-common-django: dockerfile: Dockerfile restart: unless-stopped env_file: - - .env + - .env${APP_ENV} volumes: - .:/usr/src/geonode/ - statics:/mnt/volumes/statics @@ -68,7 +68,7 @@ services: image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - - .env + - .env${APP_ENV} environment: - RESOLVER=127.0.0.11 ports: @@ -85,7 +85,7 @@ services: image: geonode/letsencrypt:2.6.0-latest container_name: letsencrypt4${COMPOSE_PROJECT_NAME} env_file: - - .env + - .env${APP_ENV} volumes: - nginx-certificates:/geonode-certificates restart: unless-stopped @@ -101,7 +101,7 @@ services: timeout: 10s retries: 2 env_file: - - .env + - .env${APP_ENV} ports: - "8080:8080" volumes: @@ -134,7 +134,7 @@ services: command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: - - .env + - .env${APP_ENV} volumes: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups From f75caa12244f17d6d8f2bbaff8466b0f0ca23cf8 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Thu, 4 Jun 2026 15:35:10 +0200 Subject: [PATCH 03/16] remove warning --- docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ad5418f9db1..850f26c43c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ x-common-django: dockerfile: Dockerfile restart: unless-stopped env_file: - - .env${APP_ENV} + - .env${APP_ENV:-} volumes: - .:/usr/src/geonode/ - statics:/mnt/volumes/statics @@ -68,7 +68,7 @@ services: image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV} + - .env${APP_ENV:-} environment: - RESOLVER=127.0.0.11 ports: @@ -85,7 +85,7 @@ services: image: geonode/letsencrypt:2.6.0-latest container_name: letsencrypt4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV} + - .env${APP_ENV:-} volumes: - nginx-certificates:/geonode-certificates restart: unless-stopped @@ -101,7 +101,7 @@ services: timeout: 10s retries: 2 env_file: - - .env${APP_ENV} + - .env${APP_ENV:-} ports: - "8080:8080" volumes: @@ -134,7 +134,7 @@ services: command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV} + - .env${APP_ENV:-} volumes: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups From 38628c723dcd173cec94e889c9125d2a74013331 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Thu, 4 Jun 2026 15:43:39 +0200 Subject: [PATCH 04/16] remove warning --- docker-compose.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 850f26c43c1..ed32f0d998c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.9' - # Common Django template for GeoNode and Celery services below x-common-django: &default-common-django @@ -54,7 +52,7 @@ services: memcached: image: memcached:alpine container_name: memcached4${COMPOSE_PROJECT_NAME} - command: memcached ${MEMCACHED_OPTIONS} + command: memcached ${MEMCACHED_OPTIONS:-} restart: unless-stopped healthcheck: test: nc -z 127.0.0.1 11211 @@ -131,7 +129,7 @@ services: db: # use geonode official postgis 15 image image: geonode/postgis:15-3.5-latest - command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS}" + command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS:-200}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: - .env${APP_ENV:-} From 2679455cd944fb725628821b5ab9015380c9e449 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 5 Jun 2026 09:49:50 +0200 Subject: [PATCH 05/16] Chage the call of the .env file --- .env_test | 2 +- .github/workflows/run-test-suite.yml | 20 ++++++++++---------- .github/workflows/tests.yml | 2 +- docker-compose.yml | 16 ++++++++++++---- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.env_test b/.env_test index ae540ec6956..b377f2e85f8 100644 --- a/.env_test +++ b/.env_test @@ -1,7 +1,7 @@ COMPOSE_PROJECT_NAME=geonode # See https://github.com/containers/podman/issues/13889 # DOCKER_BUILDKIT=0 -DOCKER_ENV=production +DOCKER_ENV=test BACKUPS_VOLUME_DRIVER=local C_FORCE_ROOT=1 diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 8e6cc8f5b8b..7dd4d560ce9 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,20 +28,20 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: APP_ENV=_test docker compose up -d + run: APP_ENV=test docker compose up -d - name: Setup test databases run: | - APP_ENV=_test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" - APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_postgres - APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode - APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode_data - APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" - APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" + APP_ENV=test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" + APP_ENV=test docker compose exec db createdb -U postgres -T postgres test_postgres + APP_ENV=test docker compose exec db createdb -U postgres -T postgres test_geonode + APP_ENV=test docker compose exec db createdb -U postgres -T postgres test_geonode_data + APP_ENV=test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" + APP_ENV=test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" - name: Run ${{ inputs.suite_name }} run: | - APP_ENV=_test docker compose exec -T django bash -s <<'BASH' + APP_ENV=test docker compose exec -T django bash -s <<'BASH' set -euo pipefail ${{ inputs.test_command }} BASH @@ -52,8 +52,8 @@ jobs: - name: Codecov run: | - APP_ENV=_test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" + APP_ENV=test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" - name: Stop the stack if: always() - run: APP_ENV=_test docker compose down -v + run: APP_ENV=test docker compose down -v diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fbd737bee9..a84e6bffe93 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: APP_ENV=_test docker compose build --progress plain + run: APP_ENV=test docker compose build --progress plain - name: Save built Docker images run: | diff --git a/docker-compose.yml b/docker-compose.yml index ed32f0d998c..7e80703211a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,7 +66,9 @@ services: image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV:-} + - .env + - path: .env_${APP_ENV:-} + required: false environment: - RESOLVER=127.0.0.11 ports: @@ -83,7 +85,9 @@ services: image: geonode/letsencrypt:2.6.0-latest container_name: letsencrypt4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV:-} + - .env + - path: .env_${APP_ENV:-} + required: false volumes: - nginx-certificates:/geonode-certificates restart: unless-stopped @@ -99,7 +103,9 @@ services: timeout: 10s retries: 2 env_file: - - .env${APP_ENV:-} + - .env + - path: .env_${APP_ENV:-} + required: false ports: - "8080:8080" volumes: @@ -132,7 +138,9 @@ services: command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS:-200}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV:-} + - .env + - path: .env_${APP_ENV:-} + required: false volumes: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups From ce488b9b69febc1c819ec93c9f1aaad24704eae8 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 5 Jun 2026 10:29:36 +0200 Subject: [PATCH 06/16] Chage the call of the .env file --- docker-compose.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7e80703211a..ed32f0d998c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -66,9 +66,7 @@ services: image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - - .env - - path: .env_${APP_ENV:-} - required: false + - .env${APP_ENV:-} environment: - RESOLVER=127.0.0.11 ports: @@ -85,9 +83,7 @@ services: image: geonode/letsencrypt:2.6.0-latest container_name: letsencrypt4${COMPOSE_PROJECT_NAME} env_file: - - .env - - path: .env_${APP_ENV:-} - required: false + - .env${APP_ENV:-} volumes: - nginx-certificates:/geonode-certificates restart: unless-stopped @@ -103,9 +99,7 @@ services: timeout: 10s retries: 2 env_file: - - .env - - path: .env_${APP_ENV:-} - required: false + - .env${APP_ENV:-} ports: - "8080:8080" volumes: @@ -138,9 +132,7 @@ services: command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS:-200}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: - - .env - - path: .env_${APP_ENV:-} - required: false + - .env${APP_ENV:-} volumes: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups From 62ee200c2033b5053cd4027679c2a29865e11b74 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 5 Jun 2026 10:30:14 +0200 Subject: [PATCH 07/16] Chage the call of the .env file --- .github/workflows/run-test-suite.yml | 20 ++++++++++---------- .github/workflows/tests.yml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 7dd4d560ce9..8e6cc8f5b8b 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,20 +28,20 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: APP_ENV=test docker compose up -d + run: APP_ENV=_test docker compose up -d - name: Setup test databases run: | - APP_ENV=test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" - APP_ENV=test docker compose exec db createdb -U postgres -T postgres test_postgres - APP_ENV=test docker compose exec db createdb -U postgres -T postgres test_geonode - APP_ENV=test docker compose exec db createdb -U postgres -T postgres test_geonode_data - APP_ENV=test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" - APP_ENV=test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" + APP_ENV=_test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" + APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_postgres + APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode + APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode_data + APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" + APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" - name: Run ${{ inputs.suite_name }} run: | - APP_ENV=test docker compose exec -T django bash -s <<'BASH' + APP_ENV=_test docker compose exec -T django bash -s <<'BASH' set -euo pipefail ${{ inputs.test_command }} BASH @@ -52,8 +52,8 @@ jobs: - name: Codecov run: | - APP_ENV=test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" + APP_ENV=_test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" - name: Stop the stack if: always() - run: APP_ENV=test docker compose down -v + run: APP_ENV=_test docker compose down -v diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a84e6bffe93..1fbd737bee9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: APP_ENV=test docker compose build --progress plain + run: APP_ENV=_test docker compose build --progress plain - name: Save built Docker images run: | From 699222174c91ce4b199501f8b58c564873301e6a Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Fri, 5 Jun 2026 17:38:36 +0200 Subject: [PATCH 08/16] remove managed.py --- docker-build.sh | 1 - docker-purge.sh | 2 -- docs/src/setup/bare/project-bare-installation.md | 4 ---- manage_dev.sh | 5 ----- manage_local.sh | 5 ----- 5 files changed, 17 deletions(-) delete mode 100755 docker-build.sh delete mode 100755 docker-purge.sh delete mode 100755 manage_dev.sh delete mode 100644 manage_local.sh diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index 64fba5a14de..00000000000 --- a/docker-build.sh +++ /dev/null @@ -1 +0,0 @@ -docker-compose build --no-cache; docker-compose stop; docker-compose up -d; docker system prune -a \ No newline at end of file diff --git a/docker-purge.sh b/docker-purge.sh deleted file mode 100755 index f66271f9983..00000000000 --- a/docker-purge.sh +++ /dev/null @@ -1,2 +0,0 @@ -docker kill $(docker ps -q); docker rm $(docker ps -a -q); docker rmi $(docker images -q); docker volume ls -qf dangling=true | xargs -r docker volume rm; docker system prune -a -docker volume prune diff --git a/docs/src/setup/bare/project-bare-installation.md b/docs/src/setup/bare/project-bare-installation.md index 68d3d517e0c..3c86c4fb2d9 100644 --- a/docs/src/setup/bare/project-bare-installation.md +++ b/docs/src/setup/bare/project-bare-installation.md @@ -71,10 +71,6 @@ pip install -e src/ --upgrade # Install GDAL Utilities for Python pip install pygdal=="`gdal-config --version`.*" -# Dev scripts -mv .override_dev_env.sample src/.override_dev_env -mv src/manage_dev.sh.sample src/manage_dev.sh -mv src/paver_dev.sh.sample src/paver_dev.sh ``` ### PostGIS database setup diff --git a/manage_dev.sh b/manage_dev.sh deleted file mode 100755 index 408f89ddb59..00000000000 --- a/manage_dev.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -a -. ./.env_dev -set +a - -python manage.py $@ diff --git a/manage_local.sh b/manage_local.sh deleted file mode 100644 index 609887a9d10..00000000000 --- a/manage_local.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -a -. ./.env_local -set +a - -python manage.py $@ From 021ff9e01995314df201e23f09732f76d05ad0f8 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Tue, 9 Jun 2026 16:53:37 +0200 Subject: [PATCH 09/16] cleanup unused files --- .codex | 0 Dockerfile | 3 --- celery.sh | 3 --- celery_dev.sh | 5 ----- flake8.sh | 4 ---- flake8.txt | 7 ------- pyproject.toml | 6 ++++++ pytest.ini | 4 ---- 8 files changed, 6 insertions(+), 26 deletions(-) delete mode 100644 .codex delete mode 100755 celery.sh delete mode 100755 celery_dev.sh delete mode 100755 flake8.sh delete mode 100644 flake8.txt delete mode 100644 pytest.ini diff --git a/.codex b/.codex deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Dockerfile b/Dockerfile index 25cfe784252..6df0fc1e1a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,6 @@ RUN chmod +x /usr/bin/wait-for-databases RUN chmod +x /usr/src/geonode/tasks.py \ && chmod +x /usr/src/geonode/entrypoint.sh -COPY celery.sh /usr/bin/celery-commands -RUN chmod +x /usr/bin/celery-commands - COPY celery-cmd /usr/bin/celery-cmd RUN chmod +x /usr/bin/celery-cmd diff --git a/celery.sh b/celery.sh deleted file mode 100755 index db816729f97..00000000000 --- a/celery.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -nohup celery -A geonode.celery_app:app beat -l DEBUG -f /var/log/celery.log &>/dev/null & -nohup celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state --scheduler=celery.beat:PersistentScheduler --loglevel=INFO --concurrency=2 -n worker1@%h -f /var/log/celery.log &>/dev/null \ No newline at end of file diff --git a/celery_dev.sh b/celery_dev.sh deleted file mode 100755 index 2335c83c098..00000000000 --- a/celery_dev.sh +++ /dev/null @@ -1,5 +0,0 @@ -set -a -. ./.env_dev -set +a - -celery -A geonode.celery_app:app worker --without-gossip --without-mingle -Ofair -B -E --statedb=worker.state --scheduler=celery.beat:PersistentScheduler --loglevel=DEBUG --concurrency=2 -n worker1@%h diff --git a/flake8.sh b/flake8.sh deleted file mode 100755 index 5b7fab077c1..00000000000 --- a/flake8.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -flake8 geonode diff --git a/flake8.txt b/flake8.txt deleted file mode 100644 index 3063e84500e..00000000000 --- a/flake8.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Check formatting with - -flake8 --config=setup.cfg - -# Auto-fix formatting with - -autopep8 --global-config setup.cfg -i -a -a [files [files ...]] diff --git a/pyproject.toml b/pyproject.toml index 248d67955d3..1339948e899 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -206,3 +206,9 @@ exclude = ''' | node_modules )/ ''' + + + +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "geonode.settings" +python_files = "tests.py test_*.py *_tests.py" \ No newline at end of file diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index b5ee808b7ba..00000000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -DJANGO_SETTINGS_MODULE = geonode.settings -# Adjust python_files to match your test file naming convention -python_files = tests.py test_*.py *_tests.py \ No newline at end of file From 5bff924d0b0acf7bf42d0383c7be90f00963a865 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Tue, 9 Jun 2026 17:41:10 +0200 Subject: [PATCH 10/16] refactor test execution --- .github/workflows/tests.yml | 14 +- dev_config.yml | 8 - pavement.py | 1113 ----------------------------------- test.sh | 40 ++ tests/pavement.py | 1113 ----------------------------------- tests/test.sh | 8 - tests/test_api_v2.sh | 4 - tests/test_csw.sh | 27 - tests/test_dev.sh | 8 - tests/test_integration.sh | 24 - tests/test_oauth2.sh | 9 - tests/test_upload.sh | 26 - 12 files changed, 47 insertions(+), 2347 deletions(-) delete mode 100644 dev_config.yml delete mode 100644 pavement.py create mode 100755 test.sh delete mode 100644 tests/pavement.py delete mode 100755 tests/test.sh delete mode 100755 tests/test_api_v2.sh delete mode 100755 tests/test_csw.sh delete mode 100755 tests/test_dev.sh delete mode 100755 tests/test_integration.sh delete mode 100755 tests/test_oauth2.sh delete mode 100755 tests/test_upload.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1fbd737bee9..3a3a41617a9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,7 +42,7 @@ jobs: with: suite_name: "Smoke Tests" test_command: > - ./tests/test.sh geonode.tests.smoke geonode.tests.test_rest_api geonode.tests.test_search + ./test.sh geonode.tests.smoke geonode.tests.test_rest_api geonode.tests.test_search geonode.tests.test_utils geonode.tests.test_headers --duration=30 --failfast main: @@ -53,7 +53,7 @@ jobs: test_command: > TESTS=$(python -c 'import sys; from geonode import settings; print(" ".join([a + ".tests" for a in settings.GEONODE_APPS if "security" not in a and "geoserver" not in a and "upload" not in a]))') && - ./tests/test.sh $TESTS geonode.thumbs.tests geonode.people.tests geonode.people.socialaccount.providers.geonode_openid_connect.tests --duration=30 --failfast + ./test.sh $TESTS geonode.thumbs.tests geonode.people.tests geonode.people.socialaccount.providers.geonode_openid_connect.tests --duration=30 --failfast security: uses: ./.github/workflows/run-test-suite.yml @@ -63,7 +63,7 @@ jobs: test_command: > TESTS=$(python -c 'import sys; from geonode import settings; print(" ".join([a + ".tests" for a in settings.GEONODE_APPS if "security" in a]))') && - ./tests/test.sh $TESTS --duration=30 --failfast + ./test.sh $TESTS --duration=30 --failfast gis_backend: uses: ./.github/workflows/run-test-suite.yml @@ -73,7 +73,7 @@ jobs: test_command: > TESTS=$(python -c 'import sys; from geonode import settings; print(" ".join([a + ".tests" for a in settings.GEONODE_APPS if "geoserver" in a]))') && - ./tests/test.sh $TESTS --duration=30 --failfast + ./test.sh $TESTS --duration=30 --failfast rest_apis: uses: ./.github/workflows/run-test-suite.yml @@ -81,7 +81,7 @@ jobs: with: suite_name: "REST API Tests" test_command: > - ./tests/test.sh geonode.api.tests geonode.base.api.tests geonode.layers.api.tests + ./test.sh geonode.api.tests geonode.base.api.tests geonode.layers.api.tests geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests --duration=30 --failfast csw: @@ -90,7 +90,7 @@ jobs: with: suite_name: "CSW Tests" test_command: > - ./tests/test.sh geonode.tests.csw geonode.catalogue.backends.tests --duration=30 --failfast + ./test.sh geonode.tests.csw geonode.catalogue.backends.tests --duration=30 --failfast upload: uses: ./.github/workflows/run-test-suite.yml @@ -98,7 +98,7 @@ jobs: with: suite_name: "Upload Tests" test_command: > - ./tests/test.sh geonode.upload --duration=30 --failfast + ./test.sh geonode.upload --duration=30 --failfast # ------------------------------------------------- # CLEANUP JOB: REMOVE ALL ARTIFACTS AT THE END diff --git a/dev_config.yml b/dev_config.yml deleted file mode 100644 index 3fa02cd768e..00000000000 --- a/dev_config.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -GEOSERVER_URL: "https://artifacts.geonode.org/geoserver/2.23.0/geoserver.war" -DATA_DIR_URL: "https://artifacts.geonode.org/geoserver/2.23.0/geonode-geoserver-ext-web-app-data.zip" -JETTY_RUNNER_URL: "https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-runner/9.4.31.v20200723/jetty-runner-9.4.31.v20200723.jar" -WINDOWS: - py2exe: "http://downloads.sourceforge.net/project/py2exe/py2exe/0.6.9/py2exe-0.6.9.win32-py2.7.exe" - pyproj: "https://pyproj.googlecode.com/files/pyproj-1.9.3.win32-py2.7.exe" - lxml: "https://pypi.python.org/packages/2.7/l/lxml/lxml-3.6.0.win32-py2.7.exe" diff --git a/pavement.py b/pavement.py deleted file mode 100644 index 6028119564d..00000000000 --- a/pavement.py +++ /dev/null @@ -1,1113 +0,0 @@ -######################################################################### -# -# Copyright (C) 2018 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### - -import fileinput -import glob -import os -import re -import shutil -import subprocess -import signal -import sys -import time -import pytz -import logging -import datetime -from dateutil.parser import parse as parsedate - -from urllib.parse import urlparse -from urllib.request import urlopen, Request - -import zipfile -from tqdm import tqdm -import requests -import math -import psutil - -import yaml -from paver.easy import ( - BuildFailure, - call_task, - cmdopts, - info, - needs, - path, - sh, - task, -) -from setuptools.command import easy_install - -try: - from paver.path import pushd -except ImportError: - from paver.easy import pushd - -from geonode.settings import ( - on_travis, - core_tests, - internal_apps_tests, - integration_tests, - integration_server_tests, - integration_upload_tests, - integration_csw_tests, - integration_bdd_tests, - INSTALLED_APPS, - GEONODE_CORE_APPS, - GEONODE_INTERNAL_APPS, - GEONODE_APPS, - OGC_SERVER, - ASYNC_SIGNALS, - CELERY_BEAT_SCHEDULER, -) - -try: - from geonode.settings import TEST_RUNNER_KEEPDB, TEST_RUNNER_PARALLEL - - _keepdb = "--keepdb" if TEST_RUNNER_KEEPDB else "" - _parallel = f"--parallel={TEST_RUNNER_PARALLEL}" if TEST_RUNNER_PARALLEL else "" -except Exception: - _keepdb = "" - _parallel = "" - -assert sys.version_info >= (2, 6), SystemError("GeoNode Build requires python 2.6 or better") - -dev_config = None -with open("dev_config.yml") as f: - dev_config = yaml.load(f, Loader=yaml.Loader) - - -logger = logging.getLogger(__name__) - - -def grab(src, dest, name): - src, dest, name = map(str, (src, dest, name)) - logger.info(f" src, dest, name --> {src} {dest} {name}") - - if not os.path.exists(dest): - logger.info(f"Downloading {name}") - elif not zipfile.is_zipfile(dest): - logger.info(f"Downloading {name} (corrupt file)") - elif not src.startswith("file://"): - r = requests.head(src) - file_time = datetime.datetime.fromtimestamp(os.path.getmtime(dest)) - url_time = file_time - for _k in ["last-modified", "Date"]: - if _k in r.headers: - url_time = r.headers[_k] - url_date = parsedate(url_time) - utc = pytz.utc - url_date = url_date.replace(tzinfo=utc) - file_time = file_time.replace(tzinfo=utc) - if url_date < file_time: - # Do not download if older than the local one - return - logger.info(f"Downloading updated {name}") - - # Local file does not exist or remote one is newer - if src.startswith("file://"): - src2 = src.replace("file://", "") - if not os.path.exists(src2): - logger.info(f"Source location ({src2}) does not exist") - else: - logger.info(f"Copying local file from {src2}") - shutil.copyfile(src2, dest) - else: - # urlretrieve(str(src), str(dest)) - # Streaming, so we can iterate over the response. - r = requests.get(src, stream=True, timeout=10, verify=False) - # Total size in bytes. - total_size = int(r.headers.get("content-length", 0)) - logger.info(f"Requesting {src}") - block_size = 1024 - wrote = 0 - with open("output.bin", "wb") as f: - for data in tqdm( - r.iter_content(block_size), total=math.ceil(total_size // block_size), unit="KB", unit_scale=False - ): - wrote += len(data) - f.write(data) - logger.info(f" total_size [{total_size}] / wrote [{wrote}] ") - if total_size != 0 and wrote != total_size: - logger.error( - f"ERROR, something went wrong. Data could not be written. Expected to write {wrote} but wrote {total_size} instead" - ) - else: - shutil.move("output.bin", dest) - try: - # Cleaning up - os.remove("output.bin") - except OSError: - pass - - -@task -@cmdopts( - [ - ("geoserver=", "g", "The location of the geoserver build (.war file)."), - ("jetty=", "j", "The location of the Jetty Runner (.jar file)."), - ("force_exec=", "", "Force GeoServer Setup."), - ] -) -def setup_geoserver(options): - """Prepare a testing instance of GeoServer.""" - # only start if using Geoserver backend - if "geonode.geoserver" not in INSTALLED_APPS: - return - if on_travis and not options.get("force_exec", False): - """Will make use of the docker container for the Integration Tests""" - return - else: - download_dir = path("downloaded") - if not download_dir.exists(): - download_dir.makedirs() - geoserver_dir = path("geoserver") - geoserver_bin = download_dir / os.path.basename(urlparse(dev_config["GEOSERVER_URL"]).path) - jetty_runner = download_dir / os.path.basename(urlparse(dev_config["JETTY_RUNNER_URL"]).path) - geoserver_data = download_dir / os.path.basename(urlparse(dev_config["DATA_DIR_URL"]).path) - grab(options.get("geoserver", dev_config["GEOSERVER_URL"]), geoserver_bin, "geoserver binary") - grab(options.get("jetty", dev_config["JETTY_RUNNER_URL"]), jetty_runner, "jetty runner") - grab(options.get("geoserver data", dev_config["DATA_DIR_URL"]), geoserver_data, "geoserver data-dir") - - if not geoserver_dir.exists(): - geoserver_dir.makedirs() - - webapp_dir = geoserver_dir / "geoserver" - if not webapp_dir: - webapp_dir.makedirs() - - logger.info("extracting geoserver") - z = zipfile.ZipFile(geoserver_bin, "r") - z.extractall(webapp_dir) - - logger.info("extracting geoserver data dir") - z = zipfile.ZipFile(geoserver_data, "r") - z.extractall(geoserver_dir) - - _configure_data_dir() - - -def _configure_data_dir(): - try: - config = path("geoserver/data/global.xml") - with open(config) as f: - xml = f.read() - m = re.search("proxyBaseUrl>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8080/geoserver{xml[m.end(1):]}" - with open(config, "w") as f: - f.write(xml) - except Exception as e: - print(e) - - try: - config = path("geoserver/data/security/filter/geonode-oauth2/config.xml") - with open(config) as f: - xml = f.read() - m = re.search("accessTokenUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/o/token/{xml[m.end(1):]}" - m = re.search("userAuthorizationUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/o/authorize/{xml[m.end(1):]}" - m = re.search("redirectUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8080/geoserver/index.html{xml[m.end(1):]}" - m = re.search("checkTokenEndpointUrl>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/api/o/v4/tokeninfo/{xml[m.end(1):]}" - m = re.search("logoutUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/account/logout/{xml[m.end(1):]}" - with open(config, "w") as f: - f.write(xml) - except Exception as e: - print(e) - - try: - config = path("geoserver/data/security/role/geonode REST role service/config.xml") - with open(config) as f: - xml = f.read() - m = re.search("baseUrl>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000{xml[m.end(1):]}" - with open(config, "w") as f: - f.write(xml) - except Exception as e: - print(e) - - -@task -def static(options): - with pushd("geonode/static"): - sh("grunt production") - - -@task -@needs( - [ - "setup_geoserver", - ] -) -def setup(options): - """Get dependencies and prepare a GeoNode development environment.""" - info( - "GeoNode development environment successfully set up." - "If you have not set up an administrative account," - ' please do so now. Use "paver start" to start up the server.' - ) - - -def grab_winfiles(url, dest, packagename): - # Add headers - headers = {"User-Agent": "Mozilla 5.10"} - request = Request(url, None, headers) - response = urlopen(request) - with open(dest, "wb") as writefile: - writefile.write(response.read()) - - -@task -def win_install_deps(options): - """ - Install all Windows Binary automatically - This can be removed as wheels become available for these packages - """ - download_dir = path("downloaded").abspath() - if not download_dir.exists(): - download_dir.makedirs() - win_packages = { - # required by transifex-client - "Py2exe": dev_config["WINDOWS"]["py2exe"], - # the wheel 1.9.4 installs but pycsw wants 1.9.3, which fails to compile - # when pycsw bumps their pyproj to 1.9.4 this can be removed. - "PyProj": dev_config["WINDOWS"]["pyproj"], - "lXML": dev_config["WINDOWS"]["lxml"], - } - failed = False - for package, url in win_packages.items(): - tempfile = download_dir / os.path.basename(url) - logger.info(f"Installing file ... {tempfile}") - grab_winfiles(url, tempfile, package) - try: - easy_install.main([tempfile]) - except Exception as e: - failed = True - logger.error("install failed with error: ", e) - os.remove(tempfile) - if failed and sys.maxsize > 2**32: - logger.error("64bit architecture is not currently supported") - logger.error("try finding the 64 binaries for py2exe, and pyproj") - elif failed: - logger.error("install failed for py2exe, and/or pyproj") - else: - print("Windows dependencies now complete. Run pip install -e geonode --use-mirrors") - - -@task -@cmdopts([("version=", "v", "Legacy GeoNode version of the existing database.")]) -def upgradedb(options): - """ - Add 'fake' data migrations for existing tables from legacy GeoNode versions - """ - version = options.get("version") - if version in {"1.1", "1.2"}: - sh("python -W ignore manage.py migrate maps 0001 --fake") - sh("python -W ignore manage.py migrate avatar 0001 --fake") - elif version is None: - print("Please specify your GeoNode version") - else: - print(f"Upgrades from version {version} are not yet supported.") - - -@task -@cmdopts([("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE")]) -def sync(options): - """ - Run the migrate and migrate management commands to create and migrate a DB - """ - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - - sh(f"{settings} python -W ignore manage.py makemigrations --noinput") - sh(f"{settings} python -W ignore manage.py migrate --noinput") - sh(f"{settings} python -W ignore manage.py loaddata sample_admin.json") - sh(f"{settings} python -W ignore manage.py loaddata geonode/base/fixtures/default_oauth_apps.json") - sh(f"{settings} python -W ignore manage.py loaddata geonode/base/fixtures/initial_data.json") - sh(f"{settings} python -W ignore manage.py set_all_datasets_alternate") - sh(f"{settings} python -W ignore manage.py collectstatic --noinput") - - -@task -def package(options): - """ - Creates a tarball to use for building the system elsewhere - """ - import tarfile - import geonode - - version = geonode.get_version() - # Use GeoNode's version for the package name. - pkgname = f"GeoNode-{version}-all" - - # Create the output directory. - out_pkg = path(pkgname) - out_pkg_tar = path(f"{pkgname}.tar.gz") - - # Create a distribution in zip format for the geonode python package. - dist_dir = path("dist") - dist_dir.rmtree() - sh("python setup.py sdist --formats=zip") - - with pushd("package"): - # Delete old tar files in that directory - for f in glob.glob("GeoNode*.tar.gz"): - old_package = path(f) - if old_package != out_pkg_tar: - old_package.remove() - - if out_pkg_tar.exists(): - info(f"There is already a package for version {version}") - return - - # Clean anything that is in the oupout package tree. - out_pkg.rmtree() - out_pkg.makedirs() - - support_folder = path("support") - install_file = path("install.sh") - - # And copy the default files from the package folder. - justcopy(support_folder, out_pkg / "support") - justcopy(install_file, out_pkg) - - geonode_dist = path("..") / "dist" / f"GeoNode-{version}.zip" - justcopy(geonode_dist, out_pkg) - - # Create a tar file with all files in the output package folder. - tar = tarfile.open(out_pkg_tar, "w:gz") - for file in out_pkg.walkfiles(): - tar.add(file) - - # Add the README with the license and important links to documentation. - tar.add("README", arcname=f"{out_pkg}/README.rst") - tar.close() - - # Remove all the files in the temporary output package directory. - out_pkg.rmtree() - - # Report the info about the new package. - info(f"{out_pkg_tar.abspath()} created") - - -@task -@needs(["start_geoserver", "start_django"]) -@cmdopts( - [ - ("bind=", "b", "Bind server to provided IP address and port number."), - ("java_path=", "j", "Full path to java install for Windows"), - ("foreground", "f", "Do not run in background but in foreground"), - ("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE"), - ], - share_with=["start_django", "start_geoserver"], -) -def start(options): - """ - Start GeoNode (Django, GeoServer & Client) - """ - info("GeoNode is now available.") - - -@task -def stop_django(options): - """ - Stop the GeoNode Django application - """ - if ASYNC_SIGNALS: - kill("python", "celery") - kill("celery", "worker") - kill("python", "runserver") - kill("python", "runmessaging") - - -@task -@cmdopts([("force_exec=", "", "Force GeoServer Stop.")]) -def stop_geoserver(options): - """ - Stop GeoServer - """ - # we use docker-compose for integration tests - if on_travis and not options.get("force_exec", False): - return - - # only start if using Geoserver backend - if "geonode.geoserver" not in INSTALLED_APPS: - return - kill("java", "geoserver") - - # Kill process. - try: - # proc = subprocess.Popen("ps -ef | grep -i -e '[j]ava\|geoserver' | - # awk '{print $2}'", - proc = subprocess.Popen( - "ps -ef | grep -i -e 'geoserver' | awk '{print $2}'", shell=True, stdout=subprocess.PIPE - ) - for pid in map(int, proc.stdout): - info(f"Stopping geoserver (process number {pid})") - os.kill(pid, signal.SIGKILL) - - # Check if the process that we killed is alive. - killed, alive = psutil.wait_procs([psutil.Process(pid=pid)], timeout=30) - for p in alive: - p.kill() - except Exception as e: - info(e) - - -@task -@needs( - [ - "stop_geoserver", - ] -) -def stop(options): - """ - Stop GeoNode - """ - # windows needs to stop the geoserver first b/c we can't tell which python - # is running, so we kill everything - info("Stopping GeoNode ...") - stop_django(options) - - -@task -@cmdopts([("bind=", "b", "Bind server to provided IP address and port number.")]) -def start_django(options): - """ - Start the GeoNode Django application - """ - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - bind = options.get("bind", "0.0.0.0:8000") - port = bind.split(":")[1] - foreground = "" if options.get("foreground", False) else "&" - sh(f"{settings} python -W ignore manage.py runserver {bind} {foreground}") - - if ASYNC_SIGNALS: - sh( - f"{settings} celery -A geonode.celery_app:app worker --autoscale=20,10 --without-gossip --without-mingle -Ofair -B -E \ - --statedb=/tmp/worker.state --scheduler={CELERY_BEAT_SCHEDULER} --loglevel=DEBUG \ - --concurrency=10 --max-tasks-per-child=10 -n worker1@%h -f celery.log {foreground}" - ) - sh(f"{settings} python -W ignore manage.py runmessaging {foreground}") - - # wait for Django to start - started = waitfor(f"http://localhost:{port}") - if not started: - info("Django never started properly or timed out.") - sys.exit(1) - - -@task -def start_messaging(options): - """ - Start the GeoNode messaging server - """ - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - foreground = "" if options.get("foreground", False) else "&" - sh(f"{settings} python -W ignore manage.py runmessaging {foreground}") - - -@task -@cmdopts([("java_path=", "j", "Full path to java install for Windows"), ("force_exec=", "", "Force GeoServer Start.")]) -def start_geoserver(options): - """ - Start GeoServer with GeoNode extensions - """ - # we use docker-compose for integration tests - if on_travis and not options.get("force_exec", False): - return - - # only start if using Geoserver backend - if "geonode.geoserver" not in INSTALLED_APPS: - return - - GEOSERVER_BASE_URL = OGC_SERVER["default"]["LOCATION"] - url = GEOSERVER_BASE_URL - - if urlparse(GEOSERVER_BASE_URL).hostname != "localhost": - logger.warning("Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'") - - if not GEOSERVER_BASE_URL.endswith("/"): - logger.error("Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'") - sys.exit(1) - - download_dir = path("downloaded").abspath() - jetty_runner = download_dir / os.path.basename(dev_config["JETTY_RUNNER_URL"]) - data_dir = path("geoserver/data").abspath() - geofence_dir = path("geoserver/data/geofence").abspath() - web_app = path("geoserver/geoserver").abspath() - log_file = path("geoserver/jetty.log").abspath() - config = path("scripts/misc/jetty-runner.xml").abspath() - jetty_port = urlparse(GEOSERVER_BASE_URL).port - - import socket - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - socket_free = True - try: - s.bind(("127.0.0.1", jetty_port)) - except OSError as e: - socket_free = False - if e.errno == 98: - info(f"Port {jetty_port} is already in use") - else: - info(f"Something else raised the socket.error exception while checking port {jetty_port}") - print(e) - finally: - s.close() - - if socket_free: - # @todo - we should not have set workdir to the datadir but a bug in geoserver - # prevents geonode security from initializing correctly otherwise - with pushd(data_dir): - javapath = "java" - if on_travis: - sh( - "sudo apt install -y openjdk-8-jre openjdk-8-jdk;" - " sudo update-java-alternatives --set java-1.8.0-openjdk-amd64;" - ' export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::");' - " export PATH=$JAVA_HOME'bin/java':$PATH;" - ) - # import subprocess - # result = subprocess.run(['update-alternatives', '--list', 'java'], stdout=subprocess.PIPE) - # javapath = result.stdout - javapath = "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java" - loggernullpath = os.devnull - - # checking if our loggernullpath exists and if not, reset it to - # something manageable - if loggernullpath == "nul": - try: - open("../../downloaded/null.txt", "w+").close() - except OSError: - print( - "Chances are that you have Geoserver currently running. You " - "can either stop all servers with paver stop or start only " - "the django application with paver start_django." - ) - sys.exit(1) - loggernullpath = "../../downloaded/null.txt" - - try: - sh(("%(javapath)s -version") % locals()) - except Exception: - logger.warning("Java was not found in your path. Trying some other options: ") - javapath_opt = None - if os.environ.get("JAVA_HOME", None): - logger.info("Using the JAVA_HOME environment variable") - javapath_opt = os.path.join(os.path.abspath(os.environ["JAVA_HOME"]), "bin", "java.exe") - elif options.get("java_path"): - javapath_opt = options.get("java_path") - else: - logger.critical( - "Paver cannot find java in the Windows Environment. " - "Please provide the --java_path flag with your full path to " - "java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" - ) - sys.exit(1) - # if there are spaces - javapath = f'START /B "" "{javapath_opt}"' - - sh( - "%(javapath)s -Xms512m -Xmx2048m -server -Dgwc.context.suffix=gwc -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m" - " -DGEOSERVER_DATA_DIR=%(data_dir)s" - " -DGEOSERVER_CSRF_DISABLED=true" - " -Dgeofence.dir=%(geofence_dir)s" - " -Djava.awt.headless=true" - # ' -Dgeofence-ovr=geofence-datasource-ovr.properties' - # workaround for JAI sealed jar issue and jetty classloader - # ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' - " -jar %(jetty_runner)s" - " --port %(jetty_port)i" - " --log %(log_file)s" - " %(config)s" - " > %(loggernullpath)s &" % locals() - ) - - info(f"Starting GeoServer on {url}") - - # wait for GeoServer to start - started = waitfor(url) - info(f"The logs are available at {log_file}") - - if not started: - # If applications did not start in time we will give the user a chance - # to inspect them and stop them manually. - info("GeoServer never started properly or timed out." "It may still be running in the background.") - sys.exit(1) - - -@task -def test(options): - """ - Run GeoNode's Unit Test Suite - """ - if on_travis: - if core_tests: - _apps = GEONODE_CORE_APPS - if internal_apps_tests: - _apps = GEONODE_INTERNAL_APPS - else: - _apps = GEONODE_APPS - - _apps_to_test = [] - for _app in _apps: - if _app and len(_app) > 0 and "geonode" in _app: - _apps_to_test.append(_app) - sh( - f"{options.get('prefix')} manage.py test geonode.tests.smoke \ -{('.tests '.join(_apps_to_test))}.tests --noinput {_keepdb} {_parallel}" - ) - - -@task -@cmdopts([("local=", "l", "Set to True if running bdd tests locally")]) -def test_bdd(options): - """ - Run GeoNode's BDD Test Suite - """ - local = str2bool(options.get("local", "false")) - if local: - call_task("reset_hard") - - call_task("setup") - call_task("sync") - if local: - sh("sleep 30") - - info("GeoNode is now available, running the bdd tests now.") - sh("py.test") - - -@task -def test_javascript(options): - with pushd("geonode/static/geonode"): - sh("./run-tests.sh") - - -@task -@cmdopts( - [ - ("name=", "n", "Run specific tests."), - ("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE"), - ("local=", "l", "Set to True if running bdd tests locally"), - ] -) -def test_integration(options): - """ - Run GeoNode's Integration test suite against the external apps - """ - prefix = options.get("prefix") - local = str2bool(options.get("local", "false")) - if local: - call_task("stop_geoserver") - _reset() - - name = options.get("name", None) - settings = options.get("settings", "") - success = False - try: - call_task("setup", options={"settings": settings, "force_exec": True}) - - if not settings: - settings = "REUSE_DB=1 DJANGO_SETTINGS_MODULE=geonode.settings" - - if name and name in ("geonode.tests.csw", "geonode.tests.integration", "geonode.geoserver.tests.integration"): - call_task("sync", options={"settings": settings}) - if local: - call_task("start_geoserver", options={"settings": settings, "force_exec": True}) - call_task("start", options={"settings": settings}) - if integration_server_tests: - call_task("setup_data", options={"settings": settings}) - elif "geonode.geoserver" in INSTALLED_APPS: - if local: - sh("cp geonode/upload/tests/test_settings.py geonode/") - settings = "geonode.test_settings" - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " "makemigrations --noinput") - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " "migrate --noinput") - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " "loaddata sample_admin.json") - sh( - f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " - "loaddata geonode/base/fixtures/default_oauth_apps.json" - ) - sh( - f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py " - "loaddata geonode/base/fixtures/initial_data.json" - ) - call_task("start_geoserver") - bind = options.get("bind", "0.0.0.0:8000") - foreground = "" if options.get("foreground", False) else "&" - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runmessaging {foreground}") - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore manage.py runserver {bind} {foreground}") - sh("sleep 30") - settings = f"REUSE_DB=1 DJANGO_SETTINGS_MODULE={settings}" - else: - call_task("sync", options={"settings": settings}) - - live_server_option = "" - info("Running the tests now...") - sh(f"{settings} {prefix} manage.py test {name} -v 3 {_keepdb} --noinput {live_server_option}") - - except BuildFailure as e: - info(f"Tests failed! {str(e)}") - else: - success = True - finally: - if local: - stop(options) - _reset() - - if not success: - sys.exit(1) - - -@task -@needs( - [ - "start_geoserver", - ] -) -@cmdopts( - [ - ("coverage", "c", "use this flag to generate coverage during test runs"), - ("local=", "l", "Set to True if running tests locally"), - ] -) -def run_tests(options): - """ - Executes the entire test suite. - """ - if options.get("coverage"): - prefix = 'coverage run --branch --source=geonode \ ---omit="*/__init__*,*/test*,*/wsgi*,*/version*,*/migrations*,\ -*/search_indexes*,*/management/*,*/context_processors*,*/upload/*"' - else: - prefix = "python" - local = options.get("local", "false") # travis uses default to false - - if not integration_tests and not integration_csw_tests and not integration_bdd_tests: - call_task("test", options={"prefix": prefix}) - elif integration_tests: - if integration_upload_tests: - call_task( - "test_integration", - options={"prefix": prefix, "name": "geonode.upload.tests.integration", "local": local}, - ) - elif integration_csw_tests: - call_task("test_integration", options={"prefix": prefix, "name": "geonode.tests.csw", "local": local}) - elif integration_bdd_tests: - call_task("test_bdd", options={"local": local}) - elif integration_server_tests: - call_task( - "test_integration", - options={"prefix": prefix, "name": "geonode.geoserver.tests.integration", "local": local}, - ) - else: - call_task( - "test_integration", options={"prefix": prefix, "name": "geonode.tests.integration", "local": local} - ) - sh("flake8 geonode") - - -@task -@needs(["stop"]) -def reset(options): - """ - Reset a development environment (Database, GeoServer & Catalogue) - """ - _reset() - - -def _reset(): - from geonode import settings - - path = os.path.join(settings.PROJECT_ROOT, "development.db") - sh(f"rm -rf {path}") - sh("rm -rf geonode/development.db") - sh("rm -rf geonode/uploaded/*") - _configure_data_dir() - - -@needs(["reset"]) -def reset_hard(options): - """ - Reset a development environment (Database, GeoServer & Catalogue) - """ - sh("git clean -dxf") - - -@task -@cmdopts( - [ - ("type=", "t", 'Import specific data type ("vector", "raster", "time")'), - ("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE"), - ] -) -def setup_data(options): - """ - Import sample data (from gisdata package) into GeoNode - """ - import gisdata - - ctype = options.get("type", None) - - data_dir = gisdata.GOOD_DATA - - if ctype in {"vector", "raster", "time"}: - data_dir = os.path.join(gisdata.GOOD_DATA, ctype) - - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - - from geonode import settings as geonode_settings - - if not os.path.exists(geonode_settings.MEDIA_ROOT): - info("media root not available, creating...") - os.makedirs(geonode_settings.MEDIA_ROOT, exist_ok=True) - - sh(f"{settings} python -W ignore manage.py importlayers -v2 -hh {geonode_settings.SITEURL} {data_dir}") - - -@needs(["package"]) -@cmdopts( - [ - ("key=", "k", "The GPG key to sign the package"), - ("ppa=", "p", "PPA this package should be published to."), - ] -) -def deb(options): - """ - Creates debian packages. - - Example uses: - paver deb - paver deb -k 12345 - paver deb -k 12345 -p geonode/testing - """ - key = options.get("key", None) - ppa = options.get("ppa", None) - - version, simple_version = versions() - - info(f"Creating package for GeoNode version {version}") - - # Get rid of any uncommitted changes to debian/changelog - info("Getting rid of any uncommitted changes in debian/changelog") - sh("git checkout package/debian/changelog") - - # Workaround for git-dch bug - # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 - sh(f"rm -rf {os.path.realpath('package')}/.git") - sh(f"ln -s {os.path.realpath('.git')} {os.path.realpath('package')}") - - with pushd("package"): - # Install requirements - # sh('sudo apt-get -y install debhelper devscripts git-buildpackage') - - # sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s' - # ' --id-length=6 --ignore-branch --release' % (simple_version))) - # In case you publish from Ubuntu Xenial (git-dch is removed from upstream) - # use the following line instead: - # sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s' - # ' --id-length=6 --ignore-branch --release' % (simple_version))) - distribution = "bionic" - # sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s' - # ' --id-length=6 --ignore-branch --release' % (distribution, simple_version))) - - deb_changelog = path("debian") / "changelog" - for idx, line in enumerate(fileinput.input([deb_changelog], inplace=True)): - if idx == 0: - logger.info(f"geonode ({simple_version}) {distribution}; urgency=high", end="") - else: - print(line.replace("urgency=medium", "urgency=high"), end="") - - # Revert workaround for git-dhc bug - sh("rm -rf .git") - - if key is None and ppa is None: - print("A local installable package") - sh("debuild -uc -us -A") - elif key is None and ppa is not None: - print("A sources package, signed by daemon") - sh("debuild -S") - elif key is not None and ppa is None: - print("A signed installable package") - sh(f"debuild -k{key} -A") - elif key is not None and ppa is not None: - print("A signed, source package") - sh(f"debuild -k{key} -S") - - if ppa is not None: - sh(f"dput ppa:{ppa} geonode_{simple_version}_source.changes") - - -@task -def publish(options): - if "GPG_KEY_GEONODE" in os.environ: - key = os.environ["GPG_KEY_GEONODE"] - else: - print("You need to set the GPG_KEY_GEONODE environment variable") - return - - if "PPA_GEONODE" in os.environ: - ppa = os.environ["PPA_GEONODE"] - else: - ppa = None - - call_task( - "deb", - options={ - "key": key, - "ppa": ppa, - # 'ppa': 'geonode/testing', - # 'ppa': 'geonode/unstable', - }, - ) - - version, simple_version = versions() - if ppa: - sh("git add package/debian/changelog") - sh(f'git commit -m "Updated changelog for version {version}"') - sh(f"git tag -f {version}") - sh(f"git push origin {version}") - sh(f"git tag -f debian/{simple_version}") - sh(f"git push origin debian/{simple_version}") - # sh('git push origin master') - sh("python setup.py sdist upload -r pypi") - - -def versions(): - import geonode - from geonode.version import get_git_changeset - - raw_version = geonode.__version__ - version = geonode.get_version() - timestamp = get_git_changeset() - - major, minor, revision, stage, edition = raw_version - - branch = "dev" - - if stage == "final": - stage = "thefinal" - - if stage == "unstable": - tail = f"{branch}{timestamp}" - else: - tail = f"{stage}{edition}" - - simple_version = f"{major}.{minor}.{revision}+{tail}" - return version, simple_version - - -def kill(arg1, arg2): - """Stops a proces that contains arg1 and is filtered by arg2""" - from subprocess import Popen, PIPE - - # Wait until ready - t0 = time.time() - # Wait no more than these many seconds - time_out = 30 - running = True - - while running and time.time() - t0 < time_out: - if os.name == "nt": - p = Popen(f'tasklist | find "{arg1}"', shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) - else: - p = Popen(f"ps aux | grep {arg1}", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - - lines = p.stdout.readlines() - - running = False - for line in lines: - # this kills all java.exe and python including self in windows - if (f"{arg2}" in str(line)) or (os.name == "nt" and f"{arg1}" in str(line)): - running = True - - # Get pid - fields = line.strip().split() - - info(f"Stopping {arg1} (process number {int(fields[1])})") - if os.name == "nt": - kill = f'taskkill /F /PID "{int(fields[1])}"' - else: - kill = f"kill -9 {int(fields[1])} 2> /dev/null" - os.system(kill) - - # Give it a little more time - time.sleep(1) - - if running: - _procs = "\n".join([str(_l).strip() for _l in lines]) - raise Exception(f"Could not stop {arg1}: " f"Running processes are\n{_procs}") - - -def waitfor(url, timeout=300): - started = False - for a in range(timeout): - try: - resp = urlopen(url) - except OSError: - pass - else: - if resp.getcode() == 200: - started = True - break - time.sleep(1) - return started - - -def _copytree(src, dst, symlinks=False, ignore=None): - if not os.path.exists(dst): - os.makedirs(dst, exist_ok=True) - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - try: - shutil.copytree(s, d, symlinks, ignore) - except Exception: - pass - elif os.path.isfile(s): - shutil.copy2(s, d) - - -def justcopy(origin, target): - if os.path.isdir(origin): - shutil.rmtree(target, ignore_errors=True) - _copytree(origin, target) - elif os.path.isfile(origin): - if not os.path.exists(target): - os.makedirs(target, exist_ok=True) - shutil.copy(origin, target) - - -def str2bool(v): - if v and len(v) > 0: - return v.lower() in ("yes", "true", "t", "1") - else: - return False \ No newline at end of file diff --git a/test.sh b/test.sh new file mode 100755 index 00000000000..8a636bebc10 --- /dev/null +++ b/test.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euo pipefail + +grant_superuser() { + local url="${1#*://}" + local user="${url%%:*}" + local host="${url#*@}"; host="${host%%:*}" + local port="${url#*:}"; port="${port#*:}"; port="${port%%/*}" + + echo "Granting SUPERUSER to ${user} (${host}:${port})..." + PGPASSWORD="${POSTGRES_PASSWORD}" psql -h "${host}" -p "${port}" -U postgres \ + -c "ALTER ROLE ${user} SUPERUSER;" +} + +grant_superuser "${DATABASE_URL}" +grant_superuser "${GEODATABASE_URL}" + +echo "PG setup done." +echo "Start loading data." + +GISDATA_DIR=$(python -c "import gisdata; print(gisdata.GOOD_DATA)") + +TYPE="${1:-}" +DATA_DIR="${GISDATA_DIR}" +if [[ "${TYPE}" =~ ^(vector|raster|time)$ ]]; then + DATA_DIR="${GISDATA_DIR}/${TYPE}" +fi + +if [[ ! -d "${MEDIA_ROOT}" ]]; then + echo "media root not available, creating..." + mkdir -p "${MEDIA_ROOT}" +fi + +python -W ignore manage.py importlayers -v2 -hh "${SITEURL}" "${DATA_DIR}" + +echo "Loading data done." +echo "Start test" +coverage run --branch --source=geonode manage.py test -v 3 --keepdb $@ +echo "test completed." \ No newline at end of file diff --git a/tests/pavement.py b/tests/pavement.py deleted file mode 100644 index 21e74565744..00000000000 --- a/tests/pavement.py +++ /dev/null @@ -1,1113 +0,0 @@ -######################################################################### -# -# Copyright (C) 2018 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### - -import fileinput -import glob -import os -import re -import shutil -import subprocess -import signal -import sys -import time -import pytz -import logging -import datetime -from dateutil.parser import parse as parsedate - -from urllib.parse import urlparse -from urllib.request import urlopen, Request - -import zipfile -from tqdm import tqdm -import requests -import math -import psutil - -import yaml -from paver.easy import ( - BuildFailure, - call_task, - cmdopts, - info, - needs, - path, - sh, - task, -) -from setuptools.command import easy_install - -try: - from paver.path import pushd -except ImportError: - from paver.easy import pushd - -from geonode.settings import ( - on_travis, - core_tests, - internal_apps_tests, - integration_tests, - integration_server_tests, - integration_upload_tests, - integration_csw_tests, - integration_bdd_tests, - INSTALLED_APPS, - GEONODE_CORE_APPS, - GEONODE_INTERNAL_APPS, - GEONODE_APPS, - OGC_SERVER, - ASYNC_SIGNALS, - CELERY_BEAT_SCHEDULER, -) - -try: - from geonode.settings import TEST_RUNNER_KEEPDB, TEST_RUNNER_PARALLEL - - _keepdb = "--keepdb" if TEST_RUNNER_KEEPDB else "" - _parallel = f"--parallel={TEST_RUNNER_PARALLEL}" if TEST_RUNNER_PARALLEL else "" -except Exception: - _keepdb = "" - _parallel = "" - -assert sys.version_info >= (2, 6), SystemError("GeoNode Build requires python 2.6 or better") - -dev_config = None -with open("dev_config.yml") as f: - dev_config = yaml.load(f, Loader=yaml.Loader) - - -logger = logging.getLogger(__name__) - - -def grab(src, dest, name): - src, dest, name = map(str, (src, dest, name)) - logger.info(f" src, dest, name --> {src} {dest} {name}") - - if not os.path.exists(dest): - logger.info(f"Downloading {name}") - elif not zipfile.is_zipfile(dest): - logger.info(f"Downloading {name} (corrupt file)") - elif not src.startswith("file://"): - r = requests.head(src) - file_time = datetime.datetime.fromtimestamp(os.path.getmtime(dest)) - url_time = file_time - for _k in ["last-modified", "Date"]: - if _k in r.headers: - url_time = r.headers[_k] - url_date = parsedate(url_time) - utc = pytz.utc - url_date = url_date.replace(tzinfo=utc) - file_time = file_time.replace(tzinfo=utc) - if url_date < file_time: - # Do not download if older than the local one - return - logger.info(f"Downloading updated {name}") - - # Local file does not exist or remote one is newer - if src.startswith("file://"): - src2 = src.replace("file://", "") - if not os.path.exists(src2): - logger.info(f"Source location ({src2}) does not exist") - else: - logger.info(f"Copying local file from {src2}") - shutil.copyfile(src2, dest) - else: - # urlretrieve(str(src), str(dest)) - # Streaming, so we can iterate over the response. - r = requests.get(src, stream=True, timeout=10, verify=False) - # Total size in bytes. - total_size = int(r.headers.get("content-length", 0)) - logger.info(f"Requesting {src}") - block_size = 1024 - wrote = 0 - with open("output.bin", "wb") as f: - for data in tqdm( - r.iter_content(block_size), total=math.ceil(total_size // block_size), unit="KB", unit_scale=False - ): - wrote += len(data) - f.write(data) - logger.info(f" total_size [{total_size}] / wrote [{wrote}] ") - if total_size != 0 and wrote != total_size: - logger.error( - f"ERROR, something went wrong. Data could not be written. Expected to write {wrote} but wrote {total_size} instead" - ) - else: - shutil.move("output.bin", dest) - try: - # Cleaning up - os.remove("output.bin") - except OSError: - pass - - -@task -@cmdopts( - [ - ("geoserver=", "g", "The location of the geoserver build (.war file)."), - ("jetty=", "j", "The location of the Jetty Runner (.jar file)."), - ("force_exec=", "", "Force GeoServer Setup."), - ] -) -def setup_geoserver(options): - """Prepare a testing instance of GeoServer.""" - # only start if using Geoserver backend - if "geonode.geoserver" not in INSTALLED_APPS: - return - if on_travis and not options.get("force_exec", False): - """Will make use of the docker container for the Integration Tests""" - return - else: - download_dir = path("downloaded") - if not download_dir.exists(): - download_dir.makedirs() - geoserver_dir = path("geoserver") - geoserver_bin = download_dir / os.path.basename(urlparse(dev_config["GEOSERVER_URL"]).path) - jetty_runner = download_dir / os.path.basename(urlparse(dev_config["JETTY_RUNNER_URL"]).path) - geoserver_data = download_dir / os.path.basename(urlparse(dev_config["DATA_DIR_URL"]).path) - grab(options.get("geoserver", dev_config["GEOSERVER_URL"]), geoserver_bin, "geoserver binary") - grab(options.get("jetty", dev_config["JETTY_RUNNER_URL"]), jetty_runner, "jetty runner") - grab(options.get("geoserver data", dev_config["DATA_DIR_URL"]), geoserver_data, "geoserver data-dir") - - if not geoserver_dir.exists(): - geoserver_dir.makedirs() - - webapp_dir = geoserver_dir / "geoserver" - if not webapp_dir: - webapp_dir.makedirs() - - logger.info("extracting geoserver") - z = zipfile.ZipFile(geoserver_bin, "r") - z.extractall(webapp_dir) - - logger.info("extracting geoserver data dir") - z = zipfile.ZipFile(geoserver_data, "r") - z.extractall(geoserver_dir) - - _configure_data_dir() - - -def _configure_data_dir(): - try: - config = path("geoserver/data/global.xml") - with open(config) as f: - xml = f.read() - m = re.search("proxyBaseUrl>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8080/geoserver{xml[m.end(1):]}" - with open(config, "w") as f: - f.write(xml) - except Exception as e: - print(e) - - try: - config = path("geoserver/data/security/filter/geonode-oauth2/config.xml") - with open(config) as f: - xml = f.read() - m = re.search("accessTokenUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/o/token/{xml[m.end(1):]}" - m = re.search("userAuthorizationUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/o/authorize/{xml[m.end(1):]}" - m = re.search("redirectUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8080/geoserver/index.html{xml[m.end(1):]}" - m = re.search("checkTokenEndpointUrl>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/api/o/v4/tokeninfo/{xml[m.end(1):]}" - m = re.search("logoutUri>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000/account/logout/{xml[m.end(1):]}" - with open(config, "w") as f: - f.write(xml) - except Exception as e: - print(e) - - try: - config = path("geoserver/data/security/role/geonode REST role service/config.xml") - with open(config) as f: - xml = f.read() - m = re.search("baseUrl>([^<]+)", xml) - xml = f"{xml[:m.start(1)]}http://localhost:8000{xml[m.end(1):]}" - with open(config, "w") as f: - f.write(xml) - except Exception as e: - print(e) - - -@task -def static(options): - with pushd("geonode/static"): - sh("grunt production") - - -@task -@needs( - [ - "setup_geoserver", - ] -) -def setup(options): - """Get dependencies and prepare a GeoNode development environment.""" - info( - "GeoNode development environment successfully set up." - "If you have not set up an administrative account," - ' please do so now. Use "paver start" to start up the server.' - ) - - -def grab_winfiles(url, dest, packagename): - # Add headers - headers = {"User-Agent": "Mozilla 5.10"} - request = Request(url, None, headers) - response = urlopen(request) - with open(dest, "wb") as writefile: - writefile.write(response.read()) - - -@task -def win_install_deps(options): - """ - Install all Windows Binary automatically - This can be removed as wheels become available for these packages - """ - download_dir = path("downloaded").abspath() - if not download_dir.exists(): - download_dir.makedirs() - win_packages = { - # required by transifex-client - "Py2exe": dev_config["WINDOWS"]["py2exe"], - # the wheel 1.9.4 installs but pycsw wants 1.9.3, which fails to compile - # when pycsw bumps their pyproj to 1.9.4 this can be removed. - "PyProj": dev_config["WINDOWS"]["pyproj"], - "lXML": dev_config["WINDOWS"]["lxml"], - } - failed = False - for package, url in win_packages.items(): - tempfile = download_dir / os.path.basename(url) - logger.info(f"Installing file ... {tempfile}") - grab_winfiles(url, tempfile, package) - try: - easy_install.main([tempfile]) - except Exception as e: - failed = True - logger.error("install failed with error: ", e) - os.remove(tempfile) - if failed and sys.maxsize > 2**32: - logger.error("64bit architecture is not currently supported") - logger.error("try finding the 64 binaries for py2exe, and pyproj") - elif failed: - logger.error("install failed for py2exe, and/or pyproj") - else: - print("Windows dependencies now complete. Run pip install -e geonode --use-mirrors") - - -@task -@cmdopts([("version=", "v", "Legacy GeoNode version of the existing database.")]) -def upgradedb(options): - """ - Add 'fake' data migrations for existing tables from legacy GeoNode versions - """ - version = options.get("version") - if version in {"1.1", "1.2"}: - sh("python -W ignore ../manage.py migrate maps 0001 --fake") - sh("python -W ignore ../manage.py migrate avatar 0001 --fake") - elif version is None: - print("Please specify your GeoNode version") - else: - print(f"Upgrades from version {version} are not yet supported.") - - -@task -@cmdopts([("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE")]) -def sync(options): - """ - Run the migrate and migrate management commands to create and migrate a DB - """ - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - - sh(f"{settings} python -W ignore ../manage.py makemigrations --noinput") - sh(f"{settings} python -W ignore ../manage.py migrate --noinput") - sh(f"{settings} python -W ignore ../manage.py loaddata sample_admin.json") - sh(f"{settings} python -W ignore ../manage.py loaddata geonode/base/fixtures/default_oauth_apps.json") - sh(f"{settings} python -W ignore ../manage.py loaddata geonode/base/fixtures/initial_data.json") - sh(f"{settings} python -W ignore ../manage.py set_all_datasets_alternate") - sh(f"{settings} python -W ignore ../manage.py collectstatic --noinput") - - -@task -def package(options): - """ - Creates a tarball to use for building the system elsewhere - """ - import tarfile - import geonode - - version = geonode.get_version() - # Use GeoNode's version for the package name. - pkgname = f"GeoNode-{version}-all" - - # Create the output directory. - out_pkg = path(pkgname) - out_pkg_tar = path(f"{pkgname}.tar.gz") - - # Create a distribution in zip format for the geonode python package. - dist_dir = path("dist") - dist_dir.rmtree() - sh("python setup.py sdist --formats=zip") - - with pushd("package"): - # Delete old tar files in that directory - for f in glob.glob("GeoNode*.tar.gz"): - old_package = path(f) - if old_package != out_pkg_tar: - old_package.remove() - - if out_pkg_tar.exists(): - info(f"There is already a package for version {version}") - return - - # Clean anything that is in the oupout package tree. - out_pkg.rmtree() - out_pkg.makedirs() - - support_folder = path("support") - install_file = path("install.sh") - - # And copy the default files from the package folder. - justcopy(support_folder, out_pkg / "support") - justcopy(install_file, out_pkg) - - geonode_dist = path("..") / "dist" / f"GeoNode-{version}.zip" - justcopy(geonode_dist, out_pkg) - - # Create a tar file with all files in the output package folder. - tar = tarfile.open(out_pkg_tar, "w:gz") - for file in out_pkg.walkfiles(): - tar.add(file) - - # Add the README with the license and important links to documentation. - tar.add("README", arcname=f"{out_pkg}/README.rst") - tar.close() - - # Remove all the files in the temporary output package directory. - out_pkg.rmtree() - - # Report the info about the new package. - info(f"{out_pkg_tar.abspath()} created") - - -@task -@needs(["start_geoserver", "start_django"]) -@cmdopts( - [ - ("bind=", "b", "Bind server to provided IP address and port number."), - ("java_path=", "j", "Full path to java install for Windows"), - ("foreground", "f", "Do not run in background but in foreground"), - ("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE"), - ], - share_with=["start_django", "start_geoserver"], -) -def start(options): - """ - Start GeoNode (Django, GeoServer & Client) - """ - info("GeoNode is now available.") - - -@task -def stop_django(options): - """ - Stop the GeoNode Django application - """ - if ASYNC_SIGNALS: - kill("python", "celery") - kill("celery", "worker") - kill("python", "runserver") - kill("python", "runmessaging") - - -@task -@cmdopts([("force_exec=", "", "Force GeoServer Stop.")]) -def stop_geoserver(options): - """ - Stop GeoServer - """ - # we use docker-compose for integration tests - if on_travis and not options.get("force_exec", False): - return - - # only start if using Geoserver backend - if "geonode.geoserver" not in INSTALLED_APPS: - return - kill("java", "geoserver") - - # Kill process. - try: - # proc = subprocess.Popen("ps -ef | grep -i -e '[j]ava\|geoserver' | - # awk '{print $2}'", - proc = subprocess.Popen( - "ps -ef | grep -i -e 'geoserver' | awk '{print $2}'", shell=True, stdout=subprocess.PIPE - ) - for pid in map(int, proc.stdout): - info(f"Stopping geoserver (process number {pid})") - os.kill(pid, signal.SIGKILL) - - # Check if the process that we killed is alive. - killed, alive = psutil.wait_procs([psutil.Process(pid=pid)], timeout=30) - for p in alive: - p.kill() - except Exception as e: - info(e) - - -@task -@needs( - [ - "stop_geoserver", - ] -) -def stop(options): - """ - Stop GeoNode - """ - # windows needs to stop the geoserver first b/c we can't tell which python - # is running, so we kill everything - info("Stopping GeoNode ...") - stop_django(options) - - -@task -@cmdopts([("bind=", "b", "Bind server to provided IP address and port number.")]) -def start_django(options): - """ - Start the GeoNode Django application - """ - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - bind = options.get("bind", "0.0.0.0:8000") - port = bind.split(":")[1] - foreground = "" if options.get("foreground", False) else "&" - sh(f"{settings} python -W ignore ../manage.py runserver {bind} {foreground}") - - if ASYNC_SIGNALS: - sh( - f"{settings} celery -A geonode.celery_app:app worker --autoscale=20,10 --without-gossip --without-mingle -Ofair -B -E \ - --statedb=/tmp/worker.state --scheduler={CELERY_BEAT_SCHEDULER} --loglevel=DEBUG \ - --concurrency=10 --max-tasks-per-child=10 -n worker1@%h -f celery.log {foreground}" - ) - sh(f"{settings} python -W ignore ../manage.py runmessaging {foreground}") - - # wait for Django to start - started = waitfor(f"http://localhost:{port}") - if not started: - info("Django never started properly or timed out.") - sys.exit(1) - - -@task -def start_messaging(options): - """ - Start the GeoNode messaging server - """ - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - foreground = "" if options.get("foreground", False) else "&" - sh(f"{settings} python -W ignore ../manage.py runmessaging {foreground}") - - -@task -@cmdopts([("java_path=", "j", "Full path to java install for Windows"), ("force_exec=", "", "Force GeoServer Start.")]) -def start_geoserver(options): - """ - Start GeoServer with GeoNode extensions - """ - # we use docker-compose for integration tests - if on_travis and not options.get("force_exec", False): - return - - # only start if using Geoserver backend - if "geonode.geoserver" not in INSTALLED_APPS: - return - - GEOSERVER_BASE_URL = OGC_SERVER["default"]["LOCATION"] - url = GEOSERVER_BASE_URL - - if urlparse(GEOSERVER_BASE_URL).hostname != "localhost": - logger.warning("Warning: OGC_SERVER['default']['LOCATION'] hostname is not equal to 'localhost'") - - if not GEOSERVER_BASE_URL.endswith("/"): - logger.error("Error: OGC_SERVER['default']['LOCATION'] does not end with a '/'") - sys.exit(1) - - download_dir = path("downloaded").abspath() - jetty_runner = download_dir / os.path.basename(dev_config["JETTY_RUNNER_URL"]) - data_dir = path("geoserver/data").abspath() - geofence_dir = path("geoserver/data/geofence").abspath() - web_app = path("geoserver/geoserver").abspath() - log_file = path("geoserver/jetty.log").abspath() - config = path("scripts/misc/jetty-runner.xml").abspath() - jetty_port = urlparse(GEOSERVER_BASE_URL).port - - import socket - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - socket_free = True - try: - s.bind(("127.0.0.1", jetty_port)) - except OSError as e: - socket_free = False - if e.errno == 98: - info(f"Port {jetty_port} is already in use") - else: - info(f"Something else raised the socket.error exception while checking port {jetty_port}") - print(e) - finally: - s.close() - - if socket_free: - # @todo - we should not have set workdir to the datadir but a bug in geoserver - # prevents geonode security from initializing correctly otherwise - with pushd(data_dir): - javapath = "java" - if on_travis: - sh( - "sudo apt install -y openjdk-8-jre openjdk-8-jdk;" - " sudo update-java-alternatives --set java-1.8.0-openjdk-amd64;" - ' export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::");' - " export PATH=$JAVA_HOME'bin/java':$PATH;" - ) - # import subprocess - # result = subprocess.run(['update-alternatives', '--list', 'java'], stdout=subprocess.PIPE) - # javapath = result.stdout - javapath = "/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java" - loggernullpath = os.devnull - - # checking if our loggernullpath exists and if not, reset it to - # something manageable - if loggernullpath == "nul": - try: - open("../../downloaded/null.txt", "w+").close() - except OSError: - print( - "Chances are that you have Geoserver currently running. You " - "can either stop all servers with paver stop or start only " - "the django application with paver start_django." - ) - sys.exit(1) - loggernullpath = "../../downloaded/null.txt" - - try: - sh(("%(javapath)s -version") % locals()) - except Exception: - logger.warning("Java was not found in your path. Trying some other options: ") - javapath_opt = None - if os.environ.get("JAVA_HOME", None): - logger.info("Using the JAVA_HOME environment variable") - javapath_opt = os.path.join(os.path.abspath(os.environ["JAVA_HOME"]), "bin", "java.exe") - elif options.get("java_path"): - javapath_opt = options.get("java_path") - else: - logger.critical( - "Paver cannot find java in the Windows Environment. " - "Please provide the --java_path flag with your full path to " - "java.exe e.g. --java_path=C:/path/to/java/bin/java.exe" - ) - sys.exit(1) - # if there are spaces - javapath = f'START /B "" "{javapath_opt}"' - - sh( - "%(javapath)s -Xms512m -Xmx2048m -server -Dgwc.context.suffix=gwc -XX:+UseConcMarkSweepGC -XX:MaxPermSize=512m" - " -DGEOSERVER_DATA_DIR=%(data_dir)s" - " -DGEOSERVER_CSRF_DISABLED=true" - " -Dgeofence.dir=%(geofence_dir)s" - " -Djava.awt.headless=true" - # ' -Dgeofence-ovr=geofence-datasource-ovr.properties' - # workaround for JAI sealed jar issue and jetty classloader - # ' -Dorg.eclipse.jetty.server.webapp.parentLoaderPriority=true' - " -jar %(jetty_runner)s" - " --port %(jetty_port)i" - " --log %(log_file)s" - " %(config)s" - " > %(loggernullpath)s &" % locals() - ) - - info(f"Starting GeoServer on {url}") - - # wait for GeoServer to start - started = waitfor(url) - info(f"The logs are available at {log_file}") - - if not started: - # If applications did not start in time we will give the user a chance - # to inspect them and stop them manually. - info("GeoServer never started properly or timed out." "It may still be running in the background.") - sys.exit(1) - - -@task -def test(options): - """ - Run GeoNode's Unit Test Suite - """ - if on_travis: - if core_tests: - _apps = GEONODE_CORE_APPS - if internal_apps_tests: - _apps = GEONODE_INTERNAL_APPS - else: - _apps = GEONODE_APPS - - _apps_to_test = [] - for _app in _apps: - if _app and len(_app) > 0 and "geonode" in _app: - _apps_to_test.append(_app) - sh( - f"{options.get('prefix')} ../manage.py test geonode.tests.smoke \ -{('.tests '.join(_apps_to_test))}.tests --noinput {_keepdb} {_parallel}" - ) - - -@task -@cmdopts([("local=", "l", "Set to True if running bdd tests locally")]) -def test_bdd(options): - """ - Run GeoNode's BDD Test Suite - """ - local = str2bool(options.get("local", "false")) - if local: - call_task("reset_hard") - - call_task("setup") - call_task("sync") - if local: - sh("sleep 30") - - info("GeoNode is now available, running the bdd tests now.") - sh("py.test") - - -@task -def test_javascript(options): - with pushd("geonode/static/geonode"): - sh("./run-tests.sh") - - -@task -@cmdopts( - [ - ("name=", "n", "Run specific tests."), - ("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE"), - ("local=", "l", "Set to True if running bdd tests locally"), - ] -) -def test_integration(options): - """ - Run GeoNode's Integration test suite against the external apps - """ - prefix = options.get("prefix") - local = str2bool(options.get("local", "false")) - if local: - call_task("stop_geoserver") - _reset() - - name = options.get("name", None) - settings = options.get("settings", "") - success = False - try: - call_task("setup", options={"settings": settings, "force_exec": True}) - - if not settings: - settings = "REUSE_DB=1 DJANGO_SETTINGS_MODULE=geonode.settings" - - if name and name in ("geonode.tests.csw", "geonode.tests.integration", "geonode.geoserver.tests.integration"): - call_task("sync", options={"settings": settings}) - if local: - call_task("start_geoserver", options={"settings": settings, "force_exec": True}) - call_task("start", options={"settings": settings}) - if integration_server_tests: - call_task("setup_data", options={"settings": settings}) - elif "geonode.geoserver" in INSTALLED_APPS: - if local: - sh("cp geonode/upload/tests/test_settings.py geonode/") - settings = "geonode.test_settings" - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py " "makemigrations --noinput") - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py " "migrate --noinput") - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py " "loaddata sample_admin.json") - sh( - f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py " - "loaddata geonode/base/fixtures/default_oauth_apps.json" - ) - sh( - f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py " - "loaddata geonode/base/fixtures/initial_data.json" - ) - call_task("start_geoserver") - bind = options.get("bind", "0.0.0.0:8000") - foreground = "" if options.get("foreground", False) else "&" - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py runmessaging {foreground}") - sh(f"DJANGO_SETTINGS_MODULE={settings} python -W ignore ../manage.py runserver {bind} {foreground}") - sh("sleep 30") - settings = f"REUSE_DB=1 DJANGO_SETTINGS_MODULE={settings}" - else: - call_task("sync", options={"settings": settings}) - - live_server_option = "" - info("Running the tests now...") - sh(f"{settings} {prefix} ../manage.py test {name} -v 3 {_keepdb} --noinput {live_server_option}") - - except BuildFailure as e: - info(f"Tests failed! {str(e)}") - else: - success = True - finally: - if local: - stop(options) - _reset() - - if not success: - sys.exit(1) - - -@task -@needs( - [ - "start_geoserver", - ] -) -@cmdopts( - [ - ("coverage", "c", "use this flag to generate coverage during test runs"), - ("local=", "l", "Set to True if running tests locally"), - ] -) -def run_tests(options): - """ - Executes the entire test suite. - """ - if options.get("coverage"): - prefix = 'coverage run --branch --source=geonode \ ---omit="*/__init__*,*/test*,*/wsgi*,*/version*,*/migrations*,\ -*/search_indexes*,*/management/*,*/context_processors*,*/upload/*"' - else: - prefix = "python" - local = options.get("local", "false") # travis uses default to false - - if not integration_tests and not integration_csw_tests and not integration_bdd_tests: - call_task("test", options={"prefix": prefix}) - elif integration_tests: - if integration_upload_tests: - call_task( - "test_integration", - options={"prefix": prefix, "name": "geonode.upload.tests.integration", "local": local}, - ) - elif integration_csw_tests: - call_task("test_integration", options={"prefix": prefix, "name": "geonode.tests.csw", "local": local}) - elif integration_bdd_tests: - call_task("test_bdd", options={"local": local}) - elif integration_server_tests: - call_task( - "test_integration", - options={"prefix": prefix, "name": "geonode.geoserver.tests.integration", "local": local}, - ) - else: - call_task( - "test_integration", options={"prefix": prefix, "name": "geonode.tests.integration", "local": local} - ) - sh("flake8 geonode") - - -@task -@needs(["stop"]) -def reset(options): - """ - Reset a development environment (Database, GeoServer & Catalogue) - """ - _reset() - - -def _reset(): - from geonode import settings - - path = os.path.join(settings.PROJECT_ROOT, "development.db") - sh(f"rm -rf {path}") - sh("rm -rf geonode/development.db") - sh("rm -rf geonode/uploaded/*") - _configure_data_dir() - - -@needs(["reset"]) -def reset_hard(options): - """ - Reset a development environment (Database, GeoServer & Catalogue) - """ - sh("git clean -dxf") - - -@task -@cmdopts( - [ - ("type=", "t", 'Import specific data type ("vector", "raster", "time")'), - ("settings=", "s", "Specify custom DJANGO_SETTINGS_MODULE"), - ] -) -def setup_data(options): - """ - Import sample data (from gisdata package) into GeoNode - """ - import gisdata - - ctype = options.get("type", None) - - data_dir = gisdata.GOOD_DATA - - if ctype in {"vector", "raster", "time"}: - data_dir = os.path.join(gisdata.GOOD_DATA, ctype) - - settings = options.get("settings", "") - if settings and "DJANGO_SETTINGS_MODULE" not in settings: - settings = f"DJANGO_SETTINGS_MODULE={settings}" - - from geonode import settings as geonode_settings - - if not os.path.exists(geonode_settings.MEDIA_ROOT): - info("media root not available, creating...") - os.makedirs(geonode_settings.MEDIA_ROOT, exist_ok=True) - - sh(f"{settings} python -W ignore ../manage.py importlayers -v2 -hh {geonode_settings.SITEURL} {data_dir}") - - -@needs(["package"]) -@cmdopts( - [ - ("key=", "k", "The GPG key to sign the package"), - ("ppa=", "p", "PPA this package should be published to."), - ] -) -def deb(options): - """ - Creates debian packages. - - Example uses: - paver deb - paver deb -k 12345 - paver deb -k 12345 -p geonode/testing - """ - key = options.get("key", None) - ppa = options.get("ppa", None) - - version, simple_version = versions() - - info(f"Creating package for GeoNode version {version}") - - # Get rid of any uncommitted changes to debian/changelog - info("Getting rid of any uncommitted changes in debian/changelog") - sh("git checkout package/debian/changelog") - - # Workaround for git-dch bug - # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=594580 - sh(f"rm -rf {os.path.realpath('package')}/.git") - sh(f"ln -s {os.path.realpath('.git')} {os.path.realpath('package')}") - - with pushd("package"): - # Install requirements - # sh('sudo apt-get -y install debhelper devscripts git-buildpackage') - - # sh(('git-dch --spawn-editor=snapshot --git-author --new-version=%s' - # ' --id-length=6 --ignore-branch --release' % (simple_version))) - # In case you publish from Ubuntu Xenial (git-dch is removed from upstream) - # use the following line instead: - # sh(('gbp dch --spawn-editor=snapshot --git-author --new-version=%s' - # ' --id-length=6 --ignore-branch --release' % (simple_version))) - distribution = "bionic" - # sh(('gbp dch --distribution=%s --force-distribution --spawn-editor=snapshot --git-author --new-version=%s' - # ' --id-length=6 --ignore-branch --release' % (distribution, simple_version))) - - deb_changelog = path("debian") / "changelog" - for idx, line in enumerate(fileinput.input([deb_changelog], inplace=True)): - if idx == 0: - logger.info(f"geonode ({simple_version}) {distribution}; urgency=high", end="") - else: - print(line.replace("urgency=medium", "urgency=high"), end="") - - # Revert workaround for git-dhc bug - sh("rm -rf .git") - - if key is None and ppa is None: - print("A local installable package") - sh("debuild -uc -us -A") - elif key is None and ppa is not None: - print("A sources package, signed by daemon") - sh("debuild -S") - elif key is not None and ppa is None: - print("A signed installable package") - sh(f"debuild -k{key} -A") - elif key is not None and ppa is not None: - print("A signed, source package") - sh(f"debuild -k{key} -S") - - if ppa is not None: - sh(f"dput ppa:{ppa} geonode_{simple_version}_source.changes") - - -@task -def publish(options): - if "GPG_KEY_GEONODE" in os.environ: - key = os.environ["GPG_KEY_GEONODE"] - else: - print("You need to set the GPG_KEY_GEONODE environment variable") - return - - if "PPA_GEONODE" in os.environ: - ppa = os.environ["PPA_GEONODE"] - else: - ppa = None - - call_task( - "deb", - options={ - "key": key, - "ppa": ppa, - # 'ppa': 'geonode/testing', - # 'ppa': 'geonode/unstable', - }, - ) - - version, simple_version = versions() - if ppa: - sh("git add package/debian/changelog") - sh(f'git commit -m "Updated changelog for version {version}"') - sh(f"git tag -f {version}") - sh(f"git push origin {version}") - sh(f"git tag -f debian/{simple_version}") - sh(f"git push origin debian/{simple_version}") - # sh('git push origin master') - sh("python setup.py sdist upload -r pypi") - - -def versions(): - import geonode - from geonode.version import get_git_changeset - - raw_version = geonode.__version__ - version = geonode.get_version() - timestamp = get_git_changeset() - - major, minor, revision, stage, edition = raw_version - - branch = "dev" - - if stage == "final": - stage = "thefinal" - - if stage == "unstable": - tail = f"{branch}{timestamp}" - else: - tail = f"{stage}{edition}" - - simple_version = f"{major}.{minor}.{revision}+{tail}" - return version, simple_version - - -def kill(arg1, arg2): - """Stops a proces that contains arg1 and is filtered by arg2""" - from subprocess import Popen, PIPE - - # Wait until ready - t0 = time.time() - # Wait no more than these many seconds - time_out = 30 - running = True - - while running and time.time() - t0 < time_out: - if os.name == "nt": - p = Popen(f'tasklist | find "{arg1}"', shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=False) - else: - p = Popen(f"ps aux | grep {arg1}", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - - lines = p.stdout.readlines() - - running = False - for line in lines: - # this kills all java.exe and python including self in windows - if (f"{arg2}" in str(line)) or (os.name == "nt" and f"{arg1}" in str(line)): - running = True - - # Get pid - fields = line.strip().split() - - info(f"Stopping {arg1} (process number {int(fields[1])})") - if os.name == "nt": - kill = f'taskkill /F /PID "{int(fields[1])}"' - else: - kill = f"kill -9 {int(fields[1])} 2> /dev/null" - os.system(kill) - - # Give it a little more time - time.sleep(1) - - if running: - _procs = "\n".join([str(_l).strip() for _l in lines]) - raise Exception(f"Could not stop {arg1}: " f"Running processes are\n{_procs}") - - -def waitfor(url, timeout=300): - started = False - for a in range(timeout): - try: - resp = urlopen(url) - except OSError: - pass - else: - if resp.getcode() == 200: - started = True - break - time.sleep(1) - return started - - -def _copytree(src, dst, symlinks=False, ignore=None): - if not os.path.exists(dst): - os.makedirs(dst, exist_ok=True) - for item in os.listdir(src): - s = os.path.join(src, item) - d = os.path.join(dst, item) - if os.path.isdir(s): - try: - shutil.copytree(s, d, symlinks, ignore) - except Exception: - pass - elif os.path.isfile(s): - shutil.copy2(s, d) - - -def justcopy(origin, target): - if os.path.isdir(origin): - shutil.rmtree(target, ignore_errors=True) - _copytree(origin, target) - elif os.path.isfile(origin): - if not os.path.exists(target): - os.makedirs(target, exist_ok=True) - shutil.copy(origin, target) - - -def str2bool(v): - if v and len(v) > 0: - return v.lower() in ("yes", "true", "t", "1") - else: - return False \ No newline at end of file diff --git a/tests/test.sh b/tests/test.sh deleted file mode 100755 index d9987fd1849..00000000000 --- a/tests/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -e -set -a -. ./.env_test -set +a - -paver setup_data -coverage run --branch --source=geonode manage.py test -v 3 --keepdb $@ \ No newline at end of file diff --git a/tests/test_api_v2.sh b/tests/test_api_v2.sh deleted file mode 100755 index 213e1def849..00000000000 --- a/tests/test_api_v2.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -e - -./test.sh geonode.base.api.tests geonode.layers.api.tests geonode.maps.api.tests geonode.documents.api.tests geonode.geoapps.api.tests diff --git a/tests/test_csw.sh b/tests/test_csw.sh deleted file mode 100755 index 9d2a7a12878..00000000000 --- a/tests/test_csw.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -export SITEURL=http://localhost:8001/ -export BACKEND=geonode.geoserver -export DOCKER_COMPOSE_VERSION=1.19.0 -export GEOSERVER_SERVER_URL=http://geoserver:8080/geoserver/ -export GEOSERVER_SERVER_PORT=8080 -export ON_TRAVIS=True -export TEST_RUNNER_KEEPDB=True -export TEST_RUN_INTEGRATION=True -export TEST_RUN_INTEGRATION_SERVER=True -export TEST_RUN_INTEGRATION_UPLOAD=False -export TEST_RUN_INTEGRATION_MONITORING=False -export TEST_RUN_INTEGRATION_CSW=True -export TEST_RUN_INTEGRATION_BDD=False -export SESSION_EXPIRED_CONTROL_ENABLED=True -export ASYNC_SIGNALS=False -export DATABASE_URL=postgis://geonode:geonode@db:5432/geonode -export GEODATABASE_URL=postgis://geonode:geonode@db:5432/geonode_data -export DEFAULT_BACKEND_DATASTORE=datastore -export DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=100 - -# echo "Initialize DB"; -# chmod +x scripts/misc/create_dbs_travis.sh; -# scripts/misc/create_dbs_travis.sh before_script; - -paver run_tests --coverage --local false \ No newline at end of file diff --git a/tests/test_dev.sh b/tests/test_dev.sh deleted file mode 100755 index 88e31a63fd5..00000000000 --- a/tests/test_dev.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -a -. ./.env_dev -set +a - -# paver setup_data -# dropdb -U postgres test_geonode -coverage run --branch --source=geonode manage.py test -v 3 --keepdb $@ diff --git a/tests/test_integration.sh b/tests/test_integration.sh deleted file mode 100755 index 3abab6e9a84..00000000000 --- a/tests/test_integration.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -export BACKEND=geonode.geoserver -export DOCKER_COMPOSE_VERSION=1.19.0 -export GEOSERVER_SERVER_URL=http://localhost:8080/geoserver/ -export GEOSERVER_SERVER_PORT=8080 -export ON_TRAVIS=True -export TEST_RUNNER_KEEPDB=True -export TEST_RUN_INTEGRATION=True -export TEST_RUN_INTEGRATION_SERVER=False -export TEST_RUN_INTEGRATION_UPLOAD=False -export TEST_RUN_INTEGRATION_MONITORING=False -export TEST_RUN_INTEGRATION_CSW=False -export TEST_RUN_INTEGRATION_BDD=False -export USER_ANALYTICS_ENABLED=False -export SESSION_EXPIRED_CONTROL_ENABLED=True -export CELERY_ALWAYS_EAGER=True - -# coverage run --branch --source=geonode manage.py test --noinput --parallel=1 $@ -echo "Initialize DB"; -chmod +x scripts/misc/create_dbs_travis.sh; -scripts/misc/create_dbs_travis.sh before_script; - -paver run_tests --coverage --local false \ No newline at end of file diff --git a/tests/test_oauth2.sh b/tests/test_oauth2.sh deleted file mode 100755 index 79056e01dcd..00000000000 --- a/tests/test_oauth2.sh +++ /dev/null @@ -1,9 +0,0 @@ -# To run this test you need first to crate a "Resource owner password-based" OAUTH2 App -BASE_URL=http://localhost:8000 -USERNAME=admin -PASSWORD=***** -CLIENT_ID=************* -CLIENT_SECRET=****************** - -TOKEN=$(http --form POST ${BASE_URL}/o/token/ grant_type=password username=${USERNAME} password=${PASSWORD} -a ${CLIENT_ID}:${CLIENT_SECRET} | jq -r '.access_token') -http ${BASE_URL}/o/userinfo/ scopes=openid,write claims=username "Authorization:Bearer ${TOKEN}" --all -v diff --git a/tests/test_upload.sh b/tests/test_upload.sh deleted file mode 100755 index 7abc2168709..00000000000 --- a/tests/test_upload.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -export SITEURL=http://localhost:8001/ -export BACKEND=geonode.geoserver -export DOCKER_COMPOSE_VERSION=1.19.0 -export GEOSERVER_SERVER_URL=http://geoserver:8080/geoserver/ -export GEOSERVER_SERVER_PORT=8080 -export ON_TRAVIS=True -export TEST_RUNNER_KEEPDB=True -export TEST_RUN_INTEGRATION=True -export TEST_RUN_INTEGRATION_SERVER=False -export TEST_RUN_INTEGRATION_UPLOAD=True -export TEST_RUN_INTEGRATION_MONITORING=False -export TEST_RUN_INTEGRATION_CSW=False -export TEST_RUN_INTEGRATION_BDD=False -export SESSION_EXPIRED_CONTROL_ENABLED=True -export ASYNC_SIGNALS=False -export DATABASE_URL=postgis://geonode:geonode@db:5432/geonode -export GEODATABASE_URL=postgis://geonode:geonode@db:5432/geonode_data -export DEFAULT_BACKEND_DATASTORE=datastore - -# echo "Initialize DB"; -# chmod +x scripts/misc/create_dbs_travis.sh; -# scripts/misc/create_dbs_travis.sh before_script; - -paver run_tests --coverage --local false \ No newline at end of file From ed061ca83b3f0a3b84dd32861f2076741e31b238 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Tue, 9 Jun 2026 18:18:33 +0200 Subject: [PATCH 11/16] refactor test execution --- geonode/tests/README | 8 -------- geonode/upload/tests/README.rst | 32 -------------------------------- pyproject.toml | 1 - 3 files changed, 41 deletions(-) delete mode 100644 geonode/tests/README delete mode 100644 geonode/upload/tests/README.rst diff --git a/geonode/tests/README b/geonode/tests/README deleted file mode 100644 index bff04c7b705..00000000000 --- a/geonode/tests/README +++ /dev/null @@ -1,8 +0,0 @@ -# To run the integration tests, do: -paver test_integration - -# To run the csw tests, do: -paver test_integration -n geonode.tests.csw - -# To run the bdd tests, do: -paver test_bdd --local true diff --git a/geonode/upload/tests/README.rst b/geonode/upload/tests/README.rst deleted file mode 100644 index 265018721ca..00000000000 --- a/geonode/upload/tests/README.rst +++ /dev/null @@ -1,32 +0,0 @@ -To run the integration tests, make sure a the sqlite db is setup:: - - python manage.py migrate - -Run geoserver but ensure that the django server is _not_ running:: - - paver start_geoserver - -While geoserver is running, run tests:: - - python manage.py test geonode.upload.tests.integration - -Or, run a specific test class or single test (leave out the dot if no test):: - - python manage.py test geonode.upload.tests.integration:. - -These tests will internally run a django server and modify the settings as -needed to adjust differences in configuration. They will also create a user -named `test_uploader` and delete any layers this user owns prior to running. - -The upload tests will load a settings module to allow specification of a postgres -database other than what you might use for other local purposes. This module is:: - - geonode.upload.tests.test_settings - -If the settings do not set the name of the OGC_SERVER DATASTORE option, -the importer tests that import into the database will not run. - -The `test_settings` module must also be supplied when launching the tests to run -the full suite including the DATASTORE tests:: - - DJANGO_SETTINGS_MODULE=geonode.upload.tests.test_settings python manage.py test geonode.upload.tests.integration diff --git a/pyproject.toml b/pyproject.toml index 1339948e899..46c7970e1b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ dependencies = [ "beautifulsoup4==4.12.3", "hyperlink==21.0.0", "urllib3==2.7.0", - "Paver==1.3.4", "python-slugify==8.0.4", "decorator==5.1.1", "celery==5.6.3", From de1def3a66a6f657db05ac86276ee3c78dc4a5df Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 10 Jun 2026 14:34:31 +0200 Subject: [PATCH 12/16] Improve dockerfile and compose structure --- Dockerfile | 41 ++-- docker-compose.yml | 84 ++++++--- entrypoint.sh | 14 +- pyproject.toml | 4 +- tasks.py | 454 +++++++++++---------------------------------- 5 files changed, 192 insertions(+), 405 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6df0fc1e1a7..e1abeea137e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,34 +1,31 @@ FROM geonode/geonode-base:latest-ubuntu-24.04 -LABEL GeoNode development team -RUN mkdir -p /usr/src/geonode -# copy local geonode src inside container -COPY . /usr/src/geonode/ +ENV LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 WORKDIR /usr/src/geonode -COPY wait-for-databases.sh /usr/bin/wait-for-databases -RUN chmod +x /usr/bin/wait-for-databases -RUN chmod +x /usr/src/geonode/tasks.py \ - && chmod +x /usr/src/geonode/entrypoint.sh +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + curl \ + wget \ + unzip \ + gnupg2 \ + locales \ + netcat-openbsd \ + && sed -i -e 's/# C.UTF-8 UTF-8/C.UTF-8 UTF-8/' /etc/locale.gen \ + && locale-gen \ + && apt-get autoremove --purge -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --no-cache-dir -U pip setuptools wheel COPY celery-cmd /usr/bin/celery-cmd RUN chmod +x /usr/bin/celery-cmd -# # Install "geonode-contribs" apps -# RUN cd /usr/src; git clone https://github.com/GeoNode/geonode-contribs.git -b master -# # Install logstash and centralized dashboard dependencies -# RUN cd /usr/src/geonode-contribs/geonode-logstash; pip install --upgrade -e . \ -# cd /usr/src/geonode-contribs/ldap; pip install --upgrade -e . +COPY . /usr/src/geonode/ -RUN yes w | pip install -e . +RUN chmod +x /usr/src/geonode/tasks.py /usr/src/geonode/entrypoint.sh -# Cleanup apt update lists -RUN apt-get autoremove --purge &&\ - apt-get clean &&\ - rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir -e . -# Export ports EXPOSE 8000 - -# We provide no command or entrypoint as this image can be used to serve the django project or run celery tasks -# ENTRYPOINT /usr/src/geonode/entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index ed32f0d998c..5b2362f7050 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -# Common Django template for GeoNode and Celery services below x-common-django: &default-common-django image: geonode/geonode:latest-ubuntu-24.04 @@ -9,7 +8,7 @@ x-common-django: env_file: - .env${APP_ENV:-} volumes: - - .:/usr/src/geonode/ + # - './src:/usr/src/geonode' # Uncomment for local development - statics:/mnt/volumes/statics - geoserver-data-dir:/geoserver_data/data - backup-restore:/backup_restore @@ -18,15 +17,22 @@ x-common-django: depends_on: db: condition: service_healthy + redis: + condition: service_healthy + memcached: + condition: service_healthy services: # Our custom django application. It includes Geonode. django: << : *default-common-django + build: + context: ./ + dockerfile: Dockerfile container_name: django4${COMPOSE_PROJECT_NAME} healthcheck: - test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://django:8000/" + test: "curl -m 10 --fail --silent --output /dev/null http://django:8000/" start_period: 60s interval: 60s timeout: 10s @@ -40,29 +46,19 @@ services: celery: << : *default-common-django container_name: celery4${COMPOSE_PROJECT_NAME} - depends_on: - django: - condition: service_healthy environment: - IS_CELERY=True entrypoint: ["/usr/src/geonode/entrypoint.sh"] command: "celery-cmd" - - # memcached service - memcached: - image: memcached:alpine - container_name: memcached4${COMPOSE_PROJECT_NAME} - command: memcached ${MEMCACHED_OPTIONS:-} - restart: unless-stopped healthcheck: - test: nc -z 127.0.0.1 11211 - interval: 30s - timeout: 30s - retries: 5 + test: "celery -b ${BROKER_URL} inspect ping" start_period: 30s + interval: 30s + timeout: 10s + retries: 2 # Nginx is serving django static and media files and proxies to django and geonode - geonode: + nginx: image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: @@ -77,6 +73,25 @@ services: - nginx-certificates:/geonode-certificates - statics:/mnt/volumes/statics restart: unless-stopped + healthcheck: + test: "if ps -o pid | grep -w 1 > /dev/null; then echo 'nginx running'; else exit 1;fi" + interval: 10s + timeout: 5s + retries: 3 + start_period: 10s + + # memcached service + memcached: + image: memcached:alpine + container_name: memcached4${COMPOSE_PROJECT_NAME} + command: memcached ${MEMCACHED_OPTIONS} + restart: unless-stopped + healthcheck: + test: ["CMD", "nc", "-z", "127.0.0.1", "11211"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s # Gets and installs letsencrypt certificates letsencrypt: @@ -87,17 +102,20 @@ services: volumes: - nginx-certificates:/geonode-certificates restart: unless-stopped + depends_on: + nginx: + condition: service_healthy # Geoserver backend geoserver: image: geonode/geoserver:2.28.x-latest container_name: geoserver4${COMPOSE_PROJECT_NAME} healthcheck: - test: "curl -m 10 --fail --silent --write-out 'HTTP CODE : %{http_code}\n' --output /dev/null http://geoserver:8080/geoserver/ows" - start_period: 60s - interval: 60s + test: ["CMD", "curl", "-f", "http://geoserver:8080/geoserver/web/wicket/resource/org.geoserver.web.GeoServerBasePage/img/logo.png"] + start_period: 30s + interval: 30s timeout: 10s - retries: 2 + retries: 3 env_file: - .env${APP_ENV:-} ports: @@ -123,7 +141,10 @@ services: - geoserver-data-dir:/geoserver_data/data restart: unless-stopped healthcheck: - test: "ls -A '/geoserver_data/data' | wc -l" + test: ["CMD-SHELL", "[ $$(ls -A /geoserver_data/data | wc -l) -gt 0 ] || exit 1"] + interval: 10s + timeout: 5s + retries: 3 # PostGIS database. db: @@ -138,10 +159,13 @@ services: - dbbackups:/pg_backups restart: unless-stopped healthcheck: - test: "pg_isready -d postgres -U postgres" - # uncomment to enable remote connections to postgres - #ports: - # - "5432:5432" + test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + # uncomment to enable remote connections to postgres + #ports: + # - "5432:5432" # Vanilla Redis service. This is needed by celery redis: @@ -150,6 +174,12 @@ services: volumes: - redisdata:/data restart: unless-stopped + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 20s + timeout: 3s + retries: 3 + start_period: 5s volumes: statics: diff --git a/entrypoint.sh b/entrypoint.sh index 18c8ab5269f..8f5b9f1353c 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,7 +14,7 @@ invoke () { echo "$@ tasks done" } -# Start cron && memcached services +# Start cron service service cron restart echo $"\n\n\n" @@ -27,16 +27,6 @@ invoke update source $HOME/.bashrc source $HOME/.override_env -echo POSTGRES_USER=$POSTGRES_USER -echo POSTGRES_PASSWORD=$POSTGRES_PASSWORD -echo DATABASE_URL=$DATABASE_URL -echo GEODATABASE_URL=$GEODATABASE_URL -echo SITEURL=$SITEURL -echo ALLOWED_HOSTS=$ALLOWED_HOSTS -echo GEOSERVER_PUBLIC_LOCATION=$GEOSERVER_PUBLIC_LOCATION - -# invoke waitfordbs - cmd="$@" if [ ${IS_CELERY} = "true" ] || [ ${IS_CELERY} = "True" ] @@ -65,6 +55,6 @@ echo "-----------------------------------------------------" echo "FINISHED DJANGO ENTRYPOINT --------------------------" echo "-----------------------------------------------------" -# Run the CMD +# Run the CMD echo "got command $cmd" exec $cmd diff --git a/pyproject.toml b/pyproject.toml index 46c7970e1b4..6909653d4a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ dependencies = [ "redis==6.2.0", # geopython dependencies - "pycsw @ git+https://github.com/geopython/pycsw.git@3.0.0-beta1", + "pycsw @ https://github.com/geopython/pycsw/archive/refs/tags/3.0.0-beta1.zip", "pyproj<3.8.0", "OWSLib==0.34.1", "SQLAlchemy==2.0.50", @@ -97,7 +97,7 @@ dependencies = [ "geonode-pinax-notifications==6.0.0.3", # GeoNode org maintained apps. - "django_geonode_mapstore_client @ git+https://github.com/GeoNode/geonode-mapstore-client.git@master", + "django_geonode_mapstore_client @ https://github.com/GeoNode/geonode-mapstore-client/archive/refs/heads/master.zip", "django-avatar==8.0.1", "geonode-oauth-toolkit==2.2.2.2", "geonode-announcements==2.0.2.2", diff --git a/tasks.py b/tasks.py index 86513d3380b..54a6b16b91b 100755 --- a/tasks.py +++ b/tasks.py @@ -1,30 +1,10 @@ # -*- coding: utf-8 -*- -######################################################################### -# -# Copyright (C) 2016 OSGeo -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -######################################################################### import os import re import ast import json import time -import docker import socket -import ipaddress import logging import datetime from pathlib import Path @@ -32,45 +12,43 @@ from urllib.parse import urlparse, urlunparse from invoke import task -BOOTSTRAP_IMAGE_CHEIP = "codenvy/che-ip:nightly" - logger = logging.getLogger(__name__) -@task -def waitfordbs(ctx): - print("**************************databases*******************************") - db_host = os.getenv("DATABASE_HOST", "db") - ctx.run(f"/usr/bin/wait-for-databases {db_host}", pty=True) - - @task def update(ctx): print("***************************setting env*********************************") - ctx.run("env", pty=True) pub_host = _geonode_public_host() - print(f"Public Hostname is {pub_host}") + print(f"Public Hostname or IP is {pub_host}") pub_port = _geonode_public_port() print(f"Public PORT is {pub_port}") + pub_protocol = "https" if pub_port == "443" else "http" if pub_protocol == "https" or pub_port == "80": pub_port = None + db_url = _update_db_connstring() geodb_url = _update_geodb_connstring() + geonode_docker_host = None - for _cnt in range(1, 29): + for _cnt in range(1, 60): try: - geonode_docker_host = str(socket.gethostbyname("geonode")) + geonode_docker_host = str(socket.gethostbyname("nginx")) break except Exception: - print(f"...waiting for NGINX to pop-up...{_cnt}") - time.sleep(1) - - override_env = "$HOME/.override_env" + if _cnt % 4 == 0: + print(f"...waiting for NGINX to pop-up...{_cnt // 4}") + time.sleep(0.25) + + if not geonode_docker_host: + geonode_docker_host = "127.0.0.1" + + override_env = os.path.expandvars("$HOME/.override_env") if os.path.exists(override_env): - os.remove(override_env) - else: - print(f"Can not delete the {override_env} file as it doesn't exists") + try: + os.remove(override_env) + except OSError: + pass if pub_port: siteurl = f"{pub_protocol}://{pub_host}:{pub_port}/" @@ -78,6 +56,7 @@ def update(ctx): else: siteurl = f"{pub_protocol}://{pub_host}/" gs_pub_loc = f"http://{pub_host}/geoserver/" + envs = { "local_settings": str(_localsettings()), "siteurl": os.environ.get("SITEURL", siteurl), @@ -89,6 +68,7 @@ def update(ctx): "geodburl": os.environ.get("GEODATABASE_URL", geodb_url), "static_root": os.environ.get("STATIC_ROOT", "/mnt/volumes/statics/static/"), "media_root": os.environ.get("MEDIA_ROOT", "/mnt/volumes/statics/uploaded/"), + "asset_root": os.environ.get("ASSETS_ROOT", "/mnt/volumes/statics/assets/"), "geoip_path": os.environ.get("GEOIP_PATH", "/mnt/volumes/statics/geoip.db"), "geonode_geodb_passwd": os.environ.get("GEONODE_GEODATABASE_PASSWORD", "geonode_data"), "default_backend_datastore": os.environ.get("DEFAULT_BACKEND_DATASTORE", "datastore"), @@ -103,204 +83,55 @@ def update(ctx): "gs_admin_pwd": os.environ.get("GEOSERVER_ADMIN_PASSWORD", "geoserver"), "override_fn": override_env, } + try: current_allowed = ast.literal_eval( - os.getenv("ALLOWED_HOSTS") - or "['{public_fqdn}', '{public_host}', 'localhost', 'django', 'geonode',]".format(**envs) + os.getenv("ALLOWED_HOSTS") or "['{public_fqdn}', '{public_host}', 'localhost', 'django', 'geonode',]".format(**envs) ) except ValueError: current_allowed = [] current_allowed.extend([str(pub_host), f"{pub_host}:{pub_port}"]) - allowed_hosts = [str(c) for c in current_allowed] + ['"geonode"', '"django"'] - - ctx.run( - "echo export DJANGO_SETTINGS_MODULE=\ -{local_settings} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEOIP_PATH=\ -{geoip_path} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEONODE_GEODATABASE_PASSWORD=\ -{geonode_geodb_passwd} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export DEFAULT_BACKEND_DATASTORE=\ -{default_backend_datastore} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEONODE_DATABASE_PASSWORD=\ -{geonode_db_passwd} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEONODE_GEODATABASE=\ -{geonode_geodb} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export DATABASE_URL=\ -{db_url} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEODATABASE_URL=\ -{geodb_url} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEONODE_DATABASE=\ -{geonode_db} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEOSERVER_LOCATION=\ -{gs_loc} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEOSERVER_WEB_UI_LOCATION=\ -{gs_web_ui_loc} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEOSERVER_PUBLIC_LOCATION=\ -{gs_pub_loc} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEOSERVER_ADMIN_PASSWORD=\ -{gs_admin_pwd} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export SITEURL=\ -{siteurl} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - 'echo export ALLOWED_HOSTS=\ -"\\"{}\\"" >> {override_fn}'.format( - allowed_hosts, **envs - ), - pty=True, - ) - ctx.run( - "echo export DATABASE_URL=\ -{dburl} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEODATABASE_URL=\ -{geodburl} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export STATIC_ROOT=\ -{static_root} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export MEDIA_ROOT=\ -{media_root} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export GEOIP_PATH=\ -{geoip_path} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export LOGIN_URL=\ -{siteurl}account/login/ >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export LOGOUT_URL=\ -{siteurl}account/logout/ >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export LOGIN_REDIRECT_URL=\ -{siteurl} >> {override_fn}".format( - **envs - ), - pty=True, - ) - ctx.run( - "echo export LOGOUT_REDIRECT_URL=\ -{siteurl} >> {override_fn}".format( - **envs - ), - pty=True, - ) + allowed_hosts = [str(c) for c in current_allowed] + ['"nginx"', '"django"'] + + # Preserved 100% original string format to avoid parsing errors + content = f"""export DJANGO_SETTINGS_MODULE={envs['local_settings']} + export GEONODE_GEODATABASE_PASSWORD={envs['geonode_geodb_passwd']} + export DEFAULT_BACKEND_DATASTORE={envs['default_backend_datastore']} + export GEONODE_DATABASE_PASSWORD={envs['geonode_db_passwd']} + export GEONODE_GEODATABASE={envs['geonode_geodb']} + export DATABASE_URL={envs['db_url']} + export GEODATABASE_URL={envs['geodb_url']} + export GEONODE_DATABASE={envs['geonode_db']} + export GEOSERVER_LOCATION={envs['gs_loc']} + export GEOSERVER_WEB_UI_LOCATION={envs['gs_web_ui_loc']} + export GEOSERVER_PUBLIC_LOCATION={envs['gs_pub_loc']} + export GEOSERVER_ADMIN_PASSWORD={envs['gs_admin_pwd']} + export SITEURL={envs['siteurl']} + export ALLOWED_HOSTS="\\"{allowed_hosts}\\"" + export DATABASE_URL={envs['dburl']} + export GEODATABASE_URL={envs['geodburl']} + export STATIC_ROOT={envs['static_root']} + export MEDIA_ROOT={envs['media_root']} + export GEOIP_PATH={envs['geoip_path']} + export LOGIN_URL={envs['siteurl']}account/login/ + export LOGOUT_URL={envs['siteurl']}account/logout/ + export LOGIN_REDIRECT_URL={envs['siteurl']} + export LOGOUT_REDIRECT_URL={envs['siteurl']}""" + + ctx.run(f'echo "{content}" >> {envs["override_fn"]}', pty=True) ctx.run(f"source {override_env}", pty=True) - print("****************************finalize env**********************************") - ctx.run("env", pty=True) @task def migrations(ctx): print("**************************migrations*******************************") - ctx.run(f"python manage.py migrate --noinput --settings={_localsettings()}", pty=True) + settings = _localsettings() ctx.run( - f"python manage.py migrate --noinput --settings={_localsettings()} --database=datastore", + f"source $HOME/.override_env && " + f"python manage.py migrate --noinput --settings={settings} && " + f"python manage.py migrate dynamic_models --noinput --settings={settings} --database=datastore", pty=True, ) - try: - ctx.run( - f"python manage.py rebuild_index --noinput --settings={_localsettings()}", - pty=True, - ) - except Exception: - pass @task @@ -311,102 +142,74 @@ def statics(ctx): media_root = os.environ.get("MEDIA_ROOT", "/mnt/volumes/statics/uploaded/") assets_root = os.environ.get("ASSETS_ROOT", "/mnt/volumes/statics/assets/") - ctx.run(f"mkdir -pv {static_root} {media_root} {assets_root}") ctx.run( + f"source $HOME/.override_env && " + f"mkdir -pv {static_root} {media_root} {assets_root} && " f"python manage.py collectstatic --noinput --settings={_localsettings()}", pty=True, ) except Exception: import traceback - traceback.print_exc() @task def prepare(ctx): print("**********************prepare fixture***************************") - ctx.run("rm -rf /tmp/default_oauth_apps_docker.json", pty=True) + for path in ["/tmp/default_oauth_apps_docker.json", "/tmp/default_site.json"]: + try: + os.remove(path) + except OSError: + pass + _prepare_oauth_fixture() - ctx.run("rm -rf /tmp/default_site.json", pty=True) _prepare_site_fixture() @task def fixtures(ctx): print("**************************fixtures********************************") - ctx.run( - f"python manage.py loaddata sample_admin \ ---settings={_localsettings()}", - pty=True, - ) - ctx.run( - f"python manage.py loaddata /tmp/default_oauth_apps_docker.json \ ---settings={_localsettings()}", - pty=True, - ) - ctx.run( - f"python manage.py loaddata geonode/base/fixtures/initial_data.json \ ---settings={_localsettings()}", - pty=True, - ) - - # Loading additional project fixtures - from django.conf import settings - - project_fixtures = getattr(settings, "PROJECT_FIXTURES", []) - - for fixture in project_fixtures: - if fixture: - print(f"Loading project fixture: {fixture}") + settings = _localsettings() + + base_cmd = ( + f"source $HOME/.override_env && " + f"python manage.py loaddata sample_admin --settings={settings} && " + f"python manage.py loaddata /tmp/default_oauth_apps_docker.json --settings={settings} && " + f"python manage.py loaddata /tmp/default_site.json --settings={settings} && " + f"python manage.py loaddata initial_data.json --settings={settings}" + ) + ctx.run(base_cmd, pty=True) + + from django.conf import settings as django_settings + project_fixtures = getattr(django_settings, "PROJECT_FIXTURES", []) + + if project_fixtures: + fixture_cmds = [ + f"python manage.py loaddata {fix} --settings={settings}" + for fix in project_fixtures if fix + ] + if fixture_cmds: try: - ctx.run( - f"python manage.py loaddata {fixture} --settings={_localsettings()}", - pty=True - ) + ctx.run("source $HOME/.override_env && " + " && ".join(fixture_cmds), pty=True) except Exception as e: - print(f"Warning: Failed to load fixture {fixture}: {e}") - - -@task -def collectstatic(ctx): - print("************************static artifacts******************************") - ctx.run( - f"django-admin collectstatic --noinput \ ---settings={_localsettings()}", - pty=True, - ) + print(f"Warning: Failed to load project fixtures: {e}") @task def updateadmin(ctx): print("***********************update admin details**************************") - ctx.run("rm -rf /tmp/django_admin_docker.json", pty=True) + try: + os.remove("/tmp/django_admin_docker.json") + except OSError: + pass + _prepare_admin_fixture( os.environ.get("ADMIN_PASSWORD", "admin"), os.environ.get("ADMIN_EMAIL", "admin@example.org"), ) ctx.run( - f"django-admin loaddata /tmp/django_admin_docker.json \ ---settings={_localsettings()}", - pty=True, - ) - - -@task -def loadthesauri(ctx): - print("**************************thesauri*******************************") - ctx.run( - f"python manage.py thesaurus autoload --settings={_localsettings()}", - pty=True, - ) - - -@task -def collectmetrics(ctx): - print("************************collect metrics******************************") - ctx.run( - f"python -W ignore manage.py collect_metrics \ ---settings={_localsettings()} -n -t xml", + f"source $HOME/.override_env && " + f"django-admin loaddata /tmp/django_admin_docker.json --settings={_localsettings()}", pty=True, ) @@ -415,64 +218,39 @@ def collectmetrics(ctx): def initialized(ctx): print("**************************init file********************************") static_root = os.environ.get("STATIC_ROOT", "/mnt/volumes/statics/static/") - lockfile_dir = Path(static_root).parent # quite ugly, we're assuming such dir exists and is writable - ctx.run(f"date > {lockfile_dir}/geonode_init.lock") - - -def _docker_host_ip(): + lockfile = Path(static_root).parent / "geonode_init.lock" + try: - client = docker.from_env(version="1.24") - ip_list = client.containers.run(BOOTSTRAP_IMAGE_CHEIP, network_mode="host").split("\n") + lockfile.write_text(datetime.datetime.now().ctime() + "\n") except Exception: - import traceback + ctx.run(f"date > {lockfile}") - traceback.print_exc() - ip_list = [ - "127.0.0.1", - ] - if len(ip_list) > 1: - print( - f"Docker daemon is running on more than one \ -address {ip_list}" - ) - print(f"Only the first address:{ip_list[0]} will be returned!") - else: - print( - f"Docker daemon is running at the following \ -address {ip_list[0]}" - ) - return ip_list[0] - -def _is_valid_ip(ip): +def _docker_host_ip(): try: - ipaddress.IPv4Address(ip) - return True - except Exception as e: - return False + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + local_ip = s.getsockname()[0] + s.close() + return local_ip + except Exception: + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "127.0.0.1" def _container_exposed_port(component, instname): port = "80" try: client = docker.from_env(version="1.24") - ports_dict = json.dumps( - [ - c.attrs["Config"]["ExposedPorts"] - for c in client.containers.list( - filters={ - "label": f"org.geonode.component={component}", - "status": "running", - } - ) - if str(instname) in c.name - ][0] - ) - for key in json.loads(ports_dict): - port = re.split("/tcp", key)[0] + for c in client.containers.list(filters={"label": f"org.geonode.component={component}", "status": "running"}): + if str(instname) in c.name: + exposed_ports = c.attrs["Config"].get("ExposedPorts", {}) + for key in exposed_ports: + return re.split("/tcp", key)[0] except Exception: import traceback - traceback.print_exc() return port @@ -502,8 +280,7 @@ def _update_geodb_connstring(): def _localsettings(): - settings = os.getenv("DJANGO_SETTINGS_MODULE", "geonode.settings") - return settings + return os.getenv("DJANGO_SETTINGS_MODULE", "geonode_project.settings") def _geonode_public_host(): @@ -513,13 +290,6 @@ def _geonode_public_host(): return gn_pub_hostip -def _geonode_public_host_ip(): - gn_pub_hostip = os.getenv("GEONODE_LB_HOST_IP", None) - if not gn_pub_hostip or not _is_valid_ip(gn_pub_hostip): - gn_pub_hostip = _docker_host_ip() - return gn_pub_hostip - - def _geonode_public_port(): gn_pub_port = os.getenv("GEONODE_LB_PORT", "") if not gn_pub_port: From 5eefc585cb0803cfd9fc0ad0ef29631be16adc6d Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 10 Jun 2026 15:01:20 +0200 Subject: [PATCH 13/16] Improve test speed --- .env_test | 2 +- .github/workflows/run-test-suite.yml | 2 +- .github/workflows/tests.yml | 2 +- tasks.py | 9 +++++++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.env_test b/.env_test index b377f2e85f8..a99dab27966 100644 --- a/.env_test +++ b/.env_test @@ -1,4 +1,4 @@ -COMPOSE_PROJECT_NAME=geonode +COMPOSE_PROJECT_NAME=geonode_test # See https://github.com/containers/podman/issues/13889 # DOCKER_BUILDKIT=0 DOCKER_ENV=test diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index 8e6cc8f5b8b..b1f59f522e3 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,7 +28,7 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: APP_ENV=_test docker compose up -d + run: APP_ENV=_test docker compose up -d --no-deps db geoserver django nginx - name: Setup test databases run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3a3a41617a9..3f0da2a79ad 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: APP_ENV=_test docker compose build --progress plain + run: APP_ENV=_test docker compose build --no-deps db geoserver django nginx --progress plain - name: Save built Docker images run: | diff --git a/tasks.py b/tasks.py index 54a6b16b91b..101f97b7c49 100755 --- a/tasks.py +++ b/tasks.py @@ -364,3 +364,12 @@ def _prepare_admin_fixture(admin_password, admin_email): ] with open("/tmp/django_admin_docker.json", "w") as fixturefile: json.dump(default_fixture, fixturefile) + + +@task +def loadthesauri(ctx): + print("**************************thesauri*******************************") + ctx.run( + f"python manage.py thesaurus autoload --settings={_localsettings()}", + pty=True, + ) From e67a0a5aa2e48cf59aa282316fb0dc7eb253a01f Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 10 Jun 2026 15:02:59 +0200 Subject: [PATCH 14/16] Improve test speed --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f0da2a79ad..8b833857e45 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: APP_ENV=_test docker compose build --no-deps db geoserver django nginx --progress plain + run: APP_ENV=_test docker compose build db geoserver django nginx --progress plain - name: Save built Docker images run: | From 4b2d1f1239bd3d3f22cb1b091466253aa8491641 Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Wed, 10 Jun 2026 15:30:56 +0200 Subject: [PATCH 15/16] fix test workflow --- .github/workflows/run-test-suite.yml | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index b1f59f522e3..adcfdd22c7a 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,7 +28,7 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: APP_ENV=_test docker compose up -d --no-deps db geoserver django nginx + run: APP_ENV=_test docker compose up -d --no-deps db geoserver django nginx data-dir-conf - name: Setup test databases run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b833857e45..49b8f697fa0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: APP_ENV=_test docker compose build db geoserver django nginx --progress plain + run: APP_ENV=_test docker compose build db geoserver django nginx data-dir-conf --progress plain - name: Save built Docker images run: | From 68d92cc3675eca129ecca40a3788767a0627485e Mon Sep 17 00:00:00 2001 From: Mattia Giupponi Date: Thu, 11 Jun 2026 10:52:23 +0200 Subject: [PATCH 16/16] Update env variable name to environment and remove trailing underscore --- .env_dev | 213 +++++++++++++++++++++++++++ .github/workflows/run-test-suite.yml | 20 +-- .github/workflows/tests.yml | 2 +- docker-compose.yml | 10 +- 4 files changed, 229 insertions(+), 16 deletions(-) create mode 100644 .env_dev diff --git a/.env_dev b/.env_dev new file mode 100644 index 00000000000..755e6c3b39a --- /dev/null +++ b/.env_dev @@ -0,0 +1,213 @@ +COMPOSE_PROJECT_NAME=geonode +# See https://github.com/containers/podman/issues/13889 +# DOCKER_BUILDKIT=0 +DOCKER_ENV=production +BACKUPS_VOLUME_DRIVER=local + +C_FORCE_ROOT=1 +FORCE_REINIT=false +INVOKE_LOG_STDOUT=true + +# LANGUAGE_CODE=it-it +# LANGUAGES=(('en-us','English'),('it-it','Italiano')) + +DJANGO_SETTINGS_MODULE=geonode.settings +GEONODE_INSTANCE_NAME=geonode + +# ################# +# backend +# ################# +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +GEONODE_DATABASE=geonode +GEONODE_DATABASE_USER=geonode +GEONODE_DATABASE_PASSWORD=geonode +GEONODE_GEODATABASE=geonode_data +GEONODE_GEODATABASE_USER=geonode +GEONODE_GEODATABASE_PASSWORD=geonode +GEONODE_DATABASE_SCHEMA=public +GEONODE_GEODATABASE_SCHEMA=public +DATABASE_HOST=localhost +DATABASE_PORT=5432 +DATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode +GEODATABASE_URL=postgis://geonode:geonode@localhost:5432/geonode_data +GEONODE_DB_CONN_MAX_AGE=0 +GEONODE_DB_CONN_TOUT=5 +DEFAULT_BACKEND_DATASTORE=datastore +BROKER_URL=redis://localhost:6379/0 +CELERY_BEAT_SCHEDULER=celery.beat:PersistentScheduler +ASYNC_SIGNALS=False + +# Harvesting Monitoring configuration +HARVESTING_MONITOR_ENABLED=False +HARVESTING_MONITOR_DELAY=60 + +SITEURL=http://localhost:8000/ + +ALLOWED_HOSTS="['django', '*']" + +# Data Uploader +TIME_ENABLED=True +MOSAIC_ENABLED=False + +# ################# +# nginx +# HTTPD Server +# ################# +GEONODE_LB_HOST_IP=django +GEONODE_LB_PORT=8000 +GEOSERVER_LB_HOST_IP=geoserver +GEOSERVER_LB_PORT=8080 +NGINX_BASE_URL=http://localhost + +# IP or domain name and port where the server can be reached on HTTPS (leave HOST empty if you want to use HTTP only) +# port where the server can be reached on HTTPS +HTTP_HOST=localhost +HTTPS_HOST= + +HTTP_PORT=8000 +HTTPS_PORT=443 + +# Let's Encrypt certificates for https encryption. You must have a domain name as HTTPS_HOST (doesn't work +# with an ip) and it must be reachable from the outside. This can be one of the following : +# disabled : we do not get a certificate at all (a placeholder certificate will be used) +# staging : we get staging certificates (are invalid, but allow to test the process completely and have much higher limit rates) +# production : we get a normal certificate (default) +LETSENCRYPT_MODE=disabled +# LETSENCRYPT_MODE=staging +# LETSENCRYPT_MODE=production + +RESOLVER=127.0.0.11 + +# ################# +# geoserver +# ################# +GEOSERVER_WEB_UI_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_PUBLIC_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_LOCATION=http://localhost:8080/geoserver/ +GEOSERVER_ADMIN_USER=admin +GEOSERVER_ADMIN_PASSWORD=geoserver + +OGC_REQUEST_TIMEOUT=30 +OGC_REQUEST_MAX_RETRIES=1 +OGC_REQUEST_BACKOFF_FACTOR=0.3 +OGC_REQUEST_POOL_MAXSIZE=10 +OGC_REQUEST_POOL_CONNECTIONS=10 + +# Java Options & Memory +ENABLE_JSONP=true +outFormat=text/javascript +GEOSERVER_JAVA_OPTS='-Djava.awt.headless=true -Xms4G -Xmx4G -Dgwc.context.suffix=gwc -XX:+UnlockDiagnosticVMOptions -XX:+LogVMOutput -XX:LogFile=/var/log/jvm.log -XX:PerfDataSamplingInterval=500 -XX:SoftRefLRUPolicyMSPerMB=36000 -XX:-UseGCOverheadLimit -XX:ParallelGCThreads=4 -Dfile.encoding=UTF8 -Djavax.servlet.request.encoding=UTF-8 -Djavax.servlet.response.encoding=UTF-8 -Duser.timezone=GMT -Dorg.geotools.shapefile.datetime=false -DGS-SHAPEFILE-CHARSET=UTF-8 -DGEOSERVER_CSRF_DISABLED=true -DPRINT_BASE_URL=http://localhost:8080/geoserver/pdf -DALLOW_ENV_PARAMETRIZATION=true -Xbootclasspath/a:/usr/local/tomcat/webapps/geoserver/WEB-INF/lib/marlin-0.9.3-Unsafe.jar -Dsun.java2d.renderer=org.marlin.pisces.MarlinRenderingEngine' + +# ################# +# Security +# ################# +# Admin Settings +# +# ADMIN_PASSWORD is used to overwrite the GeoNode admin password **ONLY** the first time +# GeoNode is run. If you need to overwrite it again, you need to set the env var FORCE_REINIT, +# otherwise the invoke updateadmin task will be skipped and the current password already stored +# in DB will honored. + +ADMIN_USERNAME=admin +ADMIN_PASSWORD=admin +ADMIN_EMAIL=admin@localhost + +# EMAIL Notifications +EMAIL_ENABLE=False +DJANGO_EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +DJANGO_EMAIL_HOST=localhost +DJANGO_EMAIL_PORT=25 +DJANGO_EMAIL_HOST_USER= +DJANGO_EMAIL_HOST_PASSWORD= +DJANGO_EMAIL_USE_TLS=False +DJANGO_EMAIL_USE_SSL=False +DEFAULT_FROM_EMAIL='GeoNode ' + +# Session/Access Control +LOCKDOWN_GEONODE=False +X_FRAME_OPTIONS="SAMEORIGIN" +SESSION_EXPIRED_CONTROL_ENABLED=True +DEFAULT_ANONYMOUS_PERMISSIONS=download +DEFAULT_REGISTERED_MEMBERS_PERMISSIONS=download + +CORS_ALLOW_ALL_ORIGINS=True +GEOSERVER_CORS_ENABLED=True +GEOSERVER_CORS_ALLOWED_ORIGINS=* +GEOSERVER_CORS_ALLOWED_METHODS=GET,POST,PUT,DELETE,HEAD,OPTIONS +GEOSERVER_CORS_ALLOWED_HEADERS=* + +# Users Registration +ACCOUNT_OPEN_SIGNUP=True +ACCOUNT_SIGNUP_FIELDS="['email*', 'username*', 'password1*', 'password2*']" +ACCOUNT_APPROVAL_REQUIRED=False +ACCOUNT_CONFIRM_EMAIL_ON_GET=False +ACCOUNT_EMAIL_VERIFICATION=none +ACCOUNT_LOGIN_METHODS="{'email', 'username'}" +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_REGISTERED_MEMBERS_GROUP_NAME=True +AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS=True + +# OAuth2 +OAUTH2_API_KEY= +OAUTH2_CLIENT_ID=Jrchz2oPY3akmzndmgUTYrs9gczlgoV20YPSvqaV +OAUTH2_CLIENT_SECRET=rCnp5txobUo83EpQEblM8fVj3QT5zb5qRfxNsuPzCqZaiRyIoxM4jdgMiZKFfePBHYXCLd7B8NlkfDBY9HKeIQPcy5Cp08KQNpRHQbjpLItDHv12GvkSeXp6OxaUETv3 + +# GeoNode APIs +API_LOCKDOWN=False + +# ################# +# Production and +# Monitoring +# ################# +DEBUG=True + +SECRET_KEY='myv-y4#7j-d*p-__@j#*3z@!y24fz8%^z2v6atuy4bo9vqr1_a' + +CACHE_BUSTING_STATIC_ENABLED=False + +MEMCACHED_ENABLED=False +MEMCACHED_BACKEND=django.core.cache.backends.memcached.PyLibMCCache +MEMCACHED_LOCATION=127.0.0.1:11211 +MEMCACHED_LOCK_EXPIRE=3600 +MEMCACHED_LOCK_TIMEOUT=10 +PERMISSION_CACHE_EXPIRATION_TIME=604800 +# +# Options for memcached binary, e.g. -vvv to log all requests and cache hits +# +MEMCACHED_OPTIONS= + +MAX_DOCUMENT_SIZE=200 +CLIENT_RESULTS_LIMIT=5 +API_LIMIT_PER_PAGE=1000 + +# GIS Client +GEONODE_CLIENT_LAYER_PREVIEW_LIBRARY=mapstore +MAPBOX_ACCESS_TOKEN= +GOOGLE_API_KEY= + +# Other Options/Contribs +MODIFY_TOPICCATEGORY=True +AVATAR_GRAVATAR_SSL=True +EXIF_ENABLED=True +CREATE_LAYER=True +FAVORITE_ENABLED=True + +# Advanced Workflow +RESOURCE_PUBLISHING=False +ADMIN_MODERATE_UPLOADS=False + +# PostgreSQL +POSTGRESQL_MAX_CONNECTIONS=200 + +# Common containers restart policy +RESTART_POLICY_CONDITION="on-failure" +RESTART_POLICY_DELAY="5s" +RESTART_POLICY_MAX_ATTEMPTS="3" +RESTART_POLICY_WINDOW=120s + +DEFAULT_MAX_PARALLEL_UPLOADS_PER_USER=5 +UPSERT_CHUNK_SIZE= 100 +UPSERT_LIMIT_ERROR_LOG=100 + +# Enable or not the XLSX / XLS upload +XLSX_UPLOAD_ENABLED=False diff --git a/.github/workflows/run-test-suite.yml b/.github/workflows/run-test-suite.yml index adcfdd22c7a..906312c306f 100644 --- a/.github/workflows/run-test-suite.yml +++ b/.github/workflows/run-test-suite.yml @@ -28,20 +28,20 @@ jobs: run: docker load -i docker_images/django.tar - name: Start the stack - run: APP_ENV=_test docker compose up -d --no-deps db geoserver django nginx data-dir-conf + run: ENVIRONMENT=test docker compose up -d --no-deps db geoserver django nginx data-dir-conf - name: Setup test databases run: | - APP_ENV=_test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" - APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_postgres - APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode - APP_ENV=_test docker compose exec db createdb -U postgres -T postgres test_geonode_data - APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" - APP_ENV=_test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" + ENVIRONMENT=test docker compose exec db psql -U postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid();" + ENVIRONMENT=test docker compose exec db createdb -U postgres -T postgres test_postgres + ENVIRONMENT=test docker compose exec db createdb -U postgres -T postgres test_geonode + ENVIRONMENT=test docker compose exec db createdb -U postgres -T postgres test_geonode_data + ENVIRONMENT=test docker compose exec db psql -U postgres -d test_geonode -c "CREATE EXTENSION IF NOT EXISTS postgis;" + ENVIRONMENT=test docker compose exec db psql -U postgres -d test_geonode_data -c "CREATE EXTENSION IF NOT EXISTS postgis;" - name: Run ${{ inputs.suite_name }} run: | - APP_ENV=_test docker compose exec -T django bash -s <<'BASH' + ENVIRONMENT=test docker compose exec -T django bash -s <<'BASH' set -euo pipefail ${{ inputs.test_command }} BASH @@ -52,8 +52,8 @@ jobs: - name: Codecov run: | - APP_ENV=_test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" + ENVIRONMENT=test docker compose exec django bash -c "bash <(curl -s https://codecov.io/bash) -t 2c0e7780-1640-45f0-93a3-e103b057d8c8" - name: Stop the stack if: always() - run: APP_ENV=_test docker compose down -v + run: ENVIRONMENT=test docker compose down -v diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 49b8f697fa0..37d65a57276 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Build all services (Docker Compose v2) - run: APP_ENV=_test docker compose build db geoserver django nginx data-dir-conf --progress plain + run: ENVIRONMENT=test docker compose build db geoserver django nginx data-dir-conf --progress plain - name: Save built Docker images run: | diff --git a/docker-compose.yml b/docker-compose.yml index 5b2362f7050..ef6b31f5067 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ x-common-django: dockerfile: Dockerfile restart: unless-stopped env_file: - - .env${APP_ENV:-} + - .env${ENVIRONMENT:+_${ENVIRONMENT}} volumes: # - './src:/usr/src/geonode' # Uncomment for local development - statics:/mnt/volumes/statics @@ -62,7 +62,7 @@ services: image: geonode/nginx:1.31.0-latest container_name: nginx4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV:-} + - .env${ENVIRONMENT:+_${ENVIRONMENT}} environment: - RESOLVER=127.0.0.11 ports: @@ -98,7 +98,7 @@ services: image: geonode/letsencrypt:2.6.0-latest container_name: letsencrypt4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV:-} + - .env${ENVIRONMENT:+_${ENVIRONMENT}} volumes: - nginx-certificates:/geonode-certificates restart: unless-stopped @@ -117,7 +117,7 @@ services: timeout: 10s retries: 3 env_file: - - .env${APP_ENV:-} + - .env${ENVIRONMENT:+_${ENVIRONMENT}} ports: - "8080:8080" volumes: @@ -153,7 +153,7 @@ services: command: postgres -c "max_connections=${POSTGRESQL_MAX_CONNECTIONS:-200}" container_name: db4${COMPOSE_PROJECT_NAME} env_file: - - .env${APP_ENV:-} + - .env${ENVIRONMENT:+_${ENVIRONMENT}} volumes: - dbdata:/var/lib/postgresql/data - dbbackups:/pg_backups