From 23ac4fc19b620143e6bb88ce84c7f24b742dd124 Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Thu, 3 Mar 2022 10:03:53 -0500 Subject: [PATCH 01/15] add placeholders for dev environment --- dispatch.yaml | 4 ++++ scripts/config/dev.yaml | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100755 scripts/config/dev.yaml diff --git a/dispatch.yaml b/dispatch.yaml index 4c85c5d..b3b71cf 100644 --- a/dispatch.yaml +++ b/dispatch.yaml @@ -17,6 +17,10 @@ dispatch: - url: "staging.api.crypto-service.io/*" service: crypto-service-staging + # dev - service +- url: "dev.api.crypto-service.io/*" + service: crypto-service-dev + # web - url: "wwww.crypto-service.io/*" service: default diff --git a/scripts/config/dev.yaml b/scripts/config/dev.yaml new file mode 100755 index 0000000..4f3050b --- /dev/null +++ b/scripts/config/dev.yaml @@ -0,0 +1,22 @@ +# Copyright 2017, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START app_yaml] +runtime: nodejs +env: flex +service: crypto-service-dev +resources: + cpu: 1 + memory_gb: 1.5 + +# [END app_yaml] From 3d1881953f230595e83259924a5c0b51960157fb Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Thu, 3 Mar 2022 10:34:13 -0500 Subject: [PATCH 02/15] update circle to use gcp secrets and env_version.json --- circle.yml | 191 +++++++++++++++++++------------------------ src/env_version.json | 9 ++ 2 files changed, 91 insertions(+), 109 deletions(-) create mode 100644 src/env_version.json diff --git a/circle.yml b/circle.yml index eea71b2..815294d 100644 --- a/circle.yml +++ b/circle.yml @@ -6,85 +6,104 @@ # than the prescribed lifecycle of 1.0. In general, we recommend using this generated # configuration as a reference rather than using it in production, though in most # cases it should duplicate the execution of your original 1.0 config. -version: 2 +version: 2.1 + +workflows: + build_deploy: + jobs: + - build + - deploy: + requires: + - build + context: + - gcloud<< pipeline.git.branch >> + filters: + branches: + only: + - master + - staging +orbs: + node: circleci/node@5.0.0 + jq: circleci/jq@2.2.0 + jobs: build: - working_directory: ~/API-market/crypto-service + working_directory: ~/crypto-service/src parallelism: 1 - shell: /bin/bash --login + shell: /bin/bash --login -e environment: MONGO_URL: mongodb://localhost:27017/data - CIRCLE_ARTIFACTS: /tmp/circleci-artifacts - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results - PYTHONPATH: ${PYTHONPATH}:${HOME}/google_appengine GCLOUD_PROJECT: cryptoservice SERVICE_VERSION: 1 # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images. # To see the list of pre-built images that CircleCI provides for most common languages see # https://circleci.com/docs/2.0/circleci-images/ docker: - - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 + - image: cimg/base:stable-18.04 - image: circleci/mongo:4.2.8 steps: # Machine Setup - - checkout - # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/' - - run: mkdir -p $CIRCLE_ARTIFACTS $CIRCLE_TEST_REPORTS - - run: - working_directory: ~/API-market/crypto-service - command: nvm install 12.12.0 && nvm alias default 12.12.0 - # Dependencies - # This would typically go in either a build or a build-and-test job when using workflows - # Restore the dependency cache - - restore_cache: - keys: - # This branch if available - - v1-dep-{{ .Branch }}- - # Default branch if not - - v1-dep-master- - # Any branch if there are none on the default branch - this should be unnecessary if you have your default branch configured correctly - - v1-dep- + - checkout: + path: ".." + - node/install: + node-version: '12.12.0' + # install npm dependencies and run build script + - node/install-packages: + override-ci-command: npm install + # Linter + - run: npm run lint + # Unit Tests + - run: npm test + - persist_to_workspace: + root: ~/ + paths: + - crypto-service + deploy: + working_directory: ~/crypto-service/src + parallelism: 1 + shell: /bin/bash --login -e + docker: + - image: google/cloud-sdk:363.0.0 + steps: + - attach_workspace: + at: ~/crypto-service + - node/install: + node-version: '12.12.0' # install gcloud and login to it - run: - working_directory: ~/API-market/crypto-service/src + working_directory: ~/ command: | - if [ "$CIRCLE_BRANCH" == "master" ] || [ "$CIRCLE_BRANCH" == "staging" ] - then - curl -o $HOME/google_appengine_1.9.30.zip https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.30.zip - unzip -q -d $HOME $HOME/google_appengine_1.9.30.zip - echo $GCLOUD_SERVICE_KEY | base64 --decode > ${HOME}/gcloud-service-key.json - echo Y | gcloud components update app - gcloud auth activate-service-account --key-file ${HOME}/gcloud-service-key.json - gcloud config set project $GCLOUD_PROJECT - fi - - # copy .env file from circleci injected environmental variables + echo $GCLOUD_SERVICE_ACCT_KEY | base64 --decode > ${HOME}/gcloud-service-key.json + gcloud auth activate-service-account --key-file ${HOME}/gcloud-service-key.json + gcloud config set project $GOOGLE_PROJECT_ID + gcloud config set app/cloud_build_timeout 2100 - run: - working_directory: ~/API-market/crypto-service/src command: | - if [ "$CIRCLE_BRANCH" == "master" ] - then - cp -rf ../scripts/config/prod.app.yaml ./app.yaml - echo $ENV_PROD | base64 --decode >> ./.env - echo $ENV_TEST_PROD | base64 --decode >> ./backend/test/config/.env.test - fi - if [ "$CIRCLE_BRANCH" == "staging" ] - then - cp -rf ../scripts/config/staging.app.yaml ./app.yaml - echo $ENV_STAGING | base64 --decode >> ./.env - echo $ENV_TEST_STAGING | base64 --decode >> ./backend/test/config/.env.test + # use new variable to map master -> prod + DEPLOY_ENV=$CIRCLE_BRANCH + if [ "$CIRCLE_BRANCH" == "master" ]; then + DEPLOY_ENV=prod fi - if [ "$CIRCLE_BRANCH" != "master" ] && [ "$CIRCLE_BRANCH" != "staging" ] - then - cp -rf ../scripts/config/staging.app.yaml ./app.yaml - echo $ENV_STAGING | base64 --decode >> ./.env - echo $ENV_TEST_STAGING | base64 --decode >> ./backend/test/config/.env.test + + cp -rf ./scripts/config/${DEPLOY_ENV}.app.yaml ./app.yaml + # process env_version.json + SECRET_VERSION=`cat env_version.json | jq -r ".${DEPLOY_ENV}.version"` + ENV_VERSION=`cat env_version.json | jq -r '.envVersion'` + # pull this from GCP + gcloud secrets versions access ${SECRET_VERSION} \ + --secret="$SECRET_NAME_OREID_SERVICE" \ + --format='get(payload.data)' | tr '_-' '/+' | base64 -d > .env + echo "Pulled down secret into .env" + # disable fail on error to output meaningful message + set +e + # grep ENV_VERSION is in env. This will fail if the incorrect version is not in the .env file + CHECK_ENV=`grep $ENV_VERSION .env` + if [ "$?" == "1" ]; then + echo ".env SCHEMA does not match: ${ENV_VERSION} not found in version: ${SECRET_VERSION}"; fi - - # create deploy_version file + # create deploy_version file - run: - working_directory: ~/API-market/crypto-service/src command: | ENV_HASH=$(sha1sum ./.env | awk '{print $1}') echo ENV_HASH = "\"$ENV_HASH\"" >> ./deploy-version @@ -93,60 +112,14 @@ jobs: CURRENTDATE=`date +"%A, %b %d, %Y %I:%M %p"` echo DEPLOY_DATE = "\"$CURRENTDATE\"" >> ./deploy-version - - # install npm dependencies and run build script - - run: - working_directory: ~/API-market/crypto-service/src - command: | - npm install - npm run build - # copy the folders we will deploy to a dist folder - - # install npm dependencies and run build script - - # Linter - - run: - working_directory: ~/API-market/crypto-service/src - command: npm run lint - - # Unit Tests - - run: - working_directory: ~/API-market/crypto-service/src - command: npm test + # build + - run: npm run build # deploy to Google Cloud - run: - working_directory: ~/API-market/crypto-service/src + working_directory: ~/API-market/oreid-service/web + no_output_timeout: 15m # timeout the build if the deploy hangs for longer than 15 minutes command: | - if [ "$CIRCLE_BRANCH" == "master" ] || [ "$CIRCLE_BRANCH" == "staging" ] - then - rm -rf ./node_modules - # deploy to AppEngine - #- ../scripts/deploy.sh --version=$SERVICE_VERSION - ../scripts/deploy.sh - fi - - # Save dependency cache - - save_cache: - key: v1-dep-{{ .Branch }}-{{ epoch }} - paths: - # This is a broad list of cache paths to include many possible development environments - # You can probably delete some of these entries - - vendor/bundle - - ~/virtualenvs - - ~/.m2 - - ~/.ivy2 - - ~/.bundle - - ~/.go_workspace - - ~/.gradle - - ~/.cache/bower - - # Teardown - # Save test results - - store_test_results: - path: /tmp/circleci-test-results - # Save artifacts - - store_artifacts: - path: /tmp/circleci-artifacts - - store_artifacts: - path: /tmp/circleci-test-results + rm -rf ./node_modules + # deploy to AppEngine + ./scripts/deploy.sh \ No newline at end of file diff --git a/src/env_version.json b/src/env_version.json new file mode 100644 index 0000000..d1c3314 --- /dev/null +++ b/src/env_version.json @@ -0,0 +1,9 @@ +{ + "envVersion": "2020-01-24_09:00", + "staging": { + "version": "1" + }, + "prod": { + "version": "1" + } +} \ No newline at end of file From eb7eb2f7fbdd06114a4a0dd01fa3495545e6fbfb Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Thu, 3 Mar 2022 11:53:02 -0500 Subject: [PATCH 03/15] add dev branch; install jq --- circle.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/circle.yml b/circle.yml index 815294d..d81b55c 100644 --- a/circle.yml +++ b/circle.yml @@ -22,6 +22,7 @@ workflows: only: - master - staging + - dev orbs: node: circleci/node@5.0.0 jq: circleci/jq@2.2.0 @@ -70,6 +71,7 @@ jobs: at: ~/crypto-service - node/install: node-version: '12.12.0' + - jq/install # install gcloud and login to it - run: working_directory: ~/ From e7d7ca541a03d29e426ecaeebf4809d559e6fe84 Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Thu, 3 Mar 2022 13:17:05 -0500 Subject: [PATCH 04/15] cleanup variable names; --- circle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index d81b55c..dbb2fe9 100644 --- a/circle.yml +++ b/circle.yml @@ -16,7 +16,7 @@ workflows: requires: - build context: - - gcloud<< pipeline.git.branch >> + - gcloud-<< pipeline.git.branch >> filters: branches: only: @@ -94,7 +94,7 @@ jobs: ENV_VERSION=`cat env_version.json | jq -r '.envVersion'` # pull this from GCP gcloud secrets versions access ${SECRET_VERSION} \ - --secret="$SECRET_NAME_OREID_SERVICE" \ + --secret="$SECRET_NAME_CRYPTO_SERVICE" \ --format='get(payload.data)' | tr '_-' '/+' | base64 -d > .env echo "Pulled down secret into .env" # disable fail on error to output meaningful message From d1146d4fe9a12415cd5740f921ffb453b9945338 Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Fri, 4 Mar 2022 13:20:59 -0500 Subject: [PATCH 05/15] rename file to correct name --- scripts/config/{dev.yaml => dev.app.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scripts/config/{dev.yaml => dev.app.yaml} (100%) diff --git a/scripts/config/dev.yaml b/scripts/config/dev.app.yaml similarity index 100% rename from scripts/config/dev.yaml rename to scripts/config/dev.app.yaml From df5309acade6ae27410d53ff0af1854c31ee62a7 Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Fri, 4 Mar 2022 13:24:43 -0500 Subject: [PATCH 06/15] fix working directory issues --- circle.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index dbb2fe9..e8a669c 100644 --- a/circle.yml +++ b/circle.yml @@ -88,7 +88,7 @@ jobs: DEPLOY_ENV=prod fi - cp -rf ./scripts/config/${DEPLOY_ENV}.app.yaml ./app.yaml + cp -rf ../scripts/config/${DEPLOY_ENV}.app.yaml ./app.yaml # process env_version.json SECRET_VERSION=`cat env_version.json | jq -r ".${DEPLOY_ENV}.version"` ENV_VERSION=`cat env_version.json | jq -r '.envVersion'` @@ -119,9 +119,8 @@ jobs: # deploy to Google Cloud - run: - working_directory: ~/API-market/oreid-service/web no_output_timeout: 15m # timeout the build if the deploy hangs for longer than 15 minutes command: | rm -rf ./node_modules # deploy to AppEngine - ./scripts/deploy.sh \ No newline at end of file + ../scripts/deploy.sh \ No newline at end of file From c2087b91a5d869aecff95b93379735a965dbabc1 Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Fri, 4 Mar 2022 13:42:31 -0500 Subject: [PATCH 07/15] output deploy_env --- circle.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index e8a669c..ae75cd6 100644 --- a/circle.yml +++ b/circle.yml @@ -87,7 +87,8 @@ jobs: if [ "$CIRCLE_BRANCH" == "master" ]; then DEPLOY_ENV=prod fi - + echo DEPLOY_ENV=${DEPLOY_ENV} + ls -lsa .. cp -rf ../scripts/config/${DEPLOY_ENV}.app.yaml ./app.yaml # process env_version.json SECRET_VERSION=`cat env_version.json | jq -r ".${DEPLOY_ENV}.version"` From 543bcfcd9a38e95bce02d86c84ecc5288cd9bf2f Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Fri, 4 Mar 2022 13:47:35 -0500 Subject: [PATCH 08/15] fix workspace/cwd --- circle.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/circle.yml b/circle.yml index ae75cd6..20bd747 100644 --- a/circle.yml +++ b/circle.yml @@ -29,7 +29,7 @@ orbs: jobs: build: - working_directory: ~/crypto-service/src + working_directory: ~/API-market/crypto-service/src parallelism: 1 shell: /bin/bash --login -e environment: @@ -56,19 +56,19 @@ jobs: # Unit Tests - run: npm test - persist_to_workspace: - root: ~/ + root: ~/API-market paths: - crypto-service deploy: - working_directory: ~/crypto-service/src + working_directory: ~/API-market/crypto-service/src parallelism: 1 shell: /bin/bash --login -e docker: - image: google/cloud-sdk:363.0.0 steps: - attach_workspace: - at: ~/crypto-service + at: ~/API-market - node/install: node-version: '12.12.0' - jq/install @@ -87,9 +87,8 @@ jobs: if [ "$CIRCLE_BRANCH" == "master" ]; then DEPLOY_ENV=prod fi - echo DEPLOY_ENV=${DEPLOY_ENV} - ls -lsa .. - cp -rf ../scripts/config/${DEPLOY_ENV}.app.yaml ./app.yaml + + cp -rf ../scripts/config/${DEPLOY_ENV}.app.yaml ./app # process env_version.json SECRET_VERSION=`cat env_version.json | jq -r ".${DEPLOY_ENV}.version"` ENV_VERSION=`cat env_version.json | jq -r '.envVersion'` From f1532d1e00ee31826843fff40814d3edcd8d5e56 Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Fri, 4 Mar 2022 13:51:15 -0500 Subject: [PATCH 09/15] fix missing file extension --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 20bd747..0a3ef02 100644 --- a/circle.yml +++ b/circle.yml @@ -88,7 +88,7 @@ jobs: DEPLOY_ENV=prod fi - cp -rf ../scripts/config/${DEPLOY_ENV}.app.yaml ./app + cp -rf ../scripts/config/${DEPLOY_ENV}.app.yaml ./app.yaml # process env_version.json SECRET_VERSION=`cat env_version.json | jq -r ".${DEPLOY_ENV}.version"` ENV_VERSION=`cat env_version.json | jq -r '.envVersion'` From 6e46bebc40ca10659f3baf8733bbfb10c4d50c1e Mon Sep 17 00:00:00 2001 From: Buddy Deck Date: Fri, 4 Mar 2022 14:38:33 -0500 Subject: [PATCH 10/15] correctly error out when schemas do not match; add dev to env_version --- circle.yml | 1 + src/env_version.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/circle.yml b/circle.yml index 0a3ef02..3f680d2 100644 --- a/circle.yml +++ b/circle.yml @@ -103,6 +103,7 @@ jobs: CHECK_ENV=`grep $ENV_VERSION .env` if [ "$?" == "1" ]; then echo ".env SCHEMA does not match: ${ENV_VERSION} not found in version: ${SECRET_VERSION}"; + exit 1; fi # create deploy_version file - run: diff --git a/src/env_version.json b/src/env_version.json index d1c3314..df94c65 100644 --- a/src/env_version.json +++ b/src/env_version.json @@ -1,5 +1,8 @@ { "envVersion": "2020-01-24_09:00", + "dev": { + "version": "1" + }, "staging": { "version": "1" }, From 678d83f560040e4beed2fcdc9a208861886d2d67 Mon Sep 17 00:00:00 2001 From: Buddy Deck <38056253+deckb@users.noreply.github.com> Date: Tue, 19 Apr 2022 10:19:08 -0400 Subject: [PATCH 11/15] update staging db; rename example.env -> .env.example --- src/{example.env => .env.example} | 2 +- src/env_version.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/{example.env => .env.example} (93%) diff --git a/src/example.env b/src/.env.example similarity index 93% rename from src/example.env rename to src/.env.example index 3a05bea..90772e6 100644 --- a/src/example.env +++ b/src/.env.example @@ -1,7 +1,7 @@ ### CRYPTO-SERVICE STAGING .ENV ### Last Change YYYY-MM-DD_HH:MM (in PST) -ENV_VERSION = "2021-01-01_09:00" +ENV_VERSION = "2020-01-24_09:00" PORT = 8080 ENVIRONMENT = "production" diff --git a/src/env_version.json b/src/env_version.json index df94c65..3767320 100644 --- a/src/env_version.json +++ b/src/env_version.json @@ -4,7 +4,7 @@ "version": "1" }, "staging": { - "version": "1" + "version": "2" }, "prod": { "version": "1" From a45895cb41ce6e1c60be0bf3ba691efb701bbfc8 Mon Sep 17 00:00:00 2001 From: Buddy Deck <38056253+deckb@users.noreply.github.com> Date: Tue, 19 Apr 2022 10:19:48 -0400 Subject: [PATCH 12/15] update docs to use .env.example --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cd53f32..3861ac0 100755 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ This app provides basic crypto and blockchain-related functions including: This service is mostly stateless - it holds no user data or keys - but it does hold developer (app) registrations in order to issue api keys ## Starting service -- Create .env file in project's root directory - copy example.env to .env and replace values -- Generate a public/private Key pair for server's use (see example.env BASE_PUBLIC_KEY, BASE_PRIVATE_KEY) +- Create .env file in project's root directory - copy .env.example to .env and replace values +- Generate a public/private Key pair for server's use (see .env.example BASE_PUBLIC_KEY, BASE_PRIVATE_KEY) - Important: Run Node 12.0 or higher on your local machine - ### `npm run dev` From 0d5a9e7e215b5f7f7c38655e874b3b050c8125f4 Mon Sep 17 00:00:00 2001 From: Warrick FitzGerald <40346010+WarrickFitz@users.noreply.github.com> Date: Tue, 11 Oct 2022 09:21:57 -0400 Subject: [PATCH 13/15] Apply outstanding snyk updates --- src/package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/package.json b/src/package.json index ad7cc3f..28fb739 100755 --- a/src/package.json +++ b/src/package.json @@ -17,20 +17,20 @@ "test": "true" }, "dependencies": { - "@open-rights-exchange/chainjs": "^2.3.2", + "@open-rights-exchange/chainjs": "^2.3.3", "analytics-node": "^3.5.0", - "axios": "^0.21.1", + "axios": "^0.25.0", "concurrently": "^4.1.2", "cors": "^2.8.4", - "dotenv": "^8.2.0", + "dotenv": "^8.6.0", "flatted": "^2.0.2", "helmet": "^4.4.1", "js-base64": "^3.6.0", "js-sha256": "^0.9.0", "lodash": "^4.17.21", "mongoose": "5.10.11", - "newrelic": "^7.1.3", - "rollbar": "^2.21.0", + "newrelic": "^7.5.2", + "rollbar": "^2.25.0", "ts-jest": "^26.5.3", "ts-node": "^8.10.1" }, From 1d6b71dc8d18cfc856993e9d1263df8689cbfce4 Mon Sep 17 00:00:00 2001 From: Dhruv Kumar Jha Date: Fri, 14 Oct 2022 22:34:45 +0530 Subject: [PATCH 14/15] Updated node version to install supported by GCloud --- src/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/package.json b/src/package.json index 28fb739..d16ad28 100755 --- a/src/package.json +++ b/src/package.json @@ -3,7 +3,7 @@ "version": "2.1.0", "private": false, "engines": { - "node": "12.x.x" + "node": "16.13.2" }, "main": "", "scripts": { From aa421985140f26ee96c46dd09fe31af719cff79d Mon Sep 17 00:00:00 2001 From: Tray Lewin Date: Sun, 16 Oct 2022 20:59:58 -0700 Subject: [PATCH 15/15] set npm >= 8.1.0 - to match 16.13.2 --- src/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/package.json b/src/package.json index d16ad28..31b89d5 100755 --- a/src/package.json +++ b/src/package.json @@ -3,6 +3,7 @@ "version": "2.1.0", "private": false, "engines": { + "npm": ">= 8.1.0", "node": "16.13.2" }, "main": "",