From bb349481dd5bcd5f75147fb7f3e0a9e14daa5158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 9 Feb 2026 14:24:16 +0100 Subject: [PATCH 1/4] Add script to migrate account_ids to the new hashing key Fixes: https://github.com/ooni/backend/issues/1080 --- scripts/fix-oonirun.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 scripts/fix-oonirun.py diff --git a/scripts/fix-oonirun.py b/scripts/fix-oonirun.py new file mode 100644 index 00000000..254fd096 --- /dev/null +++ b/scripts/fix-oonirun.py @@ -0,0 +1,39 @@ +import os +import csv +import hashlib + +ACCOUNT_ID_NEW = os.environ["HASH_PASS"] + +def hash_email_address(email_address: str, key: str) -> str: + em = email_address.encode() + return hashlib.blake2b(em, key=key.encode("utf-8"), digest_size=16).hexdigest() + + +values_list = [] +with open("dump.csv") as in_file: + csv_reader = csv.reader(in_file) + next(csv_reader) + for row in csv_reader: + if row == []: + continue + count, account_id, author = row + hashed_author = hash_email_address(author, "CHANGEME") + hashed_author_new = hash_email_address(author, ACCOUNT_ID_NEW) + # creator_account_id matches hash of old key, requires update + if hashed_author == account_id: + values_list.append(f"\n('{account_id}', '{hashed_author_new}')") + print("WILL UPDATE") + elif hashed_author_new == account_id: + print("NEW-OK NO UPDATE NEEDED") + else: + print(f"BAD: {account_id} != {hashed_author} ({author}) - update not possible") + +sql_query_final = f""" +UPDATE oonirun as t set + creator_account_id = c.new_account_id +FROM (values + {",".join(values_list)} +) as c(creator_account_id, new_account_id) +WHERE c.creator_account_id = t.creator_account_id +""" +print(sql_query_final) From 1c98d01b3366a71097298de8aeded65b604fdd3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 9 Feb 2026 17:23:20 +0100 Subject: [PATCH 2/4] Increase the allow cidr_block of pg --- scripts/fix-oonirun.py | 3 ++- tf/environments/dev/main.tf | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/fix-oonirun.py b/scripts/fix-oonirun.py index 254fd096..30bfa0ae 100644 --- a/scripts/fix-oonirun.py +++ b/scripts/fix-oonirun.py @@ -1,5 +1,6 @@ import os import csv +import sys import hashlib ACCOUNT_ID_NEW = os.environ["HASH_PASS"] @@ -10,7 +11,7 @@ def hash_email_address(email_address: str, key: str) -> str: values_list = [] -with open("dump.csv") as in_file: +with open(sys.argv[1]) as in_file: csv_reader = csv.reader(in_file) next(csv_reader) for row in csv_reader: diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 997ef36b..1da4f0a1 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -158,7 +158,9 @@ module "oonipg" { db_allocated_storage = "5" db_max_allocated_storage = null - allow_cidr_blocks = module.network.vpc_subnet_private[*].cidr_block + allow_cidr_blocks = [ + "10.0.0.0/8" + ] allow_security_groups = [module.ooni_jumphost.ec2_sg_id] tags = merge( From b955769b1c9219bf13b4ce4313414be80386ba87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 9 Feb 2026 19:35:34 +0100 Subject: [PATCH 3/4] Add support for updating also those that don't match * Allow passing in old hash key --- scripts/fix-oonirun.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/fix-oonirun.py b/scripts/fix-oonirun.py index 30bfa0ae..71fecae6 100644 --- a/scripts/fix-oonirun.py +++ b/scripts/fix-oonirun.py @@ -4,6 +4,7 @@ import hashlib ACCOUNT_ID_NEW = os.environ["HASH_PASS"] +OLD_KEY = os.environ.get("HASH_PASS_OLD", "CHANGEME") def hash_email_address(email_address: str, key: str) -> str: em = email_address.encode() @@ -18,7 +19,7 @@ def hash_email_address(email_address: str, key: str) -> str: if row == []: continue count, account_id, author = row - hashed_author = hash_email_address(author, "CHANGEME") + hashed_author = hash_email_address(author, OLD_KEY) hashed_author_new = hash_email_address(author, ACCOUNT_ID_NEW) # creator_account_id matches hash of old key, requires update if hashed_author == account_id: @@ -27,7 +28,8 @@ def hash_email_address(email_address: str, key: str) -> str: elif hashed_author_new == account_id: print("NEW-OK NO UPDATE NEEDED") else: - print(f"BAD: {account_id} != {hashed_author} ({author}) - update not possible") + print(f"BAD: {account_id} != {hashed_author} ({author}) - will update with new") + values_list.append(f"\n('{account_id}', '{hashed_author_new}')") sql_query_final = f""" UPDATE oonirun as t set From ce444579928a62910a567b965d2b9c329741c002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 10 Feb 2026 12:34:08 +0100 Subject: [PATCH 4/4] Add docstring for usage on the script --- ...x-oonirun.py => fix-creator_account_id.py} | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) rename scripts/{fix-oonirun.py => fix-creator_account_id.py} (55%) diff --git a/scripts/fix-oonirun.py b/scripts/fix-creator_account_id.py similarity index 55% rename from scripts/fix-oonirun.py rename to scripts/fix-creator_account_id.py index 71fecae6..53df6a34 100644 --- a/scripts/fix-oonirun.py +++ b/scripts/fix-creator_account_id.py @@ -1,3 +1,24 @@ +""" +This script is used to migrate account_id_hashes based on a dump from the database. + +It verifies if the old hashes correspond to the OLD_KEY and forms a SQL UPDATE +query to be run on the database to update them to HASH_PASS. + +To create a csv dump for this script, login to the postgresql instance using psql +and run: + +\copy (SELECT COUNT(*) as count, creator_account_id,author FROM oonirun GROUP BY creator_account_id,author) TO 'dump-oonirun-prod.csv' csv header; + +\copy (SELECT COUNT(*) as count, creator_account_id,email_address FROM oonifinding GROUP BY creator_account_id,email_address) TO 'dump-oonifinding-prod.csv' csv header; + +Then copy to dump.csv over and run: + +HASH_PASS_OLD=XXXX HASH_PASS=YYYY python fix-creator_account_id.py dump-oonirun-prod.csv oonirun + +HASH_PASS_OLD=XXXX HASH_PASS=YYYY python fix-creator_account_id.py dump-oonifinding-prod.csv oonifinding + +Then copy paste the SQL update script at the end into the DB to perform the update. +""" import os import csv import sys @@ -6,13 +27,16 @@ ACCOUNT_ID_NEW = os.environ["HASH_PASS"] OLD_KEY = os.environ.get("HASH_PASS_OLD", "CHANGEME") +CSV_FN = sys.argv[1] +TABLE_NAME = sys.argv[2] + def hash_email_address(email_address: str, key: str) -> str: em = email_address.encode() return hashlib.blake2b(em, key=key.encode("utf-8"), digest_size=16).hexdigest() values_list = [] -with open(sys.argv[1]) as in_file: +with open(CSV_FN) as in_file: csv_reader = csv.reader(in_file) next(csv_reader) for row in csv_reader: @@ -32,7 +56,7 @@ def hash_email_address(email_address: str, key: str) -> str: values_list.append(f"\n('{account_id}', '{hashed_author_new}')") sql_query_final = f""" -UPDATE oonirun as t set +UPDATE {TABLE_NAME} as t set creator_account_id = c.new_account_id FROM (values {",".join(values_list)}