From 316041209d13ed781a9fa63e835975d37ee6d195 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Tue, 9 Dec 2025 19:45:47 +0100 Subject: [PATCH 01/25] Checkout merge fixes --- hasura/metadata/actions.graphql | 14 +- hasura/metadata/actions.yaml | 14 + .../src/components/ElectoralLogList.tsx | 2 +- .../src/queries/ListElectoralLog.ts | 9 +- .../src/queries/customBuildQuery.ts | 11 +- packages/electoral-log/Cargo.toml | 1 + .../electoral-log/src/client/board_client.rs | 514 ++++++++--------- packages/electoral-log/src/client/mod.rs | 1 + packages/electoral-log/src/client/types.rs | 481 ++++++++++++++++ .../electoral-log/src/messages/message.rs | 5 +- .../electoral-log/src/messages/statement.rs | 4 +- packages/harvest/src/routes/electoral_log.rs | 101 +++- .../harvest/src/routes/immudb_log_audit.rs | 5 +- .../harvest/src/routes/voter_electoral_log.rs | 10 +- packages/harvest/src/types/resources.rs | 9 - .../src/commands/create_electoral_logs.rs | 2 +- .../src/commands/export_cast_votes.rs | 7 +- .../src/routes/BallotLocator.tsx | 1 - .../windmill/external-bin/generate_logs.rs | 17 +- .../windmill/src/services/electoral_log.rs | 541 ++++++------------ packages/windmill/src/services/event_list.rs | 6 +- .../windmill/src/services/insert_cast_vote.rs | 2 +- .../src/services/reports/activity_log.rs | 374 +++++------- .../src/services/reports/template_renderer.rs | 25 +- .../src/tasks/activity_logs_report.rs | 3 +- packages/windmill/src/tasks/electoral_log.rs | 4 +- .../windmill/src/tasks/generate_report.rs | 1 + packages/windmill/src/types/resources.rs | 22 +- 28 files changed, 1241 insertions(+), 945 deletions(-) create mode 100644 packages/electoral-log/src/client/types.rs mode change 100644 => 100755 packages/windmill/external-bin/generate_logs.rs diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index 6b75d67a081..d593c537163 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -428,6 +428,18 @@ type Mutation { ): LimitAccessByCountriesOutput } +type Query { + list_cast_vote_messages( + tenant_id: String! + election_event_id: String! + election_id: String + ballot_id: String! + limit: Int + offset: Int + order_by: ElectoralLogOrderBy + ): ListCastVoteMessagesOutput +} + type Query { listElectoralLog( limit: Int @@ -687,10 +699,10 @@ input PgAuditOrderBy { input ElectoralLogFilter { id: String user_id: String + username: String created: String statement_timestamp: String statement_kind: String - username: String } input ElectoralLogOrderBy { diff --git a/hasura/metadata/actions.yaml b/hasura/metadata/actions.yaml index 1c5b70adfa3..7333b78d445 100644 --- a/hasura/metadata/actions.yaml +++ b/hasura/metadata/actions.yaml @@ -935,6 +935,20 @@ actions: permissions: - role: cloudflare-write - role: admin-user + - name: list_cast_vote_messages + definition: + kind: synchronous + handler: http://{{HARVEST_DOMAIN}}/voting-portal/immudb/list-cast-vote-messages + forward_client_headers: true + request_transform: + body: + action: transform + template: "{{$body.input}}" + template_engine: Kriti + version: 2 + permissions: + - role: user + comment: List electoral log entries of statement_kind CastVote - name: listElectoralLog definition: kind: "" diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index c5697fbb446..281dac597aa 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -283,7 +283,7 @@ export const ElectoralLogList: React.FC = ({ getHeadField(record, "event_type")} /> { return { diff --git a/packages/admin-portal/src/queries/customBuildQuery.ts b/packages/admin-portal/src/queries/customBuildQuery.ts index ea57e6e7702..33912e299fe 100644 --- a/packages/admin-portal/src/queries/customBuildQuery.ts +++ b/packages/admin-portal/src/queries/customBuildQuery.ts @@ -82,11 +82,16 @@ export const customBuildQuery = name: resourceName, }, } + const builtVariables = buildVariables(introspectionResults)( + resource, + raFetchType, + params, + null + ) + const finalVariables = getElectoralLogVariables(builtVariables) return { query: getElectoralLog(params), - variables: getElectoralLogVariables( - buildVariables(introspectionResults)(resource, raFetchType, params, null) - ), + variables: finalVariables, parseResponse: (res: any) => { const response = res.data.listElectoralLog let output = { diff --git a/packages/electoral-log/Cargo.toml b/packages/electoral-log/Cargo.toml index 322bb12a1a4..d5442ff4362 100644 --- a/packages/electoral-log/Cargo.toml +++ b/packages/electoral-log/Cargo.toml @@ -29,6 +29,7 @@ tracing-log = "0.2" tracing-attributes = "0.1" tracing-subscriber = "0.3" tracing-tree = "0.4" +chrono = "0.4.41" [dev-dependencies] serial_test = "3.2" diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index 148f52e1f95..1f04fbc6196 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -2,15 +2,17 @@ // // SPDX-License-Identifier: AGPL-3.0-only -use crate::assign_value; +use crate::client::types::*; use anyhow::{anyhow, Context, Result}; +use chrono::format; use immudb_rs::{sql_value::Value, Client, CommittedSqlTx, NamedParam, Row, SqlValue, TxMode}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Display; -use strum_macros::Display; +use std::time::Duration; +use std::time::Instant; use tokio_stream::StreamExt; // Added for streaming use tracing::{error, info, instrument, warn}; @@ -20,200 +22,28 @@ const IMMUDB_DEFAULT_OFFSET: usize = 0; const ELECTORAL_LOG_TABLE: &'static str = "electoral_log_messages"; /// 36 chars + EOL + some padding const ID_VARCHAR_LENGTH: usize = 40; +const ID_KEY_VARCHAR_LENGTH: usize = 4; /// Longest possible statement kind must be < 40 const STATEMENT_KIND_VARCHAR_LENGTH: usize = 40; /// 64 chars + EOL + some padding const BALLOT_ID_VARCHAR_LENGTH: usize = 70; +/// This is the order of the cols in the where clauses, as defined in ElectoralLogVarCharColumn: +/// StatementKind, AreaId, ElectionId, UserId, BallotId, statement_timestamp. +/// +/// Other columns that have no length constraint are not indexable. +/// 'create' is not indexed, we use statement_timestamp intead. +pub const MULTI_COLUMN_INDEXES: [&'static str; 3] = [ + "(statement_kind, election_id, user_id_key, user_id, ballot_id)", // COUNT or SELECT cast_vote_messages and filter by ballot_id + "(statement_kind, user_id_key, user_id)", // Filters in Admin portal LOGS tab. + "(user_id_key, user_id)", // Filters in Admin portal LOGS tab and for the User´s logs. +]; + #[derive(Debug)] pub struct BoardClient { client: Client, } -#[derive(Debug, Clone, Display, PartialEq, Eq, Ord, PartialOrd)] -#[strum(serialize_all = "snake_case")] -pub enum ElectoralLogVarCharColumn { - StatementKind, - UserId, - BallotId, - Username, - SenderPk, - ElectionId, - AreaId, - Version, -} - -/// SQL comparison operators supported by immudb. -/// ILIKE is not supported. -#[derive(Display, Debug, Clone)] -pub enum SqlCompOperators { - #[strum(to_string = "=")] - Equal, - #[strum(to_string = "!=")] - NotEqual, - #[strum(to_string = ">")] - GreaterThan, - #[strum(to_string = "<")] - LessThan, - #[strum(to_string = ">=")] - GreaterThanOrEqual, - #[strum(to_string = "<=")] - LessThanOrEqual, - #[strum(to_string = "LIKE")] - Like, - #[strum(to_string = "IN")] - In, - #[strum(to_string = "NOT IN")] - NotIn, -} - -pub type WhereClauseBTreeMap = BTreeMap; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct ElectoralLogMessage { - pub id: i64, - pub created: i64, - pub sender_pk: String, - pub statement_timestamp: i64, - pub statement_kind: String, - pub message: Vec, - pub version: String, - pub user_id: Option, - pub username: Option, - pub election_id: Option, - pub area_id: Option, - pub ballot_id: Option, -} - -impl TryFrom<&Row> for ElectoralLogMessage { - type Error = anyhow::Error; - - fn try_from(row: &Row) -> Result { - let mut id = 0; - let mut created = 0; - let mut sender_pk = String::from(""); - let mut statement_timestamp = 0; - let mut statement_kind = String::from(""); - let mut message = vec![]; - let mut version = String::from(""); - let mut user_id: Option = None; - let mut username: Option = None; - let mut election_id: Option = None; - let mut area_id: Option = None; - let mut ballot_id: Option = None; - - for (column, value) in row.columns.iter().zip(row.values.iter()) { - // FIXME for some reason columns names appear with parentheses - let dot = column - .find('.') - .ok_or(anyhow!("invalid column found '{}'", column.as_str()))?; - let bare_column = &column[dot + 1..column.len() - 1]; - - match bare_column { - "id" => assign_value!(Value::N, value, id), - "created" => assign_value!(Value::Ts, value, created), - "sender_pk" => assign_value!(Value::S, value, sender_pk), - "statement_timestamp" => { - assign_value!(Value::Ts, value, statement_timestamp) - } - "statement_kind" => assign_value!(Value::S, value, statement_kind), - "message" => assign_value!(Value::Bs, value, message), - "version" => assign_value!(Value::S, value, version), - "user_id" => match value.value.as_ref() { - Some(Value::S(inner)) => user_id = Some(inner.clone()), - Some(Value::Null(_)) => user_id = None, - None => user_id = None, - _ => { - return Err(anyhow!( - "invalid column value for 'user_id': {:?}", - value.value.as_ref() - )) - } - }, - "username" => match value.value.as_ref() { - Some(Value::S(inner)) => username = Some(inner.clone()), - Some(Value::Null(_)) => username = None, - None => username = None, - _ => { - return Err(anyhow!( - "invalid column value for 'username': {:?}", - value.value.as_ref() - )) - } - }, - "election_id" => match value.value.as_ref() { - Some(Value::S(inner)) => election_id = Some(inner.clone()), - Some(Value::Null(_)) => election_id = None, - None => election_id = None, - _ => { - return Err(anyhow!( - "invalid column value for 'election_id': {:?}", - value.value.as_ref() - )) - } - }, - "area_id" => match value.value.as_ref() { - Some(Value::S(inner)) => area_id = Some(inner.clone()), - Some(Value::Null(_)) => area_id = None, - None => area_id = None, - _ => { - return Err(anyhow!( - "invalid column value for 'area_id': {:?}", - value.value.as_ref() - )) - } - }, - "ballot_id" => match value.value.as_ref() { - Some(Value::S(inner)) => ballot_id = Some(inner.clone()), - Some(Value::Null(_)) => ballot_id = None, - None => ballot_id = None, - _ => { - return Err(anyhow!( - "invalid column value for 'ballod_id': {:?}", - value.value.as_ref() - )) - } - }, - _ => return Err(anyhow!("invalid column found '{}'", bare_column)), - } - } - - Ok(ElectoralLogMessage { - id, - created, - sender_pk, - statement_timestamp, - statement_kind, - message, - version, - user_id, - username, - election_id, - area_id, - ballot_id, - }) - } -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct Aggregate { - pub count: i64, -} - -impl TryFrom<&Row> for Aggregate { - type Error = anyhow::Error; - - fn try_from(row: &Row) -> Result { - let mut count = 0; - - for (column, value) in row.columns.iter().zip(row.values.iter()) { - match column.as_str() { - _ => assign_value!(Value::N, value, count), - } - } - Ok(Aggregate { count }) - } -} - impl BoardClient { #[instrument(skip(password), level = "trace")] pub async fn new(server_url: &str, username: &str, password: &str) -> Result { @@ -307,16 +137,21 @@ impl BoardClient { /// columns_matcher represents the columns that will be used to filter the messages, /// The order as defined ElectoralLogVarCharColumn is important for preformance to match the indexes. /// BTreeMap ensures the order is preserved no matter the insertion sequence. - pub async fn get_electoral_log_messages_filtered( + #[instrument(skip_all, err)] + pub async fn get_electoral_log_messages_filtered( &mut self, board_db: &str, - columns_matcher: Option, + columns_matcher: Option, min_ts: Option, max_ts: Option, limit: Option, offset: Option, order_by: Option>, - ) -> Result> { + ) -> Result> + where + K: Debug + Display, + V: Debug + Display, + { self.get_filtered( board_db, columns_matcher, @@ -330,42 +165,56 @@ impl BoardClient { } #[instrument(skip_all, err)] - async fn get_filtered( + pub async fn get_electoral_log_messages_batch( + &mut self, + board_db: &str, + limit: i64, + offset: i64, + ) -> Result> { + self.get_filtered::( + board_db, + None, + None, + None, + Some(limit), + Some(offset), + None, + ) + .await + } + + #[instrument(skip(self, board_db, order_by), err)] + async fn get_filtered( &mut self, board_db: &str, - columns_matcher: Option, + columns_matcher: Option, min_ts: Option, max_ts: Option, limit: Option, offset: Option, order_by: Option>, - ) -> Result> { + ) -> Result> + where + K: Debug + Display, + V: Debug + Display, + { + let start = Instant::now(); let (min_clause, min_clause_value) = if let Some(min_ts) = min_ts { - ("AND created >= @min_ts", min_ts) + ("AND statement_timestamp >= @min_ts", min_ts) } else { ("", 0) }; let (max_clause, max_clause_value) = if let Some(max_ts) = max_ts { - ("AND created <= @max_ts", max_ts) + ("AND statement_timestamp <= @max_ts", max_ts) } else { ("", 0) }; - let mut params = vec![]; - let mut where_clause = String::from("statement_kind IS NOT NULL "); - if let Some(columns_matcher) = &columns_matcher { - for (key, (op, value)) in columns_matcher { - where_clause.push_str(&format!("AND {key} {op} @{key} ")); - params.push(NamedParam { - name: key.to_string(), - value: Some(SqlValue { - value: Some(Value::S(value.to_owned())), - }), - }) - } - } - + let (where_clause, mut params) = columns_matcher + .clone() + .unwrap_or_default() + .to_where_clause(); let order_by_clauses = if let Some(order_by) = order_by { order_by .iter() @@ -376,32 +225,6 @@ impl BoardClient { format!("ORDER BY id desc") }; - self.client.use_database(board_db).await?; - let sql = format!( - r#" - SELECT - id, - username, - user_id, - area_id, - election_id, - ballot_id, - created, - sender_pk, - statement_timestamp, - statement_kind, - message, - version - FROM {ELECTORAL_LOG_TABLE} - WHERE {where_clause} - {min_clause} - {max_clause} - {order_by_clauses} - LIMIT @limit - OFFSET @offset; - "# - ); - if min_clause_value != 0 { params.push(NamedParam { name: String::from("min_ts"), @@ -433,6 +256,46 @@ impl BoardClient { }), }); + let where_clauses = + if !where_clause.is_empty() || !min_clause.is_empty() || !max_clause.is_empty() { + format!( + r#" + WHERE {where_clause} + {min_clause} + {max_clause} + "# + ) + } else { + String::from("") + }; + + let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause(); + + self.client.use_database(board_db).await?; + let sql = format!( + r#" + SELECT + id, + username, + user_id, + area_id, + election_id, + ballot_id, + created, + sender_pk, + statement_timestamp, + statement_kind, + message, + version + FROM {ELECTORAL_LOG_TABLE} + {use_index_clause} + {where_clauses} + {order_by_clauses} + LIMIT @limit + OFFSET @offset; + "# + ); + info!("SQL query: {}", sql); let response_stream = self.client.streaming_sql_query(&sql, params) .await @@ -470,51 +333,100 @@ impl BoardClient { } } } - + let duration = start.elapsed(); + info!( + "Processed {} rows from stream in {}ms", + total_rows_fetched, + duration.as_millis() + ); Ok(messages) } - #[instrument(err)] + #[instrument(skip(self, board_db), err)] pub async fn count_electoral_log_messages( &mut self, board_db: &str, - columns_matcher: Option, + columns_matcher: Option, ) -> Result { - let mut params = vec![]; - let mut where_clause = String::from("statement_kind IS NOT NULL "); - if let Some(columns_matcher) = &columns_matcher { - for (key, (op, value)) in columns_matcher { - where_clause.push_str(&format!("AND {key} {op} @{key} ")); - params.push(NamedParam { - name: key.to_string(), - value: Some(SqlValue { - value: Some(Value::S(value.to_owned())), - }), - }) - } - } - + let start = Instant::now(); + let (where_clause, params) = columns_matcher + .clone() + .unwrap_or_default() + .to_where_clause(); + let where_clauses = if !where_clause.is_empty() { + format!( + r#" + WHERE {where_clause} + "# + ) + } else { + String::from("") + }; + let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause(); self.client.use_database(board_db).await?; - let sql = format!( - r#" - SELECT COUNT(*) - FROM {ELECTORAL_LOG_TABLE} - WHERE {where_clause} - "#, - ); - info!("SQL query: {}", sql); - let sql_query_response = self.client.sql_query(&sql, params).await?; - let mut rows_iter = sql_query_response - .get_ref() - .rows - .iter() - .map(Aggregate::try_from); - let aggregate = rows_iter - .next() - .ok_or_else(|| anyhow!("No aggregate found"))??; + let count = if use_index_clause.is_empty() && where_clauses.is_empty() && params.is_empty() + { + // if there are no constraints, just get the last id as the count to avoid a full scan of the table. + let sql = format!( + r#" + SELECT + id, + username, + user_id, + area_id, + election_id, + ballot_id, + created, + sender_pk, + statement_timestamp, + statement_kind, + message, + version + FROM {ELECTORAL_LOG_TABLE} + ORDER BY id desc + LIMIT 1 + OFFSET 0; + "# + ); + info!("SQL query: {}", sql); + let sql_query_response = self.client.sql_query(&sql, vec![]).await?; + let elog_msg = sql_query_response + .get_ref() + .rows + .iter() + .map(ElectoralLogMessage::try_from) + .next(); + match elog_msg { + Some(elog_msg) => elog_msg?.id, + None => 0, + } + } else { + let sql = format!( + r#" + SELECT COUNT(*) + FROM {ELECTORAL_LOG_TABLE} + {use_index_clause} + {where_clauses} + "#, + ); - Ok(aggregate.count as i64) + info!("SQL query: {}", sql); + let sql_query_response = self.client.sql_query(&sql, params).await?; + let mut rows_iter = sql_query_response + .get_ref() + .rows + .iter() + .map(Aggregate::try_from); + let aggregate = rows_iter + .next() + .ok_or_else(|| anyhow!("No aggregate found"))??; + aggregate.count + }; + + let duration = start.elapsed(); + info!("COUNT query took {}ms", duration.as_millis()); + Ok(count as i64) } pub async fn open_session(&mut self, database_name: &str) -> Result<()> { @@ -533,7 +445,8 @@ impl BoardClient { self.client.commit(transaction_id).await } - // Insert messages in batch using an existing session/transaction + /// Insert messages in batch using an existing session/transaction + #[instrument(skip(self, messages), err)] pub async fn insert_electoral_log_messages_batch( &mut self, transaction_id: &String, @@ -551,6 +464,7 @@ impl BoardClient { statement_timestamp, message, version, + user_id_key, user_id, username, election_id, @@ -563,6 +477,7 @@ impl BoardClient { @statement_timestamp, @message, @version, + @user_id_key, @user_id, @username, @election_id, @@ -609,6 +524,15 @@ impl BoardClient { value: Some(Value::S(message.version.clone())), }), }, + NamedParam { + name: String::from("user_id_key"), + value: Some(SqlValue { + value: match message.user_id.clone() { + Some(user_id) => Some(Value::S(user_id.chars().take(3).collect())), + None => None, + }, + }), + }, NamedParam { name: String::from("user_id"), value: Some(SqlValue { @@ -690,6 +614,7 @@ impl BoardClient { statement_timestamp, message, version, + user_id_key, user_id, username, election_id, @@ -702,6 +627,7 @@ impl BoardClient { @statement_timestamp, @message, @version, + @user_id_key, @user_id, @username, @election_id, @@ -748,6 +674,15 @@ impl BoardClient { value: Some(Value::S(message.version.clone())), }), }, + NamedParam { + name: String::from("user_id_key"), + value: Some(SqlValue { + value: match message.user_id.clone() { + Some(user_id) => Some(Value::S(user_id.chars().take(3).collect())), + None => None, + }, + }), + }, NamedParam { name: String::from("user_id"), value: Some(SqlValue { @@ -828,36 +763,36 @@ impl BoardClient { pub async fn upsert_electoral_log_db(&mut self, board_dbname: &str) -> Result<()> { let sql = format!( r#" + CREATE TABLE IF NOT EXISTS {ELECTORAL_LOG_TABLE} ( CREATE TABLE IF NOT EXISTS {ELECTORAL_LOG_TABLE} ( id INTEGER AUTO_INCREMENT, created TIMESTAMP, sender_pk VARCHAR, statement_timestamp TIMESTAMP, statement_kind VARCHAR[{STATEMENT_KIND_VARCHAR_LENGTH}], + statement_kind VARCHAR[{STATEMENT_KIND_VARCHAR_LENGTH}], message BLOB, version VARCHAR, + user_id_key VARCHAR[{ID_KEY_VARCHAR_LENGTH}], user_id VARCHAR[{ID_VARCHAR_LENGTH}], username VARCHAR, election_id VARCHAR[{ID_VARCHAR_LENGTH}], area_id VARCHAR[{ID_VARCHAR_LENGTH}], ballot_id VARCHAR[{BALLOT_ID_VARCHAR_LENGTH}], + election_id VARCHAR[{ID_VARCHAR_LENGTH}], + area_id VARCHAR[{ID_VARCHAR_LENGTH}], + ballot_id VARCHAR[{BALLOT_ID_VARCHAR_LENGTH}], PRIMARY KEY id ); "# ); - // This is the order of the cols in the where clauses, as defined in ElectoralLogVarCharColumn - // Note Username cannot be indexed because it is not constrained to 512B, but is not needded since we have user_id - // StatementKind, UserId, BallotId, Username, SenderPk, ElectionId, AreaId, Version, - let elog_indexes = vec![ - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (statement_kind, user_id, ballot_id, election_id, id)"), // To list or count cast_vote_messages and Order by id - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (statement_kind, user_id, ballot_id, election_id, statement_timestamp)"), // Order by statement_timestamp - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (statement_kind, election_id, id)"), // Order by id - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (statement_kind, election_id, statement_timestamp)"), // Order by statement_timestamp - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (user_id, election_id, area_id, id)"), // Other posible filters... - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (election_id, area_id, id)"), - format!("CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} (area_id, id)"), - ]; + let mut elog_indexes = vec![]; + for mult_col_idx in MULTI_COLUMN_INDEXES { + elog_indexes.push(format!( + "CREATE INDEX IF NOT EXISTS ON {ELECTORAL_LOG_TABLE} {mult_col_idx}" + )); + } self.upsert_database(board_dbname, &sql, elog_indexes.as_slice()) .await @@ -874,6 +809,13 @@ impl BoardClient { /// Creates the requested immudb database, only if it doesn't exist. It also creates /// the requested tables and indexes if they don't exist. + async fn upsert_database( + &mut self, + database_name: &str, + tables: &str, + indexes: &[String], + ) -> Result<()> { + /// the requested tables and indexes if they don't exist. async fn upsert_database( &mut self, database_name: &str, @@ -885,11 +827,13 @@ impl BoardClient { println!("Database not found, creating.."); self.client.create_database(database_name).await?; info!("Database created!"); + info!("Database created!"); }; self.client.use_database(database_name).await?; // List tables and create them if missing if !self.client.has_tables().await? { + info!("no tables! let's create them"); info!("no tables! let's create them"); self.client.sql_exec(&tables, vec![]).await?; } @@ -897,6 +841,10 @@ impl BoardClient { info!("Inserting index..."); self.client.sql_exec(index, vec![]).await?; } + for index in indexes { + info!("Inserting index..."); + self.client.sql_exec(index, vec![]).await?; + } Ok(()) } } @@ -954,7 +902,7 @@ pub(crate) mod tests { let ret = b.get_electoral_log_messages(BOARD_DB).await.unwrap(); assert_eq!(messages, ret); - let cols_match = BTreeMap::from([ + let cols_match = WhereClauseOrdMap::from(&[ ( ElectoralLogVarCharColumn::StatementKind, (SqlCompOperators::Equal, "".to_string()), diff --git a/packages/electoral-log/src/client/mod.rs b/packages/electoral-log/src/client/mod.rs index c13707ec34d..52900761caa 100644 --- a/packages/electoral-log/src/client/mod.rs +++ b/packages/electoral-log/src/client/mod.rs @@ -2,3 +2,4 @@ // // SPDX-License-Identifier: AGPL-3.0-only pub mod board_client; +pub mod types; diff --git a/packages/electoral-log/src/client/types.rs b/packages/electoral-log/src/client/types.rs new file mode 100644 index 00000000000..8ff7c9c0c27 --- /dev/null +++ b/packages/electoral-log/src/client/types.rs @@ -0,0 +1,481 @@ +// SPDX-FileCopyrightText: 2024 Sequent Tech +// +// SPDX-License-Identifier: AGPL-3.0-only + +use crate::assign_value; +use crate::client::board_client::MULTI_COLUMN_INDEXES; +use crate::messages::statement::{StatementBody, StatementType}; +use anyhow::{anyhow, Result}; +use chrono::{DateTime, Utc}; +use immudb_rs::{sql_value::Value, NamedParam, Row, SqlValue}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::fmt::Debug; +use std::str::FromStr; +use strum_macros::{Display, EnumString}; +use tracing::{error, info, instrument, warn}; + +#[derive(Debug, Clone, Display, PartialEq, Eq, Ord, PartialOrd, EnumString)] +#[strum(serialize_all = "snake_case")] +pub enum ElectoralLogVarCharColumn { + StatementKind, + AreaId, + ElectionId, + UserIdKey, + UserId, + BallotId, + Username, + SenderPk, + Version, +} + +/// SQL comparison operators supported by immudb. +/// ILIKE is not supported. +#[derive(Display, Debug, Clone)] +pub enum SqlCompOperators { + #[strum(to_string = "=")] + Equal(String), + #[strum(to_string = "!=")] + NotEqual(String), + #[strum(to_string = ">")] + GreaterThan(String), + #[strum(to_string = "<")] + LessThan(String), + #[strum(to_string = ">=")] + GreaterThanOrEqual(String), + #[strum(to_string = "<=")] + LessThanOrEqual(String), + #[strum(to_string = "LIKE")] + Like(String), + #[strum(to_string = "IN")] + In(Vec), + #[strum(to_string = "NOT IN")] + NotIn(Vec), +} + +/// Each column in the map is unique but it can have several filters associated with it. +/// The type will keep the order of the columns to match the multicolumn indexes. +#[derive(Debug, Clone, Default)] +pub struct WhereClauseOrdMap(BTreeMap>); + +impl WhereClauseOrdMap { + pub fn new() -> Self { + Self(BTreeMap::new()) + } + + pub fn from(tuples: &[(ElectoralLogVarCharColumn, SqlCompOperators)]) -> Self { + let mut map = WhereClauseOrdMap::new(); + for (key, value) in tuples { + map.insert(key.clone(), value.clone()); + } + map + } + + /// If the column already exists, the comparisson will be added to the existing ones. + /// Otherwise it will create the first one. + pub fn insert(&mut self, key: ElectoralLogVarCharColumn, value: SqlCompOperators) { + self.0 + .entry(key) + .and_modify(|vec| vec.push(value.clone())) + .or_insert(vec![value]); + } + + pub fn iter( + &self, + ) -> std::collections::btree_map::Iter> { + self.0.iter() + } + + /// If the first element of the map (first column in the where clause) has an index, it will be used. + /// This will match the longest possible index. + pub fn to_use_index_clause(&self) -> String { + let mut try_index_clause = String::from(""); + let mut last_index_clause_match = String::from(""); + for (col_name, _) in self.iter() { + if try_index_clause.is_empty() { + try_index_clause.push_str(&format!("({col_name}")); // For the contains() is important to mark with '(' the beginning of the index. + } else { + try_index_clause.push_str(&format!(", {col_name}")); + } + for index in MULTI_COLUMN_INDEXES { + if index.contains(&try_index_clause.as_str()) { + last_index_clause_match = format!("USE INDEX ON {index}"); + } + } + } + last_index_clause_match + } + + pub fn to_where_clause(&self) -> (String, Vec) { + let mut params = vec![]; + let mut where_clause = String::from(""); + for (col_name, comparissons) in self.iter() { + for (i, op) in comparissons.iter().enumerate() { + match op { + SqlCompOperators::In(values_vec) | SqlCompOperators::NotIn(values_vec) => { + let placeholders: Vec = values_vec + .iter() + .enumerate() + .map(|(j, _)| format!("@param_{col_name}{i}{j}")) + .collect(); + for (j, value) in values_vec.into_iter().enumerate() { + params.push(NamedParam { + name: format!("param_{col_name}{i}{j}"), + value: Some(SqlValue { + value: Some(Value::S(value.to_owned())), + }), + }); + } + if where_clause.is_empty() { + where_clause.push_str(&format!( + "{col_name} {op} ({})", + placeholders.join(", ") + )); + } else { + where_clause.push_str(&format!( + "AND {col_name} {op} ({})", + placeholders.join(", ") + )); + } + } + SqlCompOperators::Equal(value) + | SqlCompOperators::NotEqual(value) + | SqlCompOperators::GreaterThan(value) + | SqlCompOperators::LessThan(value) + | SqlCompOperators::GreaterThanOrEqual(value) + | SqlCompOperators::LessThanOrEqual(value) + | SqlCompOperators::Like(value) => { + if where_clause.is_empty() { + where_clause + .push_str(&format!("{col_name} {op} @param_{col_name}{i} ")); + } else { + where_clause + .push_str(&format!("AND {col_name} {op} @param_{col_name}{i} ")); + } + params.push(NamedParam { + name: format!("param_{col_name}{i}"), + value: Some(SqlValue { + value: Some(Value::S(value.to_owned())), + }), + }); + } + } + } + } + (where_clause, params) + } +} + +// Enumeration for the valid fields in the immudb table +#[derive(Debug, Deserialize, Hash, PartialEq, Eq, EnumString, Display, Clone)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum OrderField { + Id, + Created, + StatementTimestamp, + StatementKind, + Message, + UserId, + Username, + BallotId, + SenderPk, + LogType, + EventType, + Description, + Version, +} + +// Enumeration for the valid order directions +#[derive(Debug, Deserialize, EnumString, Display, Clone)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum OrderDirection { + Asc, + Desc, +} + +#[derive(Deserialize, Debug, Default, Clone)] +pub struct GetElectoralLogBody { + pub tenant_id: String, + pub election_event_id: String, + pub limit: Option, + pub offset: Option, + pub filter: Option>, + pub order_by: Option>, + pub election_id: Option, + pub area_ids: Option>, + pub only_with_user: Option, + pub statement_kind: Option, +} + +impl GetElectoralLogBody { + #[instrument(skip_all)] + pub fn get_min_max_ts(&self) -> Result<(Option, Option)> { + let mut min_ts: Option = None; + let mut max_ts: Option = None; + if let Some(filters_map) = &self.filter { + for (field, value) in filters_map.iter() { + match field { + OrderField::Created | OrderField::StatementTimestamp => { + let date_time_utc = DateTime::parse_from_rfc3339(&value) + .map_err(|err| anyhow!("{:?}", err))?; + let datetime = date_time_utc.with_timezone(&Utc); + let ts: i64 = datetime.timestamp(); + let ts_end: i64 = ts + 60; // Search along that minute, the second is not specified by the front. + min_ts = Some(ts); + max_ts = Some(ts_end); + } + _ => {} + } + } + } + + Ok((min_ts, max_ts)) + } + + #[instrument(skip_all)] + pub fn as_where_clause_map(&self) -> Result { + let mut cols_match_select = WhereClauseOrdMap::new(); + if let Some(filters_map) = &self.filter { + for (field, value) in filters_map.iter() { + match field { + OrderField::Id => {} // Why would someone filter the electoral log by id? + OrderField::SenderPk | OrderField::Username | OrderField::BallotId | OrderField::StatementKind | OrderField::Version => { // sql VARCHAR type + let variant = ElectoralLogVarCharColumn::from_str(field.to_string().as_str()).map_err(|_| anyhow!("Field not found"))?; + cols_match_select.insert( + variant, + SqlCompOperators::Equal(value.clone()), // Using 'Like' here would not scale for millions of entries, causing no response from immudb is some cases. + ); + } + OrderField::UserId => { + // insert user_id_mod + cols_match_select.insert( + ElectoralLogVarCharColumn::UserIdKey, + SqlCompOperators::Equal(value.clone().chars().take(3).collect()), + ); + let variant = ElectoralLogVarCharColumn::from_str(field.to_string().as_str()).map_err(|_| anyhow!("Field not found"))?; + cols_match_select.insert( + variant, + SqlCompOperators::Equal(value.clone()), + ); + } + OrderField::StatementTimestamp | OrderField::Created => {} // handled by `get_min_max_ts` + OrderField::EventType | OrderField::LogType | OrderField::Description // these have no column but are inside of Message + | OrderField::Message => {} // Message column is sql BLOB type and it´s encrypted so we can't filter it without expensive operations + } + } + } + if let Some(election_id) = &self.election_id { + if !election_id.is_empty() { + cols_match_select.insert( + ElectoralLogVarCharColumn::ElectionId, + SqlCompOperators::Equal(election_id.clone()), + ); + } + } + + if let Some(area_ids) = &self.area_ids { + if !area_ids.is_empty() { + // NOTE: `IN` values must be handled later in SQL building, here we just join them + cols_match_select.insert( + ElectoralLogVarCharColumn::AreaId, + SqlCompOperators::In(area_ids.clone()), // TODO: NullOrIn + ); + } + } + + if let Some(statement_kind) = &self.statement_kind { + cols_match_select.insert( + ElectoralLogVarCharColumn::StatementKind, + SqlCompOperators::Equal(statement_kind.to_string()), + ); + } + + Ok(cols_match_select) + } + + #[instrument] + pub fn as_cast_vote_count_and_select_clauses( + &self, + election_id: &str, + user_id: &str, + ballot_id_filter: &str, + ) -> (WhereClauseOrdMap, WhereClauseOrdMap) { + let cols_match_count = WhereClauseOrdMap::from(&[ + ( + ElectoralLogVarCharColumn::StatementKind, + SqlCompOperators::Equal(StatementType::CastVote.to_string()), + ), + ( + ElectoralLogVarCharColumn::ElectionId, + SqlCompOperators::Equal(election_id.to_string()), + ), + ]); + let mut cols_match_select = cols_match_count.clone(); + // Restrict the SQL query to user_id and ballot_id in case of filtering + if !ballot_id_filter.is_empty() { + cols_match_select.insert( + ElectoralLogVarCharColumn::UserIdKey, + SqlCompOperators::Equal(user_id.chars().take(3).collect()), + ); + cols_match_select.insert( + ElectoralLogVarCharColumn::UserId, + SqlCompOperators::Equal(user_id.to_string()), + ); + cols_match_select.insert( + ElectoralLogVarCharColumn::BallotId, + SqlCompOperators::Like(ballot_id_filter.to_string()), + ); + } + + (cols_match_count, cols_match_select) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ElectoralLogMessage { + pub id: i64, + pub created: i64, + pub sender_pk: String, + pub statement_timestamp: i64, + pub statement_kind: String, + pub message: Vec, + pub version: String, + pub user_id: Option, + pub username: Option, + pub election_id: Option, + pub area_id: Option, + pub ballot_id: Option, +} + +impl TryFrom<&Row> for ElectoralLogMessage { + type Error = anyhow::Error; + + fn try_from(row: &Row) -> Result { + let mut id = 0; + let mut created = 0; + let mut sender_pk = String::from(""); + let mut statement_timestamp = 0; + let mut statement_kind = String::from(""); + let mut message = vec![]; + let mut version = String::from(""); + let mut user_id: Option = None; + let mut username: Option = None; + let mut election_id: Option = None; + let mut area_id: Option = None; + let mut ballot_id: Option = None; + + for (column, value) in row.columns.iter().zip(row.values.iter()) { + // FIXME for some reason columns names appear with parentheses + let dot = column + .find('.') + .ok_or(anyhow!("invalid column found '{}'", column.as_str()))?; + let bare_column = &column[dot + 1..column.len() - 1]; + + match bare_column { + "id" => assign_value!(Value::N, value, id), + "created" => assign_value!(Value::Ts, value, created), + "sender_pk" => assign_value!(Value::S, value, sender_pk), + "statement_timestamp" => { + assign_value!(Value::Ts, value, statement_timestamp) + } + "statement_kind" => assign_value!(Value::S, value, statement_kind), + "message" => assign_value!(Value::Bs, value, message), + "version" => assign_value!(Value::S, value, version), + "user_id" => match value.value.as_ref() { + Some(Value::S(inner)) => user_id = Some(inner.clone()), + Some(Value::Null(_)) => user_id = None, + None => user_id = None, + _ => { + return Err(anyhow!( + "invalid column value for 'user_id': {:?}", + value.value.as_ref() + )) + } + }, + "username" => match value.value.as_ref() { + Some(Value::S(inner)) => username = Some(inner.clone()), + Some(Value::Null(_)) => username = None, + None => username = None, + _ => { + return Err(anyhow!( + "invalid column value for 'username': {:?}", + value.value.as_ref() + )) + } + }, + "election_id" => match value.value.as_ref() { + Some(Value::S(inner)) => election_id = Some(inner.clone()), + Some(Value::Null(_)) => election_id = None, + None => election_id = None, + _ => { + return Err(anyhow!( + "invalid column value for 'election_id': {:?}", + value.value.as_ref() + )) + } + }, + "area_id" => match value.value.as_ref() { + Some(Value::S(inner)) => area_id = Some(inner.clone()), + Some(Value::Null(_)) => area_id = None, + None => area_id = None, + _ => { + return Err(anyhow!( + "invalid column value for 'area_id': {:?}", + value.value.as_ref() + )) + } + }, + "ballot_id" => match value.value.as_ref() { + Some(Value::S(inner)) => ballot_id = Some(inner.clone()), + Some(Value::Null(_)) => ballot_id = None, + None => ballot_id = None, + _ => { + return Err(anyhow!( + "invalid column value for 'ballod_id': {:?}", + value.value.as_ref() + )) + } + }, + _ => return Err(anyhow!("invalid column found '{}'", bare_column)), + } + } + + Ok(ElectoralLogMessage { + id, + created, + sender_pk, + statement_timestamp, + statement_kind, + message, + version, + user_id, + username, + election_id, + area_id, + ballot_id, + }) + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct Aggregate { + pub count: i64, +} + +impl TryFrom<&Row> for Aggregate { + type Error = anyhow::Error; + + fn try_from(row: &Row) -> Result { + let mut count = 0; + + for (column, value) in row.columns.iter().zip(row.values.iter()) { + match column.as_str() { + _ => assign_value!(Value::N, value, count), + } + } + Ok(Aggregate { count }) + } +} diff --git a/packages/electoral-log/src/messages/message.rs b/packages/electoral-log/src/messages/message.rs index 7963396eb87..3468c8b9bb1 100644 --- a/packages/electoral-log/src/messages/message.rs +++ b/packages/electoral-log/src/messages/message.rs @@ -2,11 +2,12 @@ // // SPDX-License-Identifier: AGPL-3.0-only -use crate::ElectoralLogMessage; +use crate::client::types::ElectoralLogMessage; use anyhow::Result; use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use strand::hash::STRAND_HASH_LENGTH_BYTES; use strand::hash::STRAND_HASH_LENGTH_BYTES; use strand::serialization::StrandSerialize; use strand::signature::StrandSignature; @@ -26,7 +27,7 @@ use std::fmt; /// a cross-event statement pub const GENERIC_EVENT: &'static str = "Generic Event"; -#[derive(BorshSerialize, BorshDeserialize, Serialize, Deserialize, std::fmt::Debug)] +#[derive(Clone, BorshSerialize, BorshDeserialize, Serialize, Deserialize, std::fmt::Debug)] pub struct Message { pub sender: Sender, pub sender_signature: StrandSignature, diff --git a/packages/electoral-log/src/messages/statement.rs b/packages/electoral-log/src/messages/statement.rs index 181baf09361..045994c901c 100644 --- a/packages/electoral-log/src/messages/statement.rs +++ b/packages/electoral-log/src/messages/statement.rs @@ -10,7 +10,7 @@ use strum_macros::Display; use crate::messages::newtypes::*; use tracing::info; -#[derive(BorshSerialize, BorshDeserialize, Deserialize, Serialize, Debug)] +#[derive(BorshSerialize, BorshDeserialize, Deserialize, Serialize, Debug, Clone)] pub struct Statement { pub head: StatementHead, pub body: StatementBody, @@ -173,7 +173,7 @@ impl StatementHead { } } -#[derive(BorshSerialize, BorshDeserialize, Deserialize, Serialize, Debug)] +#[derive(BorshSerialize, BorshDeserialize, Deserialize, Serialize, Debug, Clone)] pub enum StatementBody { // NOT IMPLEMENTED YET, but please feel free // "Emisión de voto (sólo como registro que el sistema almacenó correctamente el voto) diff --git a/packages/harvest/src/routes/electoral_log.rs b/packages/harvest/src/routes/electoral_log.rs index 32a52d88b1f..b4a0b0a1d90 100644 --- a/packages/harvest/src/routes/electoral_log.rs +++ b/packages/harvest/src/routes/electoral_log.rs @@ -3,35 +3,120 @@ // SPDX-License-Identifier: AGPL-3.0-only use crate::services::authorization::authorize; +use crate::types::resources::TotalAggregate; use anyhow::{anyhow, Context, Result}; use deadpool_postgres::Client as DbClient; +use electoral_log::client::types::*; use rocket::http::Status; use rocket::serde::json::Json; use sequent_core::services::jwt::JwtClaims; +use sequent_core::services::keycloak::get_event_realm; use sequent_core::types::permissions::Permissions; -use serde::{Deserialize, Serialize}; -use tracing::instrument; +use tracing::{info, instrument}; +use windmill::services::database::get_keycloak_pool; use windmill::services::electoral_log::{ - list_electoral_log as get_logs, ElectoralLogRow, GetElectoralLogBody, + count_electoral_log, list_electoral_log as windmill_list_electoral_log, + ElectoralLogRow, }; +use windmill::services::users::get_users_by_username; use windmill::types::resources::DataList; -#[instrument] +#[instrument(skip(claims))] #[post("/immudb/electoral-log", format = "json", data = "")] pub async fn list_electoral_log( body: Json, claims: JwtClaims, ) -> Result>, (Status, String)> { - let input = body.into_inner(); + let mut input = body.into_inner(); authorize( &claims, true, Some(input.tenant_id.clone()), vec![Permissions::LOGS_READ], )?; - let ret_val = get_logs(input) + + // If there is username but no user_id in the filter, fill the user_id to + // inprove performance. + if let Some(filter) = &mut input.filter { + if let (Some(username), None) = ( + filter.get(&OrderField::Username), + filter.get(&OrderField::UserId), + ) { + match get_user_id( + &input.tenant_id, + &input.election_event_id, + username, + ) + .await + { + Ok(Some(user_id)) => { + filter.insert(OrderField::UserId, user_id); + } + Ok(None) => { + return Ok(Json(DataList::default())); + } + Err(e) => { + return Err((Status::InternalServerError, e.to_string())); + } + } + } + } + + let (data_res, count_res) = tokio::join!( + windmill_list_electoral_log(input.clone()), + count_electoral_log(input) + ); + + let mut data = data_res.map_err(|e| { + ( + Status::InternalServerError, + format!("Eror listing electoral log: {e:?}"), + ) + })?; + data.total.aggregate.count = count_res.map_err(|e| { + ( + Status::InternalServerError, + format!("Error counting electoral log: {e:?}"), + ) + })?; + + Ok(Json(data)) +} + +/// Get user id by username +#[instrument(err)] +pub async fn get_user_id( + tenant_id: &str, + election_event_id: &str, + username: &str, +) -> Result> { + let realm = get_event_realm(tenant_id, election_event_id); + let mut keycloak_db_client: DbClient = get_keycloak_pool() + .await + .get() .await - .map_err(|e| (Status::InternalServerError, format!("{:?}", e)))?; + .map_err(|e| anyhow!("Error getting keycloak client: {e:?}"))?; + + let keycloak_transaction = keycloak_db_client + .transaction() + .await + .map_err(|e| anyhow!("Error getting keycloak transaction: {e:?}"))?; + + let user_ids = + get_users_by_username(&keycloak_transaction, &realm, username) + .await + .map_err(|e| anyhow!("Error getting users by username: {e:?}"))?; - Ok(Json(ret_val)) + match user_ids.len() { + 0 => { + info!("Could not get users by username: Not Found"); + return Ok(None); + } + 1 => Ok(Some(user_ids[0].clone())), + _ => { + return Err(anyhow!( + "Error getting users by username: Multiple users Found" + )); + } + } } diff --git a/packages/harvest/src/routes/immudb_log_audit.rs b/packages/harvest/src/routes/immudb_log_audit.rs index 0bcbaaafd52..7cfefb5bd73 100644 --- a/packages/harvest/src/routes/immudb_log_audit.rs +++ b/packages/harvest/src/routes/immudb_log_audit.rs @@ -3,11 +3,10 @@ // SPDX-License-Identifier: AGPL-3.0-only use crate::services::authorization::authorize; -use crate::types::resources::{ - Aggregate, DataList, OrderDirection, TotalAggregate, -}; +use crate::types::resources::{Aggregate, DataList, TotalAggregate}; use anyhow::{anyhow, Context, Result}; use electoral_log::assign_value; +use electoral_log::client::types::OrderDirection; use immudb_rs::{sql_value::Value, Client, NamedParam, Row, SqlValue}; use rocket::http::Status; use rocket::response::Debug; diff --git a/packages/harvest/src/routes/voter_electoral_log.rs b/packages/harvest/src/routes/voter_electoral_log.rs index 7ea3e1f2679..727a5d2f8e2 100644 --- a/packages/harvest/src/routes/voter_electoral_log.rs +++ b/packages/harvest/src/routes/voter_electoral_log.rs @@ -5,6 +5,7 @@ use crate::services::authorization::authorize_voter_election; use crate::types::error_response::{ErrorCode, ErrorResponse, JsonError}; use anyhow::Result; +use electoral_log::client::types::*; use rocket::http::Status; use rocket::serde::json::Json; use sequent_core::ballot::ShowCastVoteLogs; @@ -15,12 +16,9 @@ use serde::Deserialize; use std::collections::HashMap; use tracing::instrument; use windmill::postgres::election_event::get_election_event_by_id; -use windmill::services::electoral_log; -use windmill::services::electoral_log::{ - CastVoteMessagesOutput, GetElectoralLogBody, OrderField, -}; +use windmill::services::electoral_log::list_cast_vote_messages_and_count; +use windmill::services::electoral_log::CastVoteMessagesOutput; use windmill::services::providers::transactions_provider::provide_hasura_transaction; -use windmill::types::resources::OrderDirection; #[derive(Deserialize, Debug)] pub struct CastVoteMessagesInput { @@ -104,7 +102,7 @@ pub async fn list_cast_vote_messages( ..Default::default() }; - let ret_val = electoral_log::list_cast_vote_messages( + let ret_val = list_cast_vote_messages_and_count( elog_input, ballot_id, &user_id, &username, ) .await diff --git a/packages/harvest/src/types/resources.rs b/packages/harvest/src/types/resources.rs index 0cc57943ea6..e205d1ea63f 100644 --- a/packages/harvest/src/types/resources.rs +++ b/packages/harvest/src/types/resources.rs @@ -17,15 +17,6 @@ pub struct TotalAggregate { pub aggregate: Aggregate, } -// Enumeration for the valid order directions -#[derive(Debug, Deserialize, EnumString, Display)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum OrderDirection { - Asc, - Desc, -} - #[derive(Deserialize, Debug)] pub struct SortPayload { pub field: String, diff --git a/packages/step-cli/src/commands/create_electoral_logs.rs b/packages/step-cli/src/commands/create_electoral_logs.rs index 019af902bc7..54547dc83d2 100644 --- a/packages/step-cli/src/commands/create_electoral_logs.rs +++ b/packages/step-cli/src/commands/create_electoral_logs.rs @@ -6,12 +6,12 @@ use crate::utils::read_config::load_external_config; use anyhow::{anyhow, Context, Result}; use chrono::Utc; use clap::Args; +use electoral_log::client::types::ElectoralLogMessage; use electoral_log::messages::message::{Message, Sender}; use electoral_log::messages::newtypes::EventIdString; use electoral_log::messages::statement::{ Statement, StatementBody, StatementEventType, StatementHead, StatementLogType, StatementType, }; -use electoral_log::ElectoralLogMessage; use fake::faker::internet::raw::Username; use fake::locales::EN; use fake::Fake; diff --git a/packages/step-cli/src/commands/export_cast_votes.rs b/packages/step-cli/src/commands/export_cast_votes.rs index 4032941fd9a..6d57e2c67c3 100644 --- a/packages/step-cli/src/commands/export_cast_votes.rs +++ b/packages/step-cli/src/commands/export_cast_votes.rs @@ -9,6 +9,9 @@ use base64::engine::general_purpose; use base64::Engine; use clap::Args; use csv::WriterBuilder; +use electoral_log::client::types::{ + ElectoralLogVarCharColumn, SqlCompOperators, WhereClauseOrdMap, +}; use electoral_log::messages::message::Message; use electoral_log::messages::newtypes::ElectionIdString; use electoral_log::messages::statement::{StatementBody, StatementType}; @@ -78,9 +81,9 @@ impl ExportCastVotes { .await .map_err(|err| anyhow!("Failed to create the client: {:?}", err))?; - let cols_match = BTreeMap::from([( + let cols_match = WhereClauseOrdMap::from(&[( ElectoralLogVarCharColumn::StatementKind, - (SqlCompOperators::Equal, StatementType::CastVote.to_string()), + SqlCompOperators::Equal(StatementType::CastVote.to_string()), )]); let order_by: Option> = None; println!("Getting messages"); diff --git a/packages/voting-portal/src/routes/BallotLocator.tsx b/packages/voting-portal/src/routes/BallotLocator.tsx index d8dc49c8032..b1436ebe363 100644 --- a/packages/voting-portal/src/routes/BallotLocator.tsx +++ b/packages/voting-portal/src/routes/BallotLocator.tsx @@ -671,7 +671,6 @@ const BallotLocatorLogic: React.FC = ({customCss}) => { const {t} = useTranslation() const [inputBallotId, setInputBallotId] = useState("") const {globalSettings} = useContext(SettingsContext) - const hasBallotId = !!ballotId const {data: dataBallotStyles} = useQuery(GET_BALLOT_STYLES) diff --git a/packages/windmill/external-bin/generate_logs.rs b/packages/windmill/external-bin/generate_logs.rs old mode 100644 new mode 100755 index 2a4393170d4..d10b5c705e9 --- a/packages/windmill/external-bin/generate_logs.rs +++ b/packages/windmill/external-bin/generate_logs.rs @@ -8,6 +8,7 @@ use base64::Engine; use chrono::{TimeZone, Utc}; use clap::Parser; use csv::Writer; +use electoral_log::client::types::ElectoralLogMessage; use electoral_log::messages::message::Message; use immudb_rs::{sql_value::Value as ImmudbSqlValue, Client}; use serde::Deserialize; @@ -187,10 +188,18 @@ async fn main() -> Result<()> { info!(total_rows_fetched, "Processed rows from stream..."); } - let elog_row = match ElectoralLogRow::try_from(individual_row) { + let elog_msg = match ElectoralLogMessage::try_from(individual_row) { + Ok(elog_msg) => elog_msg, + Err(e) => { + warn!(error = %e, "Failed to parse ImmudbRow into ElectoralLogMessage from stream batch."); + continue; + } + }; + + let elog_row = match ElectoralLogRow::try_from(elog_msg.clone()) { Ok(elog_row) => elog_row, Err(e) => { - warn!(error = %e, "Failed to parse ImmudbRow into ElectoralLogRow from stream batch."); + warn!(error = %e, "Failed to parse ImmudbRow into ElectoralLogRow from ElectoralLogMessage."); continue; } }; @@ -217,10 +226,10 @@ async fn main() -> Result<()> { }; let extracted_election_id_opt = message.election_id.clone(); - let activity_log_row = match ActivityLogRow::try_from(elog_row.clone()) { + let activity_log_row = match ActivityLogRow::try_from(elog_msg.clone()) { Ok(activity_log_row) => activity_log_row, Err(e) => { - warn!(log_id = elog_row.id, error = %e, "Failed to transform ElectoralLogRow."); + warn!(log_id = elog_msg.id, error = %e, "Failed to transform ElectoralLogMessage."); continue; } }; diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index 98b0fabc193..2ae348d72bf 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -18,6 +18,7 @@ use b3::messages::message::Signer; use base64::engine::general_purpose; use base64::Engine; use deadpool_postgres::Transaction; +use electoral_log::client::types::*; use electoral_log::assign_value; use electoral_log::messages::message::{Message, SigningData}; use electoral_log::messages::newtypes::*; @@ -28,6 +29,7 @@ use electoral_log::{ use immudb_rs::{sql_value::Value, Client, NamedParam, Row, TxMode}; use rust_decimal::prelude::ToPrimitive; use sequent_core::serialization::deserialize_with_path::{deserialize_str, deserialize_value}; +use sequent_core::util::retry::retry_with_exponential_backoff; use sequent_core::services::date::ISO8601; use sequent_core::util::retry::retry_with_exponential_backoff; use serde::{Deserialize, Serialize}; @@ -42,7 +44,7 @@ use strand::signature::{StrandSignaturePk, StrandSignatureSk}; use strum_macros::{Display, EnumString}; use tempfile::NamedTempFile; use tokio_stream::StreamExt; -use tracing::{event, info, instrument, warn, Level}; +use tracing::{info, instrument, warn}; pub const IMMUDB_ROWS_LIMIT: usize = 2500; pub const MAX_ROWS_PER_PAGE: usize = 50; @@ -763,199 +765,14 @@ impl ElectoralLog { } } -// Enumeration for the valid fields in the immudb table -#[derive(Debug, Deserialize, Hash, PartialEq, Eq, EnumString, Display, Clone)] -#[serde(rename_all = "snake_case")] -#[strum(serialize_all = "snake_case")] -pub enum OrderField { - Id, - Created, - StatementTimestamp, - StatementKind, - Message, - UserId, - Username, - BallotId, - SenderPk, - LogType, - EventType, - Description, - Version, -} - -#[derive(Deserialize, Debug, Default, Clone)] -pub struct GetElectoralLogBody { - pub tenant_id: String, - pub election_event_id: String, - pub limit: Option, - pub offset: Option, - pub filter: Option>, - pub order_by: Option>, - pub election_id: Option, - pub area_ids: Option>, - pub only_with_user: Option, - pub statement_kind: Option, -} - -impl GetElectoralLogBody { - // Returns the SQL clauses related to the request along with the parameters - #[instrument(ret)] - fn as_sql(&self, to_count: bool) -> Result<(String, Vec)> { - let mut clauses = Vec::new(); - let mut params = Vec::new(); - - // Handle filters - if let Some(filters_map) = &self.filter { - let mut where_clauses = Vec::new(); - - for (field, value) in filters_map { - info!("field = ?: {field}, value = ?: {value}"); - let param_name = format!("param_{field}"); - match field { - OrderField::Id => { // sql INTEGER type - let int_value: i64 = value.parse()?; - where_clauses.push(format!("id = @{}", param_name)); - params.push(create_named_param(param_name, Value::N(int_value))); - } - OrderField::SenderPk | OrderField::UserId | OrderField::Username | OrderField::BallotId | OrderField::StatementKind | OrderField::Version => { // sql VARCHAR type - where_clauses.push(format!("{field} LIKE @{}", param_name)); - params.push(create_named_param(param_name, Value::S(value.to_string()))); - } - OrderField::StatementTimestamp | OrderField::Created => { // sql TIMESTAMP type - // these have their own column and are inside of Message´s column as well - let datetime = ISO8601::to_date_utc(&value) - .map_err(|err| anyhow!("Failed to parse timestamp: {:?}", err))?; - let ts: i64 = datetime.timestamp(); - let ts_end: i64 = ts + 60; // Search along that minute, the second is not specified by the front. - let param_name_end = format!("{param_name}_end"); - where_clauses.push(format!("{field} >= @{} AND {field} < @{}", param_name, param_name_end)); - params.push(create_named_param(param_name, Value::Ts(ts))); - params.push(create_named_param(param_name_end, Value::Ts(ts_end))); - } - OrderField::EventType | OrderField::LogType | OrderField::Description // these have no column but are inside of Message - | OrderField::Message => {} // Message column is sql BLOB type and it´s encrypted so we can't search it without expensive operations - } - } - - if !where_clauses.is_empty() { - clauses.push(format!("WHERE {}", where_clauses.join(" AND "))); - } - }; - - // Build a single extra clause. - // This clause returns rows if: - // - @election_filter is non-empty and matches election_id, OR - // - @area_filter is non-empty and matches area_id, OR - // - Both election_id and area_id are either '' or NULL. (General to all elections log) - let mut extra_where_clauses = Vec::new(); - if self.election_id.is_some() || self.area_ids.is_some() { - let mut conds = Vec::new(); - - if let Some(election) = &self.election_id { - if !election.is_empty() { - params.push(create_named_param( - "param_election".to_string(), - Value::S(election.clone()), - )); - conds.push("election_id LIKE @param_election".to_string()); - } - } - - if let Some(area_ids) = &self.area_ids { - if !area_ids.is_empty() { - let placeholders: Vec = area_ids - .iter() - .enumerate() - .map(|(i, _)| format!("@param_area{}", i)) - .collect(); - for (i, area) in area_ids.into_iter().enumerate() { - let param_name = format!("param_area{}", i); - params.push(create_named_param( - param_name.clone(), - Value::S(area.clone()), - )); - } - conds.push(format!( - "(@param_area0 <> '' AND area_id IN ({}))", - placeholders.join(", ") - )); - } - } - - // if neither filter matches, return logs where both fields are empty or NULL. - conds.push( - "((election_id = '' OR election_id IS NULL) AND (area_id = '' OR area_id IS NULL))" - .to_string(), - ); - - extra_where_clauses.push(format!("({})", conds.join(" OR "))); - } - - // Handle only_with_user - if self.only_with_user.unwrap_or(false) { - extra_where_clauses.push("(user_id IS NOT NULL AND user_id <> '')".to_string()); - } - - // Handle - if let Some(statement_kind) = &self.statement_kind { - params.push(create_named_param( - "param_statement_kind".to_string(), - Value::S(statement_kind.to_string()), - )); - extra_where_clauses.push("(statement_kind = @param_statement_kind)".to_string()); - } - - if !extra_where_clauses.is_empty() { - match clauses.len() { - 0 => { - clauses.push(format!("WHERE {}", extra_where_clauses.join(" AND "))); - } - _ => { - let where_clause = clauses.pop().ok_or(anyhow!("Empty clause"))?; - clauses.push(format!( - "{} AND {}", - where_clause, - extra_where_clauses.join(" AND ") - )); - } - } - } - - // Handle order_by - if !to_count && self.order_by.is_some() { - let order_by_clauses: Vec = self - .order_by - .as_ref() - .ok_or(anyhow!("Empty order clause"))? - .iter() - .map(|(field, direction)| format!("{field} {direction}")) - .collect(); - if order_by_clauses.len() > 0 { - clauses.push(format!("ORDER BY {}", order_by_clauses.join(", "))); - } - } - - // Handle limit - if !to_count { - let limit_param_name = String::from("limit"); - let limit_value = self - .limit - .unwrap_or(PgConfig::from_env()?.default_sql_limit.into()); - let limit = std::cmp::min(limit_value, PgConfig::from_env()?.low_sql_limit.into()); - clauses.push(format!("LIMIT @{limit_param_name}")); - params.push(create_named_param(limit_param_name, Value::N(limit))); - } - - // Handle offset - if !to_count && self.offset.is_some() { - let offset_param_name = String::from("offset"); - let offset = std::cmp::max(self.offset.unwrap_or(0), 0); - clauses.push(format!("OFFSET @{}", offset_param_name)); - params.push(create_named_param(offset_param_name, Value::N(offset))); - } - - Ok((clauses.join(" "), params)) - } +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct StatementHeadDataString { + pub event: String, + pub kind: String, + pub timestamp: i64, + pub event_type: String, + pub log_type: String, + pub description: String, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -969,14 +786,28 @@ pub struct ElectoralLogRow { pub user_id: Option, pub username: Option, } -#[derive(Deserialize, Serialize, Debug, Clone)] -pub struct StatementHeadDataString { - pub event: String, - pub kind: String, - pub timestamp: i64, - pub event_type: String, - pub log_type: String, - pub description: String, + +// Removing this step would inprove the performance, i.e. return the final type directly from ElectoralLogMessage. +impl TryFrom for ElectoralLogRow { + type Error = anyhow::Error; + + fn try_from(elog_msg: ElectoralLogMessage) -> Result { + let serialized = general_purpose::STANDARD_NO_PAD.encode(elog_msg.message.clone()); + let deserialized_message = Message::strand_deserialize(&elog_msg.message) + .map_err(|e| anyhow!("Error deserializing message: {e:?}"))?; + + Ok(ElectoralLogRow { + id: elog_msg.id, + created: elog_msg.created, + statement_timestamp: elog_msg.statement_timestamp, + statement_kind: elog_msg.statement_kind.clone(), + message: serde_json::to_string_pretty(&deserialized_message) + .with_context(|| "Error serializing message to json")?, + data: serialized, + user_id: elog_msg.user_id.clone(), + username: elog_msg.username.clone(), + }) + } } impl ElectoralLogRow { @@ -1037,72 +868,6 @@ impl ElectoralLogRow { } } -impl TryFrom<&Row> for ElectoralLogRow { - type Error = anyhow::Error; - - fn try_from(row: &Row) -> Result { - let mut id = 0; - let mut created: i64 = 0; - let mut sender_pk = String::from(""); - let mut statement_timestamp: i64 = 0; - let mut statement_kind = String::from(""); - let mut message = vec![]; - let mut user_id = None; - let mut username = None; - - for (column, value) in row.columns.iter().zip(row.values.iter()) { - match column.as_str() { - c if c.ends_with(".id)") => { - assign_value!(Value::N, value, id) - } - c if c.ends_with(".created)") => { - assign_value!(Value::Ts, value, created) - } - c if c.ends_with(".sender_pk)") => { - assign_value!(Value::S, value, sender_pk) - } - c if c.ends_with(".statement_timestamp)") => { - assign_value!(Value::Ts, value, statement_timestamp) - } - c if c.ends_with(".statement_kind)") => { - assign_value!(Value::S, value, statement_kind) - } - c if c.ends_with(".message)") => { - assign_value!(Value::Bs, value, message) - } - c if c.ends_with(".user_id)") => match value.value.as_ref() { - Some(Value::S(inner)) => user_id = Some(inner.clone()), - Some(Value::Null(_)) => user_id = None, - None => user_id = None, - _ => return Err(anyhow!("invalid column value for 'user_id'")), - }, - c if c.ends_with(".username)") => match value.value.as_ref() { - Some(Value::S(inner)) => username = Some(inner.clone()), - Some(Value::Null(_)) => username = None, - None => username = None, - _ => return Err(anyhow!("invalid column value for 'username'")), - }, - _ => return Err(anyhow!("invalid column found '{}'", column.as_str())), - } - } - - let deserialized_message = - Message::strand_deserialize(&message).with_context(|| "Error deserializing message")?; - let serialized = general_purpose::STANDARD_NO_PAD.encode(message); - Ok(ElectoralLogRow { - id, - created, - statement_timestamp, - statement_kind, - message: serde_json::to_string_pretty(&deserialized_message) - .with_context(|| "Error serializing message to json")?, - data: serialized, - user_id, - username, - }) - } -} - #[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct CastVoteEntry { pub statement_timestamp: i64, @@ -1138,77 +903,40 @@ impl CastVoteEntry { #[instrument(err)] pub async fn list_electoral_log(input: GetElectoralLogBody) -> Result> { - let mut client: Client = get_immudb_client().await?; - let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; - let board_name = get_event_board( - input.tenant_id.as_str(), - input.election_event_id.as_str(), - &slug, - ); + let mut client = get_board_client().await?; + let board_name = get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + info!("database name = {board_name}"); + let cols_match_select = input.as_where_clause_map()?; + let order_by = input.order_by.clone(); + let (min_ts, max_ts) = input.get_min_max_ts()?; + let limit: i64 = input.limit.unwrap_or(IMMUDB_ROWS_LIMIT as i64); + let offset: i64 = input.offset.unwrap_or(0); + let mut rows: Vec = vec![]; + let electoral_log_messages = client + .get_electoral_log_messages_filtered( + &board_name, + Some(cols_match_select.clone()), + min_ts, + max_ts, + Some(limit), + Some(offset), + order_by.clone(), + ) + .await + .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; - event!(Level::INFO, "database name = {board_name}"); - info!("input = {:?}", input); - client.open_session(&board_name).await?; - let (clauses, params) = input.as_sql(false)?; - let (clauses_to_count, count_params) = input.as_sql(true)?; - info!("clauses ?:= {clauses}"); - let sql = format!( - r#" - SELECT - id, - created, - sender_pk, - statement_timestamp, - statement_kind, - message, - user_id, - username - FROM electoral_log_messages - {clauses} - "#, - ); - info!("query: {sql}"); - let sql_query_response = client.streaming_sql_query(&sql, params).await?; - - let limit: usize = input.limit.unwrap_or(IMMUDB_ROWS_LIMIT as i64).try_into()?; - - let mut rows: Vec = Vec::with_capacity(limit); - let mut resp_stream = sql_query_response.into_inner(); - while let Some(streaming_batch) = resp_stream.next().await { - let items = streaming_batch? - .rows - .iter() - .map(ElectoralLogRow::try_from) - .collect::>>()?; - rows.extend(items); + let t_entries = electoral_log_messages.len(); + info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}"); + for message in electoral_log_messages { + rows.push(message.try_into()?); } - let sql = format!( - r#" - SELECT - COUNT(*) - FROM electoral_log_messages - {clauses_to_count} - "#, - ); - let sql_query_response = client.sql_query(&sql, count_params).await?; - let mut rows_iter = sql_query_response - .get_ref() - .rows - .iter() - .map(Aggregate::try_from); - - let aggregate = rows_iter - // get the first item - .next() - // unwrap the Result and Option - .ok_or(anyhow!("No aggregate found"))??; - - client.close_session().await?; Ok(DataList { items: rows, total: TotalAggregate { - aggregate: aggregate, + aggregate: Aggregate { + count: t_entries as i64, + }, }, }) } @@ -1249,7 +977,7 @@ pub fn get_cols_match_count_and_select( /// ballot_id_filter is restricted to be an even number of characters, so that can be converted /// to a byte array #[instrument(err)] -pub async fn list_cast_vote_messages( +pub async fn list_cast_vote_messages_theirs( input: GetElectoralLogBody, ballot_id_filter: &str, user_id: &str, @@ -1326,41 +1054,126 @@ pub async fn list_cast_vote_messages( Ok(CastVoteMessagesOutput { list, total }) } -#[instrument(err)] -pub async fn count_electoral_log(input: GetElectoralLogBody) -> Result { - let mut client = get_immudb_client().await?; - let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; - let board_name = get_event_board( - input.tenant_id.as_str(), - input.election_event_id.as_str(), - &slug, +/// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input. +/// ballot_id_filter is restricted to be an even number of characters, so thatnit can be converted +/// to a byte array. +#[instrument(err, skip_all)] +pub async fn list_cast_vote_messages_and_count( + input: GetElectoralLogBody, + ballot_id_filter: &str, + user_id: &str, + username: &str, +) -> Result { + ensure!( + ballot_id_filter.chars().count() % 2 == 0 && ballot_id_filter.is_ascii(), + "Incorrect ballot_id, the length must be an even number of characters" ); + let election_id = input.election_id.clone().unwrap_or_default(); + let (cols_match_count, cols_match_select) = + input.as_cast_vote_count_and_select_clauses(&election_id, user_id, ballot_id_filter); - info!("board name: {board_name}"); - client.open_session(&board_name).await?; - - let (clauses_to_count, count_params) = input.as_sql(true)?; - let sql = format!( - r#" - SELECT COUNT(*) - FROM electoral_log_messages - {clauses_to_count} - "#, + let (data_res, count_res) = tokio::join!( + list_cast_vote_messages( + input.clone(), + ballot_id_filter, + user_id, + username, + cols_match_select + ), + async { + let mut client = get_board_client().await?; + let board_name = + get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + info!("database name = {board_name}"); + let total: usize = client + .count_electoral_log_messages(&board_name, Some(cols_match_count)) + .await? + .to_usize() + .unwrap_or(0); + Ok(total) + } ); - info!("query: {sql}"); + let mut data = data_res.map_err(|e| anyhow!("Eror listing electoral log: {e:?}"))?; + data.total = + count_res.map_err(|e: anyhow::Error| anyhow!("Error counting electoral log: {e:?}"))?; - let sql_query_response = client.sql_query(&sql, count_params).await?; + Ok(data) +} - let mut rows_iter = sql_query_response - .get_ref() - .rows - .iter() - .map(Aggregate::try_from); - let aggregate = rows_iter - .next() - .ok_or_else(|| anyhow!("No aggregate found"))??; +#[instrument(err)] +pub async fn list_cast_vote_messages( + input: GetElectoralLogBody, + ballot_id_filter: &str, + user_id: &str, + username: &str, + cols_match_select: WhereClauseOrdMap, +) -> Result { + // The limits are used to cut the output after filtering the ballot id. + // Because ballot_id cannot be filtered at SQL level the sql limit is constant + let output_limit: i64 = input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64); + let board_name = get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + info!("database name = {board_name}"); + let order_by = input.order_by.clone(); - client.close_session().await?; - Ok(aggregate.count as i64) + let limit: i64 = match ballot_id_filter.is_empty() { + false => IMMUDB_ROWS_LIMIT as i64, // When there is a filter, need to fetch all entries by batches. + true => input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), + }; + let mut offset: i64 = input.offset.unwrap_or(0); + let mut list: Vec = Vec::with_capacity(MAX_ROWS_PER_PAGE); // Filtered messages. + + let mut client = get_board_client().await?; + let mut exit = false; // Exit at the first match if the filter is not empty or when the query returns 0 entries + while (list.len() as i64) < output_limit && !exit { + let electoral_log_messages = client + .get_electoral_log_messages_filtered( + &board_name, + Some(cols_match_select.clone()), + None, + None, + Some(limit), + Some(offset), + order_by.clone(), + ) + .await + .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; + + let t_entries = electoral_log_messages.len(); + info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}"); + for message in electoral_log_messages.iter() { + match CastVoteEntry::from_elog_message(&message, username)? { + Some(entry) if !ballot_id_filter.is_empty() => { + // If there is filter exit at the first match + exit = true; + list.push(entry); + } + Some(entry) => { + // Add all the entries till the limit, when there is no filter + list.push(entry); + } + None => {} + } + if (list.len() as i64) >= output_limit || exit { + break; + } + } + exit = exit || t_entries == 0; + offset += limit; + } + Ok(CastVoteMessagesOutput { list, total: 0 }) +} + +#[instrument(err)] +pub async fn count_electoral_log(input: GetElectoralLogBody) -> Result { + let mut client = get_board_client().await?; + let board_name = get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + info!("database name = {board_name}"); + let cols_match_count = input.as_where_clause_map()?; + let total = client + .count_electoral_log_messages(&board_name, Some(cols_match_count)) + .await? + .to_u64() + .unwrap_or(0) as i64; + Ok(total) } diff --git a/packages/windmill/src/services/event_list.rs b/packages/windmill/src/services/event_list.rs index 14b83aa5b5c..0ba4fcbc9e0 100644 --- a/packages/windmill/src/services/event_list.rs +++ b/packages/windmill/src/services/event_list.rs @@ -2,12 +2,10 @@ // // SPDX-License-Identifier: AGPL-3.0-only use crate::postgres::election_event::get_election_event_by_id; -use crate::{ - postgres::scheduled_event::{insert_new_scheduled_event, insert_scheduled_event}, - types::resources::OrderDirection, -}; +use crate::postgres::scheduled_event::{insert_new_scheduled_event, insert_scheduled_event}; use anyhow::Result; use deadpool_postgres::Transaction; +use electoral_log::client::types::OrderDirection; use rocket::http::Status; use sequent_core::services::keycloak; use sequent_core::types::hasura::core::ElectionEvent; diff --git a/packages/windmill/src/services/insert_cast_vote.rs b/packages/windmill/src/services/insert_cast_vote.rs index 6f39ffe2018..660c9d08d43 100644 --- a/packages/windmill/src/services/insert_cast_vote.rs +++ b/packages/windmill/src/services/insert_cast_vote.rs @@ -980,7 +980,7 @@ async fn check_previous_votes( .filter_map(|cv| cv.area_id.and_then(|id| Uuid::parse_str(&id).ok())) .partition(|cv_area_id| cv_area_id.to_string() == area_id.to_string()); - event!(Level::INFO, "get cast votes returns same: {:?}", same); + event!(Level::DEBUG, "get cast votes returns same: {:?}", same); // Skip max votes check if max_revotes is 0, allowing unlimited votes if max_revotes > 0 && same.len() >= max_revotes { diff --git a/packages/windmill/src/services/reports/activity_log.rs b/packages/windmill/src/services/reports/activity_log.rs index 54106f6ec9d..bc8cd2eec5b 100644 --- a/packages/windmill/src/services/reports/activity_log.rs +++ b/packages/windmill/src/services/reports/activity_log.rs @@ -4,30 +4,32 @@ use super::template_renderer::*; use crate::postgres::reports::{Report, ReportType}; -use crate::services::database::PgConfig; use crate::services::documents::upload_and_return_document; -use crate::services::electoral_log::{ - count_electoral_log, list_electoral_log, ElectoralLogRow, GetElectoralLogBody, -}; +use crate::services::electoral_log::{ElectoralLogRow, IMMUDB_ROWS_LIMIT}; +use crate::services::protocol_manager::{get_board_client, get_event_board}; use crate::services::providers::email_sender::{Attachment, EmailSender}; -use crate::services::temp_path::*; -use crate::types::resources::DataList; use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use csv::WriterBuilder; use deadpool_postgres::Transaction; +use electoral_log::client::types::*; +use electoral_log::messages::message::{Message, SigningData}; use sequent_core::services::date::ISO8601; -use sequent_core::services::keycloak::{self}; use sequent_core::services::s3::get_minio_url; use sequent_core::types::hasura::core::TasksExecution; use sequent_core::types::templates::{ReportExtraConfig, SendTemplateBody}; use sequent_core::util::temp_path::*; use serde::{Deserialize, Serialize}; +use std::mem; +use strand::serialization::StrandDeserialize; use strum_macros::EnumString; use tempfile::NamedTempFile; use tracing::{debug, info, instrument, warn}; -#[derive(Serialize, Deserialize, Debug, Clone, EnumString, PartialEq)] +const KB: f64 = 1024.0; +const MB: f64 = 1024.0 * KB; + +#[derive(Serialize, Deserialize, Debug, Clone, EnumString, PartialEq, Copy)] pub enum ReportFormat { CSV, PDF, @@ -46,43 +48,17 @@ pub struct ActivityLogRow { user_id: String, } -/// Struct for User Data -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct UserData { - pub act_log: Vec, - pub electoral_log: Vec, -} - -/// Struct for System Data -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct SystemData { - pub rendered_user_template: String, -} - -/// Implementation of TemplateRenderer for Activity Logs -#[derive(Debug)] -pub struct ActivityLogsTemplate { - ids: ReportOrigins, - report_format: ReportFormat, -} - -impl ActivityLogsTemplate { - pub fn new(ids: ReportOrigins, report_format: ReportFormat) -> Self { - ActivityLogsTemplate { ids, report_format } - } -} - -impl TryFrom for ActivityLogRow { +impl TryFrom for ActivityLogRow { type Error = anyhow::Error; - fn try_from(electoral_log: ElectoralLogRow) -> Result { - let user_id = match electoral_log.user_id() { + fn try_from(electoral_log: ElectoralLogMessage) -> Result { + let user_id = match electoral_log.user_id { Some(user_id) => user_id.to_string(), None => "-".to_string(), }; let statement_timestamp: String = if let Ok(datetime_parsed) = - ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.statement_timestamp()) + ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.statement_timestamp) { datetime_parsed.to_rfc3339() } else { @@ -90,34 +66,119 @@ impl TryFrom for ActivityLogRow { }; let created: String = if let Ok(datetime_parsed) = - ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.created()) + ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.created) { datetime_parsed.to_rfc3339() } else { return Err(anyhow::anyhow!("Error parsing created")); }; - let head_data = electoral_log - .statement_head_data() - .with_context(|| "Error to get head data.")?; - let event_type = head_data.event_type; - let log_type = head_data.log_type; + let deserialized_message = Message::strand_deserialize(&electoral_log.message) + .map_err(|e| anyhow!("Error deserializing message: {e:?}"))?; + + let head_data = deserialized_message.statement.head.clone(); + let event_type = head_data.event_type.to_string(); + let log_type = head_data.log_type.to_string(); let description = head_data.description; Ok(ActivityLogRow { - id: electoral_log.id(), + id: electoral_log.id, user_id: user_id, created, statement_timestamp, - statement_kind: electoral_log.statement_kind().to_string(), + statement_kind: electoral_log.statement_kind, event_type, log_type, description, - message: electoral_log.message().to_string(), + message: deserialized_message.to_string(), }) } } +/// Struct for User Data +/// act_log is for PDF +/// electoral_log is for CSV +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct UserData { + pub act_log: Vec, + pub electoral_log: Vec, +} + +/// Struct for System Data +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SystemData { + pub rendered_user_template: String, +} + +/// Implementation of TemplateRenderer for Activity Logs +#[derive(Debug)] +pub struct ActivityLogsTemplate { + ids: ReportOrigins, + report_format: ReportFormat, + board_name: String, +} + +impl ActivityLogsTemplate { + pub fn new(ids: ReportOrigins, report_format: ReportFormat) -> Self { + let board_name = get_event_board(ids.tenant_id.as_str(), ids.election_event_id.as_str()); + ActivityLogsTemplate { + ids, + report_format, + board_name, + } + } + + // Export data + #[instrument(err, skip(self))] + pub async fn generate_export_csv_data(&self, name: &str) -> Result { + let limit = IMMUDB_ROWS_LIMIT as i64; + let mut offset: i64 = 0; + // Create a temporary file to write CSV data + let mut temp_file = + generate_temp_file(&name, ".csv").with_context(|| "Error creating named temp file")?; + let mut csv_writer = WriterBuilder::new().from_writer(temp_file.as_file_mut()); + let total = self + .count_items(None) + .await + .map_err(|e| anyhow!("Error count_items in activity logs data: {e:?}"))? + .unwrap_or(0); + while offset < total { + info!("offset: {offset}, total: {total}"); + // Prepare user data + let user_data = self + .prepare_user_data_batch(None, None, &mut offset, limit) + .await + .map_err(|e| anyhow!("Error preparing activity logs data: {e:?}"))?; + + let s1 = user_data.electoral_log.len() * (mem::size_of::()); + let s2 = user_data.act_log.len() * (mem::size_of::()); + let kb = (s1 + s2) as f64 / KB; + let mb = (s1 + s2) as f64 / MB; + info!("Logs batch size: {kb:.2} KB, {mb:.2} MB"); + + for item in user_data.electoral_log { + let mut item_clone = item.clone(); + + // Replace newline characters in the message field + item_clone.message = item_clone.message.replace('\n', " ").replace('\r', " "); + // Serialize each item to CSV + csv_writer + .serialize(item_clone) + .map_err(|e| anyhow!("Error serializing to CSV: {e:?}"))?; + } + offset += limit; + } + + // Flush and finish writing to the temporary file + csv_writer + .flush() + .map_err(|e| anyhow!("Error flushing CSV writer: {e:?}"))?; + drop(csv_writer); + + Ok(temp_file) + } +} + #[async_trait] impl TemplateRenderer for ActivityLogsTemplate { type UserData = UserData; @@ -150,140 +211,61 @@ impl TemplateRenderer for ActivityLogsTemplate { fn prefix(&self) -> String { format!("activity_logs_{}", rand::random::()) } - async fn count_items(&self, _hasura_transaction: &Transaction<'_>) -> Result> { - let input = GetElectoralLogBody { - tenant_id: self.ids.tenant_id.clone(), - election_event_id: self.ids.election_event_id.clone(), - limit: None, - offset: None, - filter: None, - order_by: None, - area_ids: None, - only_with_user: None, - election_id: None, - statement_kind: None, - }; - Ok(count_electoral_log(input).await.ok()) + + async fn count_items( + &self, + _hasura_transaction: Option<&Transaction<'_>>, + ) -> Result> { + let mut client = get_board_client().await?; + let total = client + .count_electoral_log_messages(&self.board_name, None) + .await + .map_err(|e| anyhow!("Error counting electoral log messages: {e:?}"))?; + Ok(Some(total)) } + #[instrument(err, skip_all)] async fn prepare_user_data_batch( &self, - _hasura_transaction: &Transaction<'_>, - _keycloak_transaction: &Transaction<'_>, + _hasura_transaction: Option<&Transaction<'_>>, + _keycloak_transaction: Option<&Transaction<'_>>, offset: &mut i64, limit: i64, ) -> Result { let mut act_log: Vec = vec![]; - let mut elect_logs: Vec = vec![]; - - let electoral_logs: DataList = list_electoral_log(GetElectoralLogBody { - tenant_id: self.ids.tenant_id.clone(), - election_event_id: self.ids.election_event_id.clone(), - limit: Some(limit), - offset: Some(*offset), - filter: None, - order_by: None, - area_ids: None, - only_with_user: None, - election_id: None, - statement_kind: None, - }) - .await - .map_err(|e| anyhow!("Error listing electoral logs: {e:?}"))?; - - let is_empty = electoral_logs.items.is_empty(); - - for electoral_log in electoral_logs.items { - elect_logs.push(electoral_log.clone()); - let head_data = electoral_log - .statement_head_data() - .with_context(|| "Error to get head data.")?; - let event_type = head_data.event_type; - let log_type = head_data.log_type; - let description = head_data.description; - let activity_log = electoral_log.try_into()?; - info!("activity_log = {activity_log:?}"); - let activity_log = ActivityLogRow { - event_type, - log_type, - description, - ..activity_log - }; - info!("activity_log = {activity_log:?}"); - act_log.push(activity_log); + let mut electoral_log: Vec = vec![]; + let mut client = get_board_client().await?; + let electoral_log_msgs = client + .get_electoral_log_messages_batch(&self.board_name, limit, *offset) + .await + .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; + info!("Format: {:#?}", self.report_format); + for entry in electoral_log_msgs { + match self.report_format { + ReportFormat::PDF => { + act_log.push(entry.try_into()?); + } + ReportFormat::CSV => { + electoral_log.push(entry.clone().try_into()?); + } + } } - let total = electoral_logs.total.aggregate.count; - Ok(UserData { act_log, - electoral_log: elect_logs, + electoral_log, }) } + #[instrument(err, skip_all)] async fn prepare_user_data( &self, _hasura_transaction: &Transaction<'_>, _keycloak_transaction: &Transaction<'_>, ) -> Result { - let mut act_log: Vec = vec![]; - let mut elect_logs: Vec = vec![]; - let mut offset = 0; - let limit = PgConfig::from_env() - .with_context(|| "Error obtaining Pg config from env.")? - .default_sql_batch_size as i64; - - loop { - let electoral_logs: DataList = - list_electoral_log(GetElectoralLogBody { - tenant_id: self.ids.tenant_id.clone(), - election_event_id: self.ids.election_event_id.clone(), - limit: Some(limit), - offset: Some(offset), - filter: None, - order_by: None, - area_ids: None, - only_with_user: None, - election_id: None, - statement_kind: None, - }) - .await - .map_err(|e| anyhow!("Error listing electoral logs: {e:?}"))?; - - let is_empty = electoral_logs.items.is_empty(); - - for electoral_log in electoral_logs.items { - elect_logs.push(electoral_log.clone()); - let head_data = electoral_log - .statement_head_data() - .with_context(|| "Error to get head data.")?; - let event_type = head_data.event_type; - let log_type = head_data.log_type; - let description = head_data.description; - let activity_log = electoral_log.try_into()?; - info!("activity_log = {activity_log:?}"); - let activity_log = ActivityLogRow { - event_type, - log_type, - description, - ..activity_log - }; - info!("activity_log = {activity_log:?}"); - act_log.push(activity_log); - } - - let total = electoral_logs.total.aggregate.count; - if is_empty || offset >= total { - break; - } - - offset += limit; - } - - Ok(UserData { - act_log, - electoral_log: elect_logs, - }) + Err(anyhow!( + "prepare_user_data should not be used for this report type, use prepare_user_data_batch instead" + )) } #[instrument(err, skip_all)] @@ -330,16 +312,10 @@ impl TemplateRenderer for ActivityLogsTemplate { ) .await } else { - // Generate CSV report - // Prepare user data - let user_data = self - .prepare_user_data(hasura_transaction, keycloak_transaction) - .await - .map_err(|e| anyhow!("Error preparing activity logs data into CSV: {e:?}"))?; - - // Generate CSV file using generate_report_data + // Generate CSV file using generate_export_csv_data let name = format!("export-election-event-logs-{}", election_event_id); - let temp_file = generate_report_data(&user_data.act_log, &name) + let temp_file = self + .generate_export_csv_data(&name) .await .map_err(|e| anyhow!("Error generating export data: {e:?}"))?; @@ -413,61 +389,3 @@ impl TemplateRenderer for ActivityLogsTemplate { } } } - -/// Maintains the generate_export_data function as before. -/// This function can be used by other report types that need to generate CSV files. -#[instrument(err, skip(act_log))] -pub async fn generate_report_data(act_log: &[ActivityLogRow], name: &str) -> Result { - // Create a temporary file to write CSV data - let mut temp_file = - generate_temp_file(&name, ".csv").with_context(|| "Error creating named temp file")?; - let mut csv_writer = WriterBuilder::new().from_writer(temp_file.as_file_mut()); - - for item in act_log { - let mut item_clean = item.clone(); - - // Replace newline characters in the message field - item_clean.message = item_clean.message.replace('\n', " ").replace('\r', " "); - // Serialize each item to CSV - csv_writer - .serialize(item_clean) - .map_err(|e| anyhow!("Error serializing to CSV: {e:?}"))?; - } - // Flush and finish writing to the temporary file - csv_writer - .flush() - .map_err(|e| anyhow!("Error flushing CSV writer: {e:?}"))?; - drop(csv_writer); - - Ok(temp_file) -} - -// Export data -#[instrument(err, skip(act_log))] -pub async fn generate_export_data( - act_log: &[ElectoralLogRow], - name: &str, -) -> Result { - // Create a temporary file to write CSV data - let mut temp_file = - generate_temp_file(&name, ".csv").with_context(|| "Error creating named temp file")?; - let mut csv_writer = WriterBuilder::new().from_writer(temp_file.as_file_mut()); - - for item in act_log { - let mut item_clean = item.clone(); - - // Replace newline characters in the message field - item_clean.message = item_clean.message.replace('\n', " ").replace('\r', " "); - // Serialize each item to CSV - csv_writer - .serialize(item_clean) - .map_err(|e| anyhow!("Error serializing to CSV: {e:?}"))?; - } - // Flush and finish writing to the temporary file - csv_writer - .flush() - .map_err(|e| anyhow!("Error flushing CSV writer: {e:?}"))?; - drop(csv_writer); - - Ok(temp_file) -} diff --git a/packages/windmill/src/services/reports/template_renderer.rs b/packages/windmill/src/services/reports/template_renderer.rs index d57bb376975..a8b931007b3 100644 --- a/packages/windmill/src/services/reports/template_renderer.rs +++ b/packages/windmill/src/services/reports/template_renderer.rs @@ -111,13 +111,16 @@ pub trait TemplateRenderer: Debug { /// or from other place than the reports TAB. fn get_initial_template_alias(&self) -> Option; - async fn count_items(&self, hasura_transaction: &Transaction<'_>) -> Result> { + async fn count_items( + &self, + hasura_transaction: Option<&Transaction<'_>>, + ) -> Result> { Ok(None) } async fn prepare_user_data_batch( &self, - hasura_transaction: &Transaction<'_>, - keycloak_transaction: &Transaction<'_>, + hasura_transaction: Option<&Transaction<'_>>, + keycloak_transaction: Option<&Transaction<'_>>, offset: &mut i64, limit: i64, ) -> Result { @@ -384,9 +387,14 @@ pub trait TemplateRenderer: Debug { } else { if let (Some(o), Some(l)) = (offset, limit) { info!("Batched processing: offset = {o}, limit = {l}"); - self.prepare_user_data_batch(hasura_transaction, keycloak_transaction, o, l) - .await - .map_err(|e| anyhow!("Error preparing batched user data: {e:?}"))? + self.prepare_user_data_batch( + Some(hasura_transaction), + Some(keycloak_transaction), + o, + l, + ) + .await + .map_err(|e| anyhow!("Error preparing batched user data: {e:?}"))? } else { self.prepare_user_data(hasura_transaction, keycloak_transaction) .await @@ -499,7 +507,10 @@ pub trait TemplateRenderer: Debug { anyhow!("Error providing the user template and extra config: {e:?}") })?; - let items_count = self.count_items(&hasura_transaction).await?.unwrap_or(0); + let items_count = self + .count_items(Some(&hasura_transaction)) + .await? + .unwrap_or(0); let report_options = ext_cfg.report_options.clone(); let per_report_limit = report_options .max_items_per_report diff --git a/packages/windmill/src/tasks/activity_logs_report.rs b/packages/windmill/src/tasks/activity_logs_report.rs index 2571310262a..f0a2ea1f632 100644 --- a/packages/windmill/src/tasks/activity_logs_report.rs +++ b/packages/windmill/src/tasks/activity_logs_report.rs @@ -19,7 +19,7 @@ use crate::{ use anyhow::{anyhow, Context}; use celery::error::TaskError; use deadpool_postgres::Client as DbClient; -use tracing::instrument; +use tracing::{info, instrument}; #[instrument(err)] #[wrap_map_err::wrap_map_err(TaskError)] @@ -68,6 +68,7 @@ pub async fn generate_activity_logs_report( format, ); + info!("Generating activity logs report"); let _ = report .execute_report( &document_id, diff --git a/packages/windmill/src/tasks/electoral_log.rs b/packages/windmill/src/tasks/electoral_log.rs index 790a4f232ca..2ec0a862740 100644 --- a/packages/windmill/src/tasks/electoral_log.rs +++ b/packages/windmill/src/tasks/electoral_log.rs @@ -15,13 +15,13 @@ use crate::types::error::{Error, Result}; use anyhow::{anyhow, Context}; use celery::error::TaskError; use deadpool_postgres::Client as DbClient; -use electoral_log::client::board_client::ElectoralLogMessage; +use electoral_log::client::types::ElectoralLogMessage; use immudb_rs::TxMode; use sequent_core::serialization::deserialize_with_path::deserialize_str; use sequent_core::services::keycloak::get_event_realm; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use tracing::{event, info, instrument}; +use tracing::{info, instrument}; use lapin::{ options::{BasicAckOptions, BasicGetOptions, QueueDeclareOptions}, diff --git a/packages/windmill/src/tasks/generate_report.rs b/packages/windmill/src/tasks/generate_report.rs index 5eb0e0583a7..8e2113e3702 100644 --- a/packages/windmill/src/tasks/generate_report.rs +++ b/packages/windmill/src/tasks/generate_report.rs @@ -114,6 +114,7 @@ pub async fn generate_report( .await?; }; } + info!("To generate report type: {report_type_str}"); match ReportType::from_str(&report_type_str) { Ok(ReportType::INITIALIZATION_REPORT) => { let report = InitializationTemplate::new(ids); diff --git a/packages/windmill/src/types/resources.rs b/packages/windmill/src/types/resources.rs index 6974d1c752d..dd909aa5945 100644 --- a/packages/windmill/src/types/resources.rs +++ b/packages/windmill/src/types/resources.rs @@ -7,31 +7,31 @@ use immudb_rs::{sql_value::Value, Client, NamedParam, Row, SqlValue}; use serde::{Deserialize, Serialize}; use strum_macros::{Display, EnumString}; -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Default)] pub struct Aggregate { pub count: i64, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Default)] pub struct TotalAggregate { pub aggregate: Aggregate, } -// Enumeration for the valid order directions -#[derive(Debug, Deserialize, EnumString, Display, Clone)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum OrderDirection { - Asc, - Desc, -} - #[derive(Serialize, Deserialize, Debug)] pub struct DataList { pub items: Vec, pub total: TotalAggregate, } +impl Default for DataList { + fn default() -> Self { + DataList { + items: vec![], + total: TotalAggregate::default(), + } + } +} + impl TryFrom<&Row> for Aggregate { type Error = anyhow::Error; From 4308855fbda0b61b9e237c0de2272fa8c827bb05 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Wed, 10 Dec 2025 20:33:59 +0000 Subject: [PATCH 02/25] wip --- hasura/metadata/actions.graphql | 12 ------ hasura/metadata/actions.yaml | 14 ------- packages/Cargo.lock | 1 + .../electoral-log/src/client/board_client.rs | 7 ---- .../electoral-log/src/messages/message.rs | 1 - .../src/services/ceremonies/tally_ceremony.rs | 2 - .../windmill/src/services/electoral_log.rs | 37 +++++++++++++------ .../services/export/export_election_event.rs | 13 ++----- .../src/services/reports/activity_log.rs | 6 ++- 9 files changed, 35 insertions(+), 58 deletions(-) diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index d593c537163..05079f9ad08 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -460,18 +460,6 @@ type Query { ): DataListPgAudit } -type Query { - list_cast_vote_messages( - tenant_id: String! - election_event_id: String! - election_id: String - ballot_id: String! - limit: Int - offset: Int - order_by: ElectoralLogOrderBy - ): ListCastVoteMessagesOutput -} - type Query { list_keys_ceremony(election_event_id: String!): ListKeysCeremonyOutput } diff --git a/hasura/metadata/actions.yaml b/hasura/metadata/actions.yaml index 7333b78d445..1c5b70adfa3 100644 --- a/hasura/metadata/actions.yaml +++ b/hasura/metadata/actions.yaml @@ -935,20 +935,6 @@ actions: permissions: - role: cloudflare-write - role: admin-user - - name: list_cast_vote_messages - definition: - kind: synchronous - handler: http://{{HARVEST_DOMAIN}}/voting-portal/immudb/list-cast-vote-messages - forward_client_headers: true - request_transform: - body: - action: transform - template: "{{$body.input}}" - template_engine: Kriti - version: 2 - permissions: - - role: user - comment: List electoral log entries of statement_kind CastVote - name: listElectoralLog definition: kind: "" diff --git a/packages/Cargo.lock b/packages/Cargo.lock index 523866062a4..fc010fee214 100644 --- a/packages/Cargo.lock +++ b/packages/Cargo.lock @@ -2916,6 +2916,7 @@ version = "0.1.0" dependencies = [ "anyhow", "borsh", + "chrono", "clap", "hex", "immudb-rs", diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index 1f04fbc6196..d9fb6cd8f13 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -809,13 +809,6 @@ impl BoardClient { /// Creates the requested immudb database, only if it doesn't exist. It also creates /// the requested tables and indexes if they don't exist. - async fn upsert_database( - &mut self, - database_name: &str, - tables: &str, - indexes: &[String], - ) -> Result<()> { - /// the requested tables and indexes if they don't exist. async fn upsert_database( &mut self, database_name: &str, diff --git a/packages/electoral-log/src/messages/message.rs b/packages/electoral-log/src/messages/message.rs index 3468c8b9bb1..871bf04d012 100644 --- a/packages/electoral-log/src/messages/message.rs +++ b/packages/electoral-log/src/messages/message.rs @@ -7,7 +7,6 @@ use anyhow::Result; use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; -use strand::hash::STRAND_HASH_LENGTH_BYTES; use strand::hash::STRAND_HASH_LENGTH_BYTES; use strand::serialization::StrandSerialize; use strand::signature::StrandSignature; diff --git a/packages/windmill/src/services/ceremonies/tally_ceremony.rs b/packages/windmill/src/services/ceremonies/tally_ceremony.rs index 4fb6a59b6ba..7ad88762b59 100644 --- a/packages/windmill/src/services/ceremonies/tally_ceremony.rs +++ b/packages/windmill/src/services/ceremonies/tally_ceremony.rs @@ -559,9 +559,7 @@ pub async fn update_tally_ceremony( if new_execution_status == TallyExecutionStatus::IN_PROGRESS { let tally_elections_ids = tally_session.election_ids.clone(); - let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; - // Save this in the electoral log let board_name: String = get_event_board(&tenant_id, &election_event_id, &slug); let electoral_log = ElectoralLog::for_admin_user( diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index 2ae348d72bf..096e21d6407 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -12,24 +12,20 @@ use crate::services::vault; use crate::tasks::electoral_log::{ enqueue_electoral_log_event, LogEventInput, INTERNAL_MESSAGE_TYPE, }; -use crate::types::resources::{Aggregate, DataList, OrderDirection, TotalAggregate}; +use crate::types::resources::{Aggregate, DataList, TotalAggregate}; use anyhow::{anyhow, ensure, Context, Result}; use b3::messages::message::Signer; use base64::engine::general_purpose; use base64::Engine; use deadpool_postgres::Transaction; -use electoral_log::client::types::*; use electoral_log::assign_value; +use electoral_log::client::types::*; use electoral_log::messages::message::{Message, SigningData}; use electoral_log::messages::newtypes::*; use electoral_log::messages::statement::{StatementBody, StatementType}; -use electoral_log::{ - ElectoralLogMessage, ElectoralLogVarCharColumn, SqlCompOperators, WhereClauseBTreeMap, -}; use immudb_rs::{sql_value::Value, Client, NamedParam, Row, TxMode}; use rust_decimal::prelude::ToPrimitive; use sequent_core::serialization::deserialize_with_path::{deserialize_str, deserialize_value}; -use sequent_core::util::retry::retry_with_exponential_backoff; use sequent_core::services::date::ISO8601; use sequent_core::util::retry::retry_with_exponential_backoff; use serde::{Deserialize, Serialize}; @@ -904,7 +900,12 @@ impl CastVoteEntry { #[instrument(err)] pub async fn list_electoral_log(input: GetElectoralLogBody) -> Result> { let mut client = get_board_client().await?; - let board_name = get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; + let board_name = get_event_board( + input.tenant_id.as_str(), + input.election_event_id.as_str(), + &slug, + ); info!("database name = {board_name}"); let cols_match_select = input.as_where_clause_map()?; let order_by = input.order_by.clone(); @@ -1082,8 +1083,12 @@ pub async fn list_cast_vote_messages_and_count( ), async { let mut client = get_board_client().await?; - let board_name = - get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; + let board_name = get_event_board( + input.tenant_id.as_str(), + input.election_event_id.as_str(), + &slug, + ); info!("database name = {board_name}"); let total: usize = client .count_electoral_log_messages(&board_name, Some(cols_match_count)) @@ -1112,7 +1117,12 @@ pub async fn list_cast_vote_messages( // The limits are used to cut the output after filtering the ballot id. // Because ballot_id cannot be filtered at SQL level the sql limit is constant let output_limit: i64 = input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64); - let board_name = get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; + let board_name = get_event_board( + input.tenant_id.as_str(), + input.election_event_id.as_str(), + &slug, + ); info!("database name = {board_name}"); let order_by = input.order_by.clone(); @@ -1167,7 +1177,12 @@ pub async fn list_cast_vote_messages( #[instrument(err)] pub async fn count_electoral_log(input: GetElectoralLogBody) -> Result { let mut client = get_board_client().await?; - let board_name = get_event_board(input.tenant_id.as_str(), input.election_event_id.as_str()); + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; + let board_name = get_event_board( + input.tenant_id.as_str(), + input.election_event_id.as_str(), + &slug, + ); info!("database name = {board_name}"); let cols_match_count = input.as_where_clause_map()?; let total = client diff --git a/packages/windmill/src/services/export/export_election_event.rs b/packages/windmill/src/services/export/export_election_event.rs index d8cf25f297f..2669160a857 100644 --- a/packages/windmill/src/services/export/export_election_event.rs +++ b/packages/windmill/src/services/export/export_election_event.rs @@ -357,17 +357,10 @@ pub async fn process_export_zip( ReportFormat::CSV, // Assuming CSV format for this export ); - // Prepare user data - let user_data = activity_logs_template - .prepare_user_data(&hasura_transaction, &hasura_transaction) + let temp_activity_logs_file = activity_logs_template + .generate_export_csv_data(&activity_logs_filename) .await - .map_err(|e| anyhow!("Error preparing activity logs data: {e:?}"))?; - - // Generate the CSV file using generate_export_data - let temp_activity_logs_file = - activity_log::generate_export_data(&user_data.electoral_log, &activity_logs_filename) - .await - .map_err(|e| anyhow!("Error generating export data: {e:?}"))?; + .map_err(|e| anyhow!("Error generating export data: {e:?}"))?; zip_writer .start_file(&activity_logs_filename, options) diff --git a/packages/windmill/src/services/reports/activity_log.rs b/packages/windmill/src/services/reports/activity_log.rs index bc8cd2eec5b..17281f127e8 100644 --- a/packages/windmill/src/services/reports/activity_log.rs +++ b/packages/windmill/src/services/reports/activity_log.rs @@ -120,7 +120,11 @@ pub struct ActivityLogsTemplate { impl ActivityLogsTemplate { pub fn new(ids: ReportOrigins, report_format: ReportFormat) -> Self { - let board_name = get_event_board(ids.tenant_id.as_str(), ids.election_event_id.as_str()); + let board_name = get_event_board( + ids.tenant_id.as_str(), + ids.election_event_id.as_str(), + &slug, + ); ActivityLogsTemplate { ids, report_format, From c440511f5e3b99ed6507126ff6879d6f9858f034 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Wed, 10 Dec 2025 20:45:10 +0000 Subject: [PATCH 03/25] wip --- .../windmill/src/services/electoral_log.rs | 218 +++++++++--------- .../src/services/reports/activity_log.rs | 1 + 2 files changed, 110 insertions(+), 109 deletions(-) diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index 096e21d6407..b2b5cefc008 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -942,118 +942,118 @@ pub async fn list_electoral_log(input: GetElectoralLogBody) -> Result (WhereClauseBTreeMap, WhereClauseBTreeMap) { - let cols_match_count = BTreeMap::from([ - ( - ElectoralLogVarCharColumn::StatementKind, - (SqlCompOperators::Equal, StatementType::CastVote.to_string()), - ), - ( - ElectoralLogVarCharColumn::ElectionId, - (SqlCompOperators::Equal, election_id.to_string()), - ), - ]); - let mut cols_match_select = cols_match_count.clone(); - // Restrict the SQL query to user_id and ballot_id in case of filtering - if !ballot_id_filter.is_empty() { - cols_match_select.insert( - ElectoralLogVarCharColumn::UserId, - (SqlCompOperators::Equal, user_id.to_string()), - ); - cols_match_select.insert( - ElectoralLogVarCharColumn::BallotId, - (SqlCompOperators::Like, ballot_id_filter.to_string()), - ); - } - - (cols_match_count, cols_match_select) -} +// #[instrument] +// pub fn get_cols_match_count_and_select( +// election_id: &str, +// user_id: &str, +// ballot_id_filter: &str, +// ) -> (WhereClauseBTreeMap, WhereClauseBTreeMap) { +// let cols_match_count = BTreeMap::from([ +// ( +// ElectoralLogVarCharColumn::StatementKind, +// (SqlCompOperators::Equal, StatementType::CastVote.to_string()), +// ), +// ( +// ElectoralLogVarCharColumn::ElectionId, +// (SqlCompOperators::Equal, election_id.to_string()), +// ), +// ]); +// let mut cols_match_select = cols_match_count.clone(); +// // Restrict the SQL query to user_id and ballot_id in case of filtering +// if !ballot_id_filter.is_empty() { +// cols_match_select.insert( +// ElectoralLogVarCharColumn::UserId, +// SqlCompOperators::Equal(user_id.to_string()), +// ); +// cols_match_select.insert( +// ElectoralLogVarCharColumn::BallotId, +// SqlCompOperators::Like(ballot_id_filter.to_string()), +// ); +// } + +// (cols_match_count, cols_match_select) +// } /// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input /// ballot_id_filter is restricted to be an even number of characters, so that can be converted /// to a byte array -#[instrument(err)] -pub async fn list_cast_vote_messages_theirs( - input: GetElectoralLogBody, - ballot_id_filter: &str, - user_id: &str, - username: &str, -) -> Result { - ensure!( - ballot_id_filter.chars().count() % 2 == 0 && ballot_id_filter.is_ascii(), - "Incorrect ballot_id, the length must be an even number of characters" - ); - // The limits are used to cut the output after filtering the ballot id. - // Because ballot_id cannot be filtered at SQL level the sql limit is constant - let output_limit: i64 = input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64); - let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; - let board_name = get_event_board( - input.tenant_id.as_str(), - input.election_event_id.as_str(), - &slug, - ); - info!("database name = {board_name}"); - let order_by = input.order_by.clone(); - let election_id = input.election_id.clone().unwrap_or_default(); - - let limit: i64 = match ballot_id_filter.is_empty() { - false => IMMUDB_ROWS_LIMIT as i64, // When there is a filter, need to fetch all entries by batches. - true => input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), - }; - let mut offset: i64 = input.offset.unwrap_or(0); - let mut list: Vec = Vec::with_capacity(MAX_ROWS_PER_PAGE); // Filtered messages. - let (cols_match_count, cols_match_select) = - get_cols_match_count_and_select(&election_id, user_id, ballot_id_filter); - let mut client = get_board_client().await?; - let total = client - .count_electoral_log_messages(&board_name, Some(cols_match_count)) - .await? - .to_u64() - .unwrap_or(0) as usize; - let mut filter_matched = false; // Exit at the first match if the filter is not empty - while (list.len() as i64) < output_limit && (offset < total as i64) && !filter_matched { - let electoral_log_messages = client - .get_electoral_log_messages_filtered( - &board_name, - Some(cols_match_select.clone()), - None, - None, - Some(limit), - Some(offset), - order_by.clone(), - ) - .await - .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; - - let t_entries = electoral_log_messages.len(); - info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}, total: {total}"); - for message in electoral_log_messages.iter() { - match CastVoteEntry::from_elog_message(&message)? { - Some(entry) if !ballot_id_filter.is_empty() => { - // If there is filter exit at the first match - filter_matched = true; - list.push(entry); - } - Some(entry) => { - // Add all the entries till the limit, when there is no filter - list.push(entry); - } - None => {} - } - if (list.len() as i64) >= output_limit || filter_matched { - break; - } - } - offset += limit; - } - - Ok(CastVoteMessagesOutput { list, total }) -} +// #[instrument(err)] +// pub async fn list_cast_vote_messages_theirs( +// input: GetElectoralLogBody, +// ballot_id_filter: &str, +// user_id: &str, +// username: &str, +// ) -> Result { +// ensure!( +// ballot_id_filter.chars().count() % 2 == 0 && ballot_id_filter.is_ascii(), +// "Incorrect ballot_id, the length must be an even number of characters" +// ); +// // The limits are used to cut the output after filtering the ballot id. +// // Because ballot_id cannot be filtered at SQL level the sql limit is constant +// let output_limit: i64 = input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64); +// let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; +// let board_name = get_event_board( +// input.tenant_id.as_str(), +// input.election_event_id.as_str(), +// &slug, +// ); +// info!("database name = {board_name}"); +// let order_by = input.order_by.clone(); +// let election_id = input.election_id.clone().unwrap_or_default(); + +// let limit: i64 = match ballot_id_filter.is_empty() { +// false => IMMUDB_ROWS_LIMIT as i64, // When there is a filter, need to fetch all entries by batches. +// true => input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), +// }; +// let mut offset: i64 = input.offset.unwrap_or(0); +// let mut list: Vec = Vec::with_capacity(MAX_ROWS_PER_PAGE); // Filtered messages. +// let (cols_match_count, cols_match_select) = +// get_cols_match_count_and_select(&election_id, user_id, ballot_id_filter); +// let mut client = get_board_client().await?; +// let total = client +// .count_electoral_log_messages(&board_name, Some(cols_match_count)) +// .await? +// .to_u64() +// .unwrap_or(0) as usize; +// let mut filter_matched = false; // Exit at the first match if the filter is not empty +// while (list.len() as i64) < output_limit && (offset < total as i64) && !filter_matched { +// let electoral_log_messages = client +// .get_electoral_log_messages_filtered( +// &board_name, +// Some(cols_match_select.clone()), +// None, +// None, +// Some(limit), +// Some(offset), +// order_by.clone(), +// ) +// .await +// .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; + +// let t_entries = electoral_log_messages.len(); +// info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}, total: {total}"); +// for message in electoral_log_messages.iter() { +// match CastVoteEntry::from_elog_message(&message)? { +// Some(entry) if !ballot_id_filter.is_empty() => { +// // If there is filter exit at the first match +// filter_matched = true; +// list.push(entry); +// } +// Some(entry) => { +// // Add all the entries till the limit, when there is no filter +// list.push(entry); +// } +// None => {} +// } +// if (list.len() as i64) >= output_limit || filter_matched { +// break; +// } +// } +// offset += limit; +// } + +// Ok(CastVoteMessagesOutput { list, total }) +// } /// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input. /// ballot_id_filter is restricted to be an even number of characters, so thatnit can be converted @@ -1152,7 +1152,7 @@ pub async fn list_cast_vote_messages( let t_entries = electoral_log_messages.len(); info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}"); for message in electoral_log_messages.iter() { - match CastVoteEntry::from_elog_message(&message, username)? { + match CastVoteEntry::from_elog_message(&message)? { Some(entry) if !ballot_id_filter.is_empty() => { // If there is filter exit at the first match exit = true; diff --git a/packages/windmill/src/services/reports/activity_log.rs b/packages/windmill/src/services/reports/activity_log.rs index 17281f127e8..72d819f0cef 100644 --- a/packages/windmill/src/services/reports/activity_log.rs +++ b/packages/windmill/src/services/reports/activity_log.rs @@ -120,6 +120,7 @@ pub struct ActivityLogsTemplate { impl ActivityLogsTemplate { pub fn new(ids: ReportOrigins, report_format: ReportFormat) -> Self { + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; let board_name = get_event_board( ids.tenant_id.as_str(), ids.election_event_id.as_str(), From ce0992acfc2e829fc606977bf165f63304440de2 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Thu, 11 Dec 2025 07:36:23 +0000 Subject: [PATCH 04/25] Fixes --- .../src/services/reports/activity_log.rs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/windmill/src/services/reports/activity_log.rs b/packages/windmill/src/services/reports/activity_log.rs index 72d819f0cef..abca138c659 100644 --- a/packages/windmill/src/services/reports/activity_log.rs +++ b/packages/windmill/src/services/reports/activity_log.rs @@ -115,22 +115,11 @@ pub struct SystemData { pub struct ActivityLogsTemplate { ids: ReportOrigins, report_format: ReportFormat, - board_name: String, } impl ActivityLogsTemplate { pub fn new(ids: ReportOrigins, report_format: ReportFormat) -> Self { - let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; - let board_name = get_event_board( - ids.tenant_id.as_str(), - ids.election_event_id.as_str(), - &slug, - ); - ActivityLogsTemplate { - ids, - report_format, - board_name, - } + ActivityLogsTemplate { ids, report_format } } // Export data @@ -222,8 +211,14 @@ impl TemplateRenderer for ActivityLogsTemplate { _hasura_transaction: Option<&Transaction<'_>>, ) -> Result> { let mut client = get_board_client().await?; + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; + let board_name = get_event_board( + self.ids.tenant_id.as_str(), + self.ids.election_event_id.as_str(), + &slug, + ); let total = client - .count_electoral_log_messages(&self.board_name, None) + .count_electoral_log_messages(&board_name, None) .await .map_err(|e| anyhow!("Error counting electoral log messages: {e:?}"))?; Ok(Some(total)) @@ -240,8 +235,14 @@ impl TemplateRenderer for ActivityLogsTemplate { let mut act_log: Vec = vec![]; let mut electoral_log: Vec = vec![]; let mut client = get_board_client().await?; + let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; + let board_name = get_event_board( + self.ids.tenant_id.as_str(), + self.ids.election_event_id.as_str(), + &slug, + ); let electoral_log_msgs = client - .get_electoral_log_messages_batch(&self.board_name, limit, *offset) + .get_electoral_log_messages_batch(&board_name, limit, *offset) .await .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; info!("Format: {:#?}", self.report_format); From df27f03be87143e4c940fe0e9c49e69b8967b133 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Thu, 11 Dec 2025 10:44:57 +0000 Subject: [PATCH 05/25] Fix duplications --- .../electoral-log/src/client/board_client.rs | 11 -- .../windmill/src/services/electoral_log.rs | 119 +----------------- 2 files changed, 5 insertions(+), 125 deletions(-) diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index d9fb6cd8f13..d5898eab36e 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -763,14 +763,12 @@ impl BoardClient { pub async fn upsert_electoral_log_db(&mut self, board_dbname: &str) -> Result<()> { let sql = format!( r#" - CREATE TABLE IF NOT EXISTS {ELECTORAL_LOG_TABLE} ( CREATE TABLE IF NOT EXISTS {ELECTORAL_LOG_TABLE} ( id INTEGER AUTO_INCREMENT, created TIMESTAMP, sender_pk VARCHAR, statement_timestamp TIMESTAMP, statement_kind VARCHAR[{STATEMENT_KIND_VARCHAR_LENGTH}], - statement_kind VARCHAR[{STATEMENT_KIND_VARCHAR_LENGTH}], message BLOB, version VARCHAR, user_id_key VARCHAR[{ID_KEY_VARCHAR_LENGTH}], @@ -779,9 +777,6 @@ impl BoardClient { election_id VARCHAR[{ID_VARCHAR_LENGTH}], area_id VARCHAR[{ID_VARCHAR_LENGTH}], ballot_id VARCHAR[{BALLOT_ID_VARCHAR_LENGTH}], - election_id VARCHAR[{ID_VARCHAR_LENGTH}], - area_id VARCHAR[{ID_VARCHAR_LENGTH}], - ballot_id VARCHAR[{BALLOT_ID_VARCHAR_LENGTH}], PRIMARY KEY id ); "# @@ -820,13 +815,11 @@ impl BoardClient { println!("Database not found, creating.."); self.client.create_database(database_name).await?; info!("Database created!"); - info!("Database created!"); }; self.client.use_database(database_name).await?; // List tables and create them if missing if !self.client.has_tables().await? { - info!("no tables! let's create them"); info!("no tables! let's create them"); self.client.sql_exec(&tables, vec![]).await?; } @@ -834,10 +827,6 @@ impl BoardClient { info!("Inserting index..."); self.client.sql_exec(index, vec![]).await?; } - for index in indexes { - info!("Inserting index..."); - self.client.sql_exec(index, vec![]).await?; - } Ok(()) } } diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index b2b5cefc008..999c6d3c3b5 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -942,119 +942,6 @@ pub async fn list_electoral_log(input: GetElectoralLogBody) -> Result (WhereClauseBTreeMap, WhereClauseBTreeMap) { -// let cols_match_count = BTreeMap::from([ -// ( -// ElectoralLogVarCharColumn::StatementKind, -// (SqlCompOperators::Equal, StatementType::CastVote.to_string()), -// ), -// ( -// ElectoralLogVarCharColumn::ElectionId, -// (SqlCompOperators::Equal, election_id.to_string()), -// ), -// ]); -// let mut cols_match_select = cols_match_count.clone(); -// // Restrict the SQL query to user_id and ballot_id in case of filtering -// if !ballot_id_filter.is_empty() { -// cols_match_select.insert( -// ElectoralLogVarCharColumn::UserId, -// SqlCompOperators::Equal(user_id.to_string()), -// ); -// cols_match_select.insert( -// ElectoralLogVarCharColumn::BallotId, -// SqlCompOperators::Like(ballot_id_filter.to_string()), -// ); -// } - -// (cols_match_count, cols_match_select) -// } - -/// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input -/// ballot_id_filter is restricted to be an even number of characters, so that can be converted -/// to a byte array -// #[instrument(err)] -// pub async fn list_cast_vote_messages_theirs( -// input: GetElectoralLogBody, -// ballot_id_filter: &str, -// user_id: &str, -// username: &str, -// ) -> Result { -// ensure!( -// ballot_id_filter.chars().count() % 2 == 0 && ballot_id_filter.is_ascii(), -// "Incorrect ballot_id, the length must be an even number of characters" -// ); -// // The limits are used to cut the output after filtering the ballot id. -// // Because ballot_id cannot be filtered at SQL level the sql limit is constant -// let output_limit: i64 = input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64); -// let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; -// let board_name = get_event_board( -// input.tenant_id.as_str(), -// input.election_event_id.as_str(), -// &slug, -// ); -// info!("database name = {board_name}"); -// let order_by = input.order_by.clone(); -// let election_id = input.election_id.clone().unwrap_or_default(); - -// let limit: i64 = match ballot_id_filter.is_empty() { -// false => IMMUDB_ROWS_LIMIT as i64, // When there is a filter, need to fetch all entries by batches. -// true => input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), -// }; -// let mut offset: i64 = input.offset.unwrap_or(0); -// let mut list: Vec = Vec::with_capacity(MAX_ROWS_PER_PAGE); // Filtered messages. -// let (cols_match_count, cols_match_select) = -// get_cols_match_count_and_select(&election_id, user_id, ballot_id_filter); -// let mut client = get_board_client().await?; -// let total = client -// .count_electoral_log_messages(&board_name, Some(cols_match_count)) -// .await? -// .to_u64() -// .unwrap_or(0) as usize; -// let mut filter_matched = false; // Exit at the first match if the filter is not empty -// while (list.len() as i64) < output_limit && (offset < total as i64) && !filter_matched { -// let electoral_log_messages = client -// .get_electoral_log_messages_filtered( -// &board_name, -// Some(cols_match_select.clone()), -// None, -// None, -// Some(limit), -// Some(offset), -// order_by.clone(), -// ) -// .await -// .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; - -// let t_entries = electoral_log_messages.len(); -// info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}, total: {total}"); -// for message in electoral_log_messages.iter() { -// match CastVoteEntry::from_elog_message(&message)? { -// Some(entry) if !ballot_id_filter.is_empty() => { -// // If there is filter exit at the first match -// filter_matched = true; -// list.push(entry); -// } -// Some(entry) => { -// // Add all the entries till the limit, when there is no filter -// list.push(entry); -// } -// None => {} -// } -// if (list.len() as i64) >= output_limit || filter_matched { -// break; -// } -// } -// offset += limit; -// } - -// Ok(CastVoteMessagesOutput { list, total }) -// } - /// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input. /// ballot_id_filter is restricted to be an even number of characters, so thatnit can be converted /// to a byte array. @@ -1106,6 +993,10 @@ pub async fn list_cast_vote_messages_and_count( Ok(data) } + +/// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input +/// ballot_id_filter is restricted to be an even number of characters, so that can be converted +/// to a byte array #[instrument(err)] pub async fn list_cast_vote_messages( input: GetElectoralLogBody, @@ -1134,7 +1025,7 @@ pub async fn list_cast_vote_messages( let mut list: Vec = Vec::with_capacity(MAX_ROWS_PER_PAGE); // Filtered messages. let mut client = get_board_client().await?; - let mut exit = false; // Exit at the first match if the filter is not empty or when the query returns 0 entries + let mut exit = false; // Exit at the first match if the filter by ballot_id is not empty or when the query returns 0 entries while (list.len() as i64) < output_limit && !exit { let electoral_log_messages = client .get_electoral_log_messages_filtered( From fbf764ba58e6c851367783bafdbdfb92224d0c20 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Fri, 12 Dec 2025 08:15:39 +0000 Subject: [PATCH 06/25] Fix --- .../windmill/src/services/electoral_log.rs | 85 +++++++------------ 1 file changed, 30 insertions(+), 55 deletions(-) diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index 999c6d3c3b5..ad23c23d175 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -880,20 +880,20 @@ pub struct CastVoteMessagesOutput { } impl CastVoteEntry { - pub fn from_elog_message(entry: &ElectoralLogMessage) -> Result, anyhow::Error> { + pub fn from_elog_message(entry: &ElectoralLogMessage) -> Result { let ballot_id = entry.ballot_id.clone().unwrap_or_default(); let username = entry.username.clone(); let message: &Message = &Message::strand_deserialize(&entry.message) .map_err(|err| anyhow!("Failed to deserialize message: {:?}", err))?; let message = Some(message.to_string()); - Ok(Some(CastVoteEntry { + Ok(CastVoteEntry { statement_timestamp: entry.statement_timestamp, statement_kind: StatementType::CastVote.to_string(), ballot_id, username, message, - })) + }) } } @@ -960,7 +960,7 @@ pub async fn list_cast_vote_messages_and_count( let (cols_match_count, cols_match_select) = input.as_cast_vote_count_and_select_clauses(&election_id, user_id, ballot_id_filter); - let (data_res, count_res) = tokio::join!( + let (entries_res, count_res) = tokio::join!( list_cast_vote_messages( input.clone(), ballot_id_filter, @@ -986,14 +986,12 @@ pub async fn list_cast_vote_messages_and_count( } ); - let mut data = data_res.map_err(|e| anyhow!("Eror listing electoral log: {e:?}"))?; - data.total = + let list = entries_res.map_err(|e| anyhow!("Error listing electoral log: {e:?}"))?; + let total = count_res.map_err(|e: anyhow::Error| anyhow!("Error counting electoral log: {e:?}"))?; - - Ok(data) + Ok(CastVoteMessagesOutput { list, total }) } - /// Returns the entries for statement_kind = "CastVote" which ballot_id matches the input /// ballot_id_filter is restricted to be an even number of characters, so that can be converted /// to a byte array @@ -1004,10 +1002,7 @@ pub async fn list_cast_vote_messages( user_id: &str, username: &str, cols_match_select: WhereClauseOrdMap, -) -> Result { - // The limits are used to cut the output after filtering the ballot id. - // Because ballot_id cannot be filtered at SQL level the sql limit is constant - let output_limit: i64 = input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64); +) -> Result> { let slug = std::env::var("ENV_SLUG").with_context(|| "missing env var ENV_SLUG")?; let board_name = get_event_board( input.tenant_id.as_str(), @@ -1016,53 +1011,33 @@ pub async fn list_cast_vote_messages( ); info!("database name = {board_name}"); let order_by = input.order_by.clone(); - + let mut offset: i64 = input.offset.unwrap_or(0); let limit: i64 = match ballot_id_filter.is_empty() { - false => IMMUDB_ROWS_LIMIT as i64, // When there is a filter, need to fetch all entries by batches. + false => 1, // When there is a filter, limit to 1 result because ballot_id is unique true => input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), }; - let mut offset: i64 = input.offset.unwrap_or(0); + let mut client = get_board_client().await?; + let electoral_log_messages = client + .get_electoral_log_messages_filtered( + &board_name, + Some(cols_match_select.clone()), + None, + None, + Some(limit), + Some(offset), + order_by.clone(), + ) + .await + .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; + let t_entries = electoral_log_messages.len(); + info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}"); let mut list: Vec = Vec::with_capacity(MAX_ROWS_PER_PAGE); // Filtered messages. + list = electoral_log_messages + .iter() + .map(|message| CastVoteEntry::from_elog_message(message)) + .collect::>>()?; - let mut client = get_board_client().await?; - let mut exit = false; // Exit at the first match if the filter by ballot_id is not empty or when the query returns 0 entries - while (list.len() as i64) < output_limit && !exit { - let electoral_log_messages = client - .get_electoral_log_messages_filtered( - &board_name, - Some(cols_match_select.clone()), - None, - None, - Some(limit), - Some(offset), - order_by.clone(), - ) - .await - .map_err(|err| anyhow!("Failed to get filtered messages: {:?}", err))?; - - let t_entries = electoral_log_messages.len(); - info!("Got {t_entries} entries. Offset: {offset}, limit: {limit}"); - for message in electoral_log_messages.iter() { - match CastVoteEntry::from_elog_message(&message)? { - Some(entry) if !ballot_id_filter.is_empty() => { - // If there is filter exit at the first match - exit = true; - list.push(entry); - } - Some(entry) => { - // Add all the entries till the limit, when there is no filter - list.push(entry); - } - None => {} - } - if (list.len() as i64) >= output_limit || exit { - break; - } - } - exit = exit || t_entries == 0; - offset += limit; - } - Ok(CastVoteMessagesOutput { list, total: 0 }) + Ok(list) } #[instrument(err)] From f40b1f92c39b4b618fdcf18f3da8cd8cdacb1aba Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Mon, 15 Dec 2025 07:24:14 +0000 Subject: [PATCH 07/25] wip --- hasura/metadata/actions.graphql | 26 +++++++++---------- .../windmill/src/services/electoral_log.rs | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index 05079f9ad08..6b75d67a081 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -428,18 +428,6 @@ type Mutation { ): LimitAccessByCountriesOutput } -type Query { - list_cast_vote_messages( - tenant_id: String! - election_event_id: String! - election_id: String - ballot_id: String! - limit: Int - offset: Int - order_by: ElectoralLogOrderBy - ): ListCastVoteMessagesOutput -} - type Query { listElectoralLog( limit: Int @@ -460,6 +448,18 @@ type Query { ): DataListPgAudit } +type Query { + list_cast_vote_messages( + tenant_id: String! + election_event_id: String! + election_id: String + ballot_id: String! + limit: Int + offset: Int + order_by: ElectoralLogOrderBy + ): ListCastVoteMessagesOutput +} + type Query { list_keys_ceremony(election_event_id: String!): ListKeysCeremonyOutput } @@ -687,10 +687,10 @@ input PgAuditOrderBy { input ElectoralLogFilter { id: String user_id: String - username: String created: String statement_timestamp: String statement_kind: String + username: String } input ElectoralLogOrderBy { diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index ad23c23d175..7f7cf7eb16e 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -883,7 +883,7 @@ impl CastVoteEntry { pub fn from_elog_message(entry: &ElectoralLogMessage) -> Result { let ballot_id = entry.ballot_id.clone().unwrap_or_default(); let username = entry.username.clone(); - let message: &Message = &Message::strand_deserialize(&entry.message) + let message: Message = Message::strand_deserialize(&entry.message) .map_err(|err| anyhow!("Failed to deserialize message: {:?}", err))?; let message = Some(message.to_string()); From bcd40a83b88c349d1e64db10ad1a31e4a2e1e74c Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Mon, 15 Dec 2025 11:19:47 +0000 Subject: [PATCH 08/25] Fix indexes usage --- packages/electoral-log/src/client/types.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/electoral-log/src/client/types.rs b/packages/electoral-log/src/client/types.rs index 8ff7c9c0c27..8e579e06996 100644 --- a/packages/electoral-log/src/client/types.rs +++ b/packages/electoral-log/src/client/types.rs @@ -87,19 +87,22 @@ impl WhereClauseOrdMap { self.0.iter() } - /// If the first element of the map (first column in the where clause) has an index, it will be used. - /// This will match the longest possible index. + /// USE INDEX ON clause for multicolumn indexes. + /// Where clause is longer than the index: the last index matched will be used. + /// Where clause is shorter than the index: No index will be used because it causes errors in unmmudb. + /// Will return the longest possible index to use or None. pub fn to_use_index_clause(&self) -> String { let mut try_index_clause = String::from(""); let mut last_index_clause_match = String::from(""); for (col_name, _) in self.iter() { if try_index_clause.is_empty() { - try_index_clause.push_str(&format!("({col_name}")); // For the contains() is important to mark with '(' the beginning of the index. + try_index_clause.push_str(&format!("({col_name}")); } else { try_index_clause.push_str(&format!(", {col_name}")); } for index in MULTI_COLUMN_INDEXES { - if index.contains(&try_index_clause.as_str()) { + let try_index_clause_closed = format!("{try_index_clause})"); + if index.eq(&try_index_clause_closed) { last_index_clause_match = format!("USE INDEX ON {index}"); } } @@ -220,10 +223,10 @@ impl GetElectoralLogBody { match field { OrderField::Created | OrderField::StatementTimestamp => { let date_time_utc = DateTime::parse_from_rfc3339(&value) - .map_err(|err| anyhow!("{:?}", err))?; + .map_err(|err| anyhow!("Error parsing timestamp: {err:?}"))?; let datetime = date_time_utc.with_timezone(&Utc); let ts: i64 = datetime.timestamp(); - let ts_end: i64 = ts + 60; // Search along that minute, the second is not specified by the front. + let ts_end: i64 = ts + 60; // Search along that minute, the second is not specified by the frontend UI. min_ts = Some(ts); max_ts = Some(ts_end); } From 67f17ff2bec529d9a77fc738b231cc41eb498487 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Mon, 15 Dec 2025 16:08:07 +0000 Subject: [PATCH 09/25] Fix timestamp filter --- .../src/components/ElectoralLogList.tsx | 4 ++ .../electoral-log/src/client/board_client.rs | 53 ++++++++++++------- packages/electoral-log/src/client/types.rs | 2 +- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index 281dac597aa..63915827f00 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -204,11 +204,15 @@ export const ElectoralLogList: React.FC = ({ key={"created"} source={"created"} label={String(t("logsScreen.column.created"))} + inputProps={{step: 1}} + parse={(value) => (value ? new Date(value).toISOString() : value)} />, (value ? new Date(value).toISOString() : value)} />, = @min_ts", min_ts) + ("statement_timestamp >= @min_ts", min_ts) } else { ("", 0) }; let (max_clause, max_clause_value) = if let Some(max_ts) = max_ts { - ("AND statement_timestamp <= @max_ts", max_ts) + ("statement_timestamp <= @max_ts", max_ts) } else { ("", 0) }; - let (where_clause, mut params) = columns_matcher - .clone() - .unwrap_or_default() - .to_where_clause(); let order_by_clauses = if let Some(order_by) = order_by { order_by .iter() @@ -256,18 +257,34 @@ impl BoardClient { }), }); - let where_clauses = - if !where_clause.is_empty() || !min_clause.is_empty() || !max_clause.is_empty() { - format!( - r#" - WHERE {where_clause} - {min_clause} - {max_clause} - "# - ) - } else { - String::from("") - }; + let mut where_clauses = where_clause.clone(); + match (min_clause.is_empty(), where_clauses.is_empty()) { + (true, _) => {} + (false, true) => { + where_clauses.push_str(min_clause); + } + (false, false) => { + // where_clauses is not empty, put the AND + where_clauses.push_str(&format!(" AND {min_clause}")); + } + }; + match (max_clause.is_empty(), where_clauses.is_empty()) { + (true, _) => {} + (false, true) => { + where_clauses.push_str(max_clause); + } + (false, false) => { + // where_clauses is not empty, put the AND + where_clauses.push_str(&format!(" AND {max_clause}")); + } + }; + if !where_clauses.is_empty() { + where_clauses = format!( + r#" + WHERE {where_clauses} + "# + ); + } let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause(); diff --git a/packages/electoral-log/src/client/types.rs b/packages/electoral-log/src/client/types.rs index 8e579e06996..1851ae7229b 100644 --- a/packages/electoral-log/src/client/types.rs +++ b/packages/electoral-log/src/client/types.rs @@ -226,7 +226,7 @@ impl GetElectoralLogBody { .map_err(|err| anyhow!("Error parsing timestamp: {err:?}"))?; let datetime = date_time_utc.with_timezone(&Utc); let ts: i64 = datetime.timestamp(); - let ts_end: i64 = ts + 60; // Search along that minute, the second is not specified by the frontend UI. + let ts_end: i64 = ts + 90_000; // Search along that day. min_ts = Some(ts); max_ts = Some(ts_end); } From 110fd0125a12700e7a265b40626f1f886e435caa Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Mon, 15 Dec 2025 18:01:01 +0000 Subject: [PATCH 10/25] Fix --- packages/windmill/src/services/electoral_log.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/windmill/src/services/electoral_log.rs b/packages/windmill/src/services/electoral_log.rs index 7f7cf7eb16e..8f52b39373f 100644 --- a/packages/windmill/src/services/electoral_log.rs +++ b/packages/windmill/src/services/electoral_log.rs @@ -1011,10 +1011,12 @@ pub async fn list_cast_vote_messages( ); info!("database name = {board_name}"); let order_by = input.order_by.clone(); - let mut offset: i64 = input.offset.unwrap_or(0); - let limit: i64 = match ballot_id_filter.is_empty() { - false => 1, // When there is a filter, limit to 1 result because ballot_id is unique - true => input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), + let (limit, offset) = match ballot_id_filter.is_empty() { + false => (1, 0), // When there is a filter, limit to 1 the result because ballot_id is unique and offset to 0 to scan the whole table + true => ( + input.limit.unwrap_or(MAX_ROWS_PER_PAGE as i64), + input.offset.unwrap_or(0), + ), }; let mut client = get_board_client().await?; let electoral_log_messages = client From 6b290637bd405f4b181d406b90295e87609d4a46 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Mon, 15 Dec 2025 19:14:21 +0000 Subject: [PATCH 11/25] Fix step-cli build --- packages/step-cli/src/commands/export_cast_votes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/step-cli/src/commands/export_cast_votes.rs b/packages/step-cli/src/commands/export_cast_votes.rs index 6d57e2c67c3..8fb9b06ce82 100644 --- a/packages/step-cli/src/commands/export_cast_votes.rs +++ b/packages/step-cli/src/commands/export_cast_votes.rs @@ -15,7 +15,7 @@ use electoral_log::client::types::{ use electoral_log::messages::message::Message; use electoral_log::messages::newtypes::ElectionIdString; use electoral_log::messages::statement::{StatementBody, StatementType}; -use electoral_log::{BoardClient, ElectoralLogVarCharColumn, SqlCompOperators}; +use electoral_log::BoardClient; use sequent_core::encrypt::shorten_hash; use serde::Serialize; use serde_json::Value; From f21a59a2fb138b315ee46c50a97ccf1e1a351d4e Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Wed, 17 Dec 2025 09:48:42 +0000 Subject: [PATCH 12/25] Release notes and UI filter timezone fix. --- .../08-releases/01-release-next/release-next.md | 13 +++++++++++-- .../src/components/ElectoralLogList.tsx | 16 ++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/docusaurus/docs/08-releases/01-release-next/release-next.md b/docs/docusaurus/docs/08-releases/01-release-next/release-next.md index 1759bdf1cdf..9ac2ad14e36 100644 --- a/docs/docusaurus/docs/08-releases/01-release-next/release-next.md +++ b/docs/docusaurus/docs/08-releases/01-release-next/release-next.md @@ -61,6 +61,7 @@ bad merge from main. ## ✨ Instant-runoff Voting (IRV/RCV) System support Support for Instant-runoff elections. Adaptations and implementations were added: + - Velvet tally and make the tally operation configurable. - Admin portal, selectable counting algorithm at contest level, UI tally results and results report. - Voting portal and ballot verifier for preferential order. @@ -106,7 +107,6 @@ Fix showing 'event' or 'election' instead of the actual election event or electi - Issue: [#8426](https://github.com/sequentech/meta/issues/8426) - ## 🐞 Error with tenants and templates in Admin portal. Fixed issues that prevented tenant creation and template creation and deletion @@ -120,8 +120,17 @@ Now tally view checks if it's an automatic ceremony based on only the keys cerem - Issue: [#8472](https://github.com/sequentech/meta/issues/8472) -## ✨ Documentation: Structural Changes +✨ Documentation: Structural Changes Documentation sidebar order has been restructured - Issue: [#8672](https://github.com/sequentech/meta/issues/8672) + +## ✨ Scalable export of electoral/immudb logs + +Support batching queries to export electoral logs into CSV files. +Implementation with explicit ´USE INDEX ON´ to ensure index usage in the immudb queries. +Breaking changes to change immudb the multicolumn indexes and other performance inprovements. +Unify queries to immudb to use always board_client.rs in the electoral-log module. + +- Issue: [#6753](https://github.com/sequentech/meta/issues/6753) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index 63915827f00..d030c26e116 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -204,15 +204,23 @@ export const ElectoralLogList: React.FC = ({ key={"created"} source={"created"} label={String(t("logsScreen.column.created"))} - inputProps={{step: 1}} - parse={(value) => (value ? new Date(value).toISOString() : value)} + slotProps={{ + htmlInput: { + step: 1, // Adds seconds field to the datetime input + } + }} + parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, (value ? new Date(value).toISOString() : value)} + slotProps={{ + htmlInput: { + step: 1, // Adds seconds field to the datetime input + } + }} + parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, Date: Wed, 17 Dec 2025 09:50:42 +0000 Subject: [PATCH 13/25] Format --- packages/admin-portal/src/components/ElectoralLogList.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index d030c26e116..50136feb152 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -207,7 +207,7 @@ export const ElectoralLogList: React.FC = ({ slotProps={{ htmlInput: { step: 1, // Adds seconds field to the datetime input - } + }, }} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, @@ -218,7 +218,7 @@ export const ElectoralLogList: React.FC = ({ slotProps={{ htmlInput: { step: 1, // Adds seconds field to the datetime input - } + }, }} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, From 938c18f07f40ee544591b53d5d8fcc36b439af50 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Wed, 17 Dec 2025 14:29:48 +0000 Subject: [PATCH 14/25] wip --- .../admin-portal/src/components/ElectoralLogList.tsx | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index 50136feb152..281dac597aa 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -204,23 +204,11 @@ export const ElectoralLogList: React.FC = ({ key={"created"} source={"created"} label={String(t("logsScreen.column.created"))} - slotProps={{ - htmlInput: { - step: 1, // Adds seconds field to the datetime input - }, - }} - parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, (value ? new Date(`${value}Z`).toISOString() : value)} />, Date: Thu, 18 Dec 2025 10:46:07 +0100 Subject: [PATCH 15/25] Update release-next.md --- .../docusaurus/docs/08-releases/01-release-next/release-next.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docusaurus/docs/08-releases/01-release-next/release-next.md b/docs/docusaurus/docs/08-releases/01-release-next/release-next.md index b3980d6fd5e..55d8d5d9f44 100644 --- a/docs/docusaurus/docs/08-releases/01-release-next/release-next.md +++ b/docs/docusaurus/docs/08-releases/01-release-next/release-next.md @@ -126,7 +126,7 @@ Now tally view checks if it's an automatic ceremony based on only the keys cerem - Issue: [#8472](https://github.com/sequentech/meta/issues/8472) -✨ Documentation: Structural Changes +## ✨ Documentation: Structural Changes Documentation sidebar order has been restructured From b20800facef9efcac4c5018e576ebf45b01cca72 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Fri, 19 Dec 2025 12:53:51 +0100 Subject: [PATCH 16/25] Rev --- packages/electoral-log/src/client/board_client.rs | 4 ++-- packages/electoral-log/src/client/types.rs | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index c08d91997e3..6e95dce25a5 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -286,7 +286,7 @@ impl BoardClient { ); } - let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause(); + let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause().unwrap_or_default(); self.client.use_database(board_db).await?; let sql = format!( @@ -379,7 +379,7 @@ impl BoardClient { } else { String::from("") }; - let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause(); + let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause().unwrap_or_default(); self.client.use_database(board_db).await?; let count = if use_index_clause.is_empty() && where_clauses.is_empty() && params.is_empty() diff --git a/packages/electoral-log/src/client/types.rs b/packages/electoral-log/src/client/types.rs index 1851ae7229b..f1a3096d278 100644 --- a/packages/electoral-log/src/client/types.rs +++ b/packages/electoral-log/src/client/types.rs @@ -91,9 +91,9 @@ impl WhereClauseOrdMap { /// Where clause is longer than the index: the last index matched will be used. /// Where clause is shorter than the index: No index will be used because it causes errors in unmmudb. /// Will return the longest possible index to use or None. - pub fn to_use_index_clause(&self) -> String { + pub fn to_use_index_clause(&self) -> Option { let mut try_index_clause = String::from(""); - let mut last_index_clause_match = String::from(""); + let mut last_index_clause_match = None; for (col_name, _) in self.iter() { if try_index_clause.is_empty() { try_index_clause.push_str(&format!("({col_name}")); @@ -103,7 +103,7 @@ impl WhereClauseOrdMap { for index in MULTI_COLUMN_INDEXES { let try_index_clause_closed = format!("{try_index_clause})"); if index.eq(&try_index_clause_closed) { - last_index_clause_match = format!("USE INDEX ON {index}"); + last_index_clause_match = Some(format!("USE INDEX ON {index}")); } } } @@ -281,10 +281,9 @@ impl GetElectoralLogBody { if let Some(area_ids) = &self.area_ids { if !area_ids.is_empty() { - // NOTE: `IN` values must be handled later in SQL building, here we just join them cols_match_select.insert( ElectoralLogVarCharColumn::AreaId, - SqlCompOperators::In(area_ids.clone()), // TODO: NullOrIn + SqlCompOperators::In(area_ids.clone()), ); } } From 88ec4eebc9f797bc66dfb34c827a8837b726529d Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Fri, 19 Dec 2025 12:20:28 +0000 Subject: [PATCH 17/25] Format --- packages/electoral-log/src/client/board_client.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index 6e95dce25a5..e9d20f72c1f 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -286,7 +286,10 @@ impl BoardClient { ); } - let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause().unwrap_or_default(); + let use_index_clause = columns_matcher + .unwrap_or_default() + .to_use_index_clause() + .unwrap_or_default(); self.client.use_database(board_db).await?; let sql = format!( @@ -379,7 +382,10 @@ impl BoardClient { } else { String::from("") }; - let use_index_clause = columns_matcher.unwrap_or_default().to_use_index_clause().unwrap_or_default(); + let use_index_clause = columns_matcher + .unwrap_or_default() + .to_use_index_clause() + .unwrap_or_default(); self.client.use_database(board_db).await?; let count = if use_index_clause.is_empty() && where_clauses.is_empty() && params.is_empty() From 0d30b138e24d001d16d12d9f0e46d798b198e2f8 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Fri, 16 Jan 2026 10:10:16 +0000 Subject: [PATCH 18/25] Fix filter by username in tenant realm --- packages/harvest/src/routes/electoral_log.rs | 28 ++++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/packages/harvest/src/routes/electoral_log.rs b/packages/harvest/src/routes/electoral_log.rs index b4a0b0a1d90..9d36af8d80b 100644 --- a/packages/harvest/src/routes/electoral_log.rs +++ b/packages/harvest/src/routes/electoral_log.rs @@ -10,7 +10,7 @@ use electoral_log::client::types::*; use rocket::http::Status; use rocket::serde::json::Json; use sequent_core::services::jwt::JwtClaims; -use sequent_core::services::keycloak::get_event_realm; +use sequent_core::services::keycloak::{get_event_realm, get_tenant_realm}; use sequent_core::types::permissions::Permissions; use tracing::{info, instrument}; use windmill::services::database::get_keycloak_pool; @@ -90,7 +90,8 @@ pub async fn get_user_id( election_event_id: &str, username: &str, ) -> Result> { - let realm = get_event_realm(tenant_id, election_event_id); + let tenant_realm = get_tenant_realm(tenant_id); + let event_realm = get_event_realm(tenant_id, election_event_id); let mut keycloak_db_client: DbClient = get_keycloak_pool() .await .get() @@ -102,10 +103,27 @@ pub async fn get_user_id( .await .map_err(|e| anyhow!("Error getting keycloak transaction: {e:?}"))?; - let user_ids = - get_users_by_username(&keycloak_transaction, &realm, username) + // Get user id by username, first look in the tenant realm which has less + // users. Then if not found, look in the event realm. + let mut user_ids = + get_users_by_username(&keycloak_transaction, &tenant_realm, username) .await - .map_err(|e| anyhow!("Error getting users by username: {e:?}"))?; + .map_err(|e| { + anyhow!( + "Error getting users by username in tenant realm: {e:?}" + ) + })?; + if user_ids.is_empty() { + user_ids = get_users_by_username( + &keycloak_transaction, + &event_realm, + username, + ) + .await + .map_err(|e| { + anyhow!("Error getting users by username in event realm: {e:?}") + })?; + } match user_ids.len() { 0 => { From 108123da75892b070a294c94e96841cd17036efb Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Fri, 16 Jan 2026 10:33:42 +0000 Subject: [PATCH 19/25] Fixes COLUMN labels --- packages/admin-portal/src/components/ElectoralLogList.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index 281dac597aa..2e90acd600e 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -280,7 +280,10 @@ export const ElectoralLogList: React.FC = ({ new Date(record.statement_timestamp * 1000).toUTCString() } /> - + Date: Wed, 21 Jan 2026 11:30:05 +0000 Subject: [PATCH 20/25] Fix unwrap risk --- packages/sequent-core/src/ballot_codec/bases.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/sequent-core/src/ballot_codec/bases.rs b/packages/sequent-core/src/ballot_codec/bases.rs index b6e7e0720f0..3f16ad3e443 100644 --- a/packages/sequent-core/src/ballot_codec/bases.rs +++ b/packages/sequent-core/src/ballot_codec/bases.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-only use crate::{ballot::*, types::ceremonies::CountingAlgType}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use std::convert::TryInto; pub trait BasesCodec { @@ -24,7 +24,9 @@ impl BasesCodec for Contest { CountingAlgType::Cumulative => { self.cumulative_number_of_checkboxes() + 1u64 } - _ => (self.max_votes + 1i64).try_into().unwrap(), + _ => (self.max_votes + 1i64).try_into().map_err(|_e| { + anyhow!("Failed to convert {} to u64", self.max_votes + 1i64) + })?, }; let num_valid_candidates: usize = self From 53cff449eca8f9c34d57b0dc1c4a0830b278a72a Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Wed, 21 Jan 2026 11:33:25 +0000 Subject: [PATCH 21/25] Filter fixes, timestamps with range --- hasura/metadata/actions.graphql | 6 ++- packages/admin-portal/graphql.schema.json | 28 ++++++++++- .../src/components/ElectoralLogList.tsx | 46 +++++++++++++++--- packages/admin-portal/src/gql/graphql.ts | 6 ++- .../src/queries/customBuildQuery.ts | 6 ++- packages/admin-portal/src/translations/cat.ts | 4 ++ packages/admin-portal/src/translations/en.ts | 4 ++ packages/admin-portal/src/translations/es.ts | 4 ++ packages/admin-portal/src/translations/eu.ts | 4 ++ packages/admin-portal/src/translations/fr.ts | 4 ++ packages/admin-portal/src/translations/gl.ts | 4 ++ packages/admin-portal/src/translations/nl.ts | 4 ++ packages/admin-portal/src/translations/tl.ts | 4 ++ packages/ballot-verifier/graphql.schema.json | 28 ++++++++++- packages/ballot-verifier/src/gql/graphql.ts | 6 ++- .../electoral-log/src/client/board_client.rs | 16 +++---- packages/electoral-log/src/client/types.rs | 48 +++++++++++++++---- packages/graphql.schema.json | 24 +++++++++- packages/harvest/src/routes/electoral_log.rs | 6 +-- packages/step-cli/src/graphql/schema.json | 24 +++++++++- packages/voting-portal/graphql.schema.json | 28 ++++++++++- packages/voting-portal/src/gql/graphql.ts | 6 ++- 22 files changed, 263 insertions(+), 47 deletions(-) diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index 059c21bcd65..65f1d56fd43 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -691,8 +691,10 @@ input PgAuditOrderBy { input ElectoralLogFilter { id: String user_id: String - created: String - statement_timestamp: String + created_min: String + created_max: String + statement_timestamp_min: String + statement_timestamp_max: String statement_kind: String username: String } diff --git a/packages/admin-portal/graphql.schema.json b/packages/admin-portal/graphql.schema.json index ab1b071bacd..00a513e11ec 100644 --- a/packages/admin-portal/graphql.schema.json +++ b/packages/admin-portal/graphql.schema.json @@ -2308,7 +2308,19 @@ "fields": null, "inputFields": [ { - "name": "created", + "name": "created_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_min", "description": null, "type": { "kind": "SCALAR", @@ -2344,7 +2356,19 @@ "deprecationReason": null }, { - "name": "statement_timestamp", + "name": "statement_timestamp_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "statement_timestamp_min", "description": null, "type": { "kind": "SCALAR", diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index 2e90acd600e..a0d91958d1e 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -201,14 +201,48 @@ export const ElectoralLogList: React.FC = ({ label={String(t("logsScreen.column.username"))} />, (value ? new Date(`${value}Z`).toISOString() : value)} + />, + (value ? new Date(`${value}Z`).toISOString() : value)} />, (value ? new Date(`${value}Z`).toISOString() : value)} + />, + (value ? new Date(`${value}Z`).toISOString() : value)} />, ; + created_max?: InputMaybe; + created_min?: InputMaybe; id?: InputMaybe; statement_kind?: InputMaybe; - statement_timestamp?: InputMaybe; + statement_timestamp_max?: InputMaybe; + statement_timestamp_min?: InputMaybe; user_id?: InputMaybe; username?: InputMaybe; }; diff --git a/packages/admin-portal/src/queries/customBuildQuery.ts b/packages/admin-portal/src/queries/customBuildQuery.ts index 33912e299fe..f9add530da4 100644 --- a/packages/admin-portal/src/queries/customBuildQuery.ts +++ b/packages/admin-portal/src/queries/customBuildQuery.ts @@ -67,8 +67,10 @@ export const customBuildQuery = "election_event_id", "user_id", "username", - "created", - "statement_timestamp", + "created_min", + "created_max", + "statement_timestamp_min", + "statement_timestamp_max", "statement_kind", ] Object.keys(params.filter).forEach((f) => { diff --git a/packages/admin-portal/src/translations/cat.ts b/packages/admin-portal/src/translations/cat.ts index 23622940185..d79f7a1902f 100644 --- a/packages/admin-portal/src/translations/cat.ts +++ b/packages/admin-portal/src/translations/cat.ts @@ -81,7 +81,11 @@ const catalanTranslation: TranslationType = { id: "ID", statement_kind: "Tipus de declaració", created: "Creat", + created_min: "Creat Mín", + created_max: "Creat Màx", statement_timestamp: "Marca de temps de declaració", + statement_timestamp_min: "Marca de temps de declaració Mín", + statement_timestamp_max: "Marca de temps de declaració Màx", message: "Missatge", user_id: "ID d'usuari", username: "Nom d'Usuari", diff --git a/packages/admin-portal/src/translations/en.ts b/packages/admin-portal/src/translations/en.ts index fe56d5beea1..2baf35ee023 100644 --- a/packages/admin-portal/src/translations/en.ts +++ b/packages/admin-portal/src/translations/en.ts @@ -27,7 +27,11 @@ const englishTranslation = { id: "Id", statement_kind: "Statement kind", created: "Created", + created_min: "Created Min", + created_max: "Created Max", statement_timestamp: "Statement Timestamp", + statement_timestamp_min: "Statement Timestamp Min", + statement_timestamp_max: "Statement Timestamp Max", message: "Message", user_id: "User Id", username: "Username", diff --git a/packages/admin-portal/src/translations/es.ts b/packages/admin-portal/src/translations/es.ts index 6280bbcad03..7c8e3523735 100644 --- a/packages/admin-portal/src/translations/es.ts +++ b/packages/admin-portal/src/translations/es.ts @@ -20,7 +20,11 @@ const spanishTranslation: TranslationType = { id: "ID", statement_kind: "Tipo de declaración", created: "Creado", + created_min: "Creado Mín", + created_max: "Creado Máx", statement_timestamp: "Marca de tiempo de declaración", + statement_timestamp_min: "Marca de tiempo de declaración Mín", + statement_timestamp_max: "Marca de tiempo de declaración Máx", message: "Mensaje", user_id: "ID de usuario", username: "Nombre de Usuario", diff --git a/packages/admin-portal/src/translations/eu.ts b/packages/admin-portal/src/translations/eu.ts index a5f2db8b26c..cb5aa3b0e14 100644 --- a/packages/admin-portal/src/translations/eu.ts +++ b/packages/admin-portal/src/translations/eu.ts @@ -28,7 +28,11 @@ const basqueTranslation: TranslationType = { id: "IDa", statement_kind: "Adierazpen mota", created: "Sortua", + created_min: "Sortua Min", + created_max: "Sortua Max", statement_timestamp: "Adierazpen denbora-marka", + statement_timestamp_min: "Adierazpen denbora-marka Min", + statement_timestamp_max: "Adierazpen denbora-marka Max", message: "Mezua", user_id: "Erabiltzaile IDa", username: "Erabiltzaile izena", diff --git a/packages/admin-portal/src/translations/fr.ts b/packages/admin-portal/src/translations/fr.ts index 0076713f435..b0398143a45 100644 --- a/packages/admin-portal/src/translations/fr.ts +++ b/packages/admin-portal/src/translations/fr.ts @@ -28,7 +28,11 @@ const frenchTranslation: TranslationType = { id: "ID", statement_kind: "Type de déclaration", created: "Créé", + created_min: "Créé Min", + created_max: "Créé Max", statement_timestamp: "Horodatage de déclaration", + statement_timestamp_min: "Horodatage de déclaration Min", + statement_timestamp_max: "Horodatage de déclaration Max", message: "Message", user_id: "ID utilisateur", username: "Nom d'Utilisateur", diff --git a/packages/admin-portal/src/translations/gl.ts b/packages/admin-portal/src/translations/gl.ts index 2b479976251..1c1e04b7972 100644 --- a/packages/admin-portal/src/translations/gl.ts +++ b/packages/admin-portal/src/translations/gl.ts @@ -28,7 +28,11 @@ const galegoTranslation: TranslationType = { id: "ID", statement_kind: "Tipo de declaración", created: "Creado", + created_min: "Creado Mín", + created_max: "Creado Máx", statement_timestamp: "Marca de tiempo de declaración", + statement_timestamp_min: "Marca de tiempo de declaración Mín", + statement_timestamp_max: "Marca de tiempo de declaración Máx", message: "Mensaje", user_id: "ID de usuario", username: "Nombre de Usuario", diff --git a/packages/admin-portal/src/translations/nl.ts b/packages/admin-portal/src/translations/nl.ts index 2680e57b0fd..2031bff536d 100644 --- a/packages/admin-portal/src/translations/nl.ts +++ b/packages/admin-portal/src/translations/nl.ts @@ -27,7 +27,11 @@ const dutchTranslation: TranslationType = { id: "Id", statement_kind: "Soort verklaring", created: "Aangemaakt", + created_min: "Aangemaakt Min", + created_max: "Aangemaakt Max", statement_timestamp: "Tijdstempel verklaring", + statement_timestamp_min: "Tijdstempel verklaring Min", + statement_timestamp_max: "Tijdstempel verklaring Max", message: "Bericht", user_id: "Gebruikers-ID", username: "Gebruikersnaam", diff --git a/packages/admin-portal/src/translations/tl.ts b/packages/admin-portal/src/translations/tl.ts index 8cd5df426cb..ba3e11e2ea6 100644 --- a/packages/admin-portal/src/translations/tl.ts +++ b/packages/admin-portal/src/translations/tl.ts @@ -28,7 +28,11 @@ const tagalogTranslation: TranslationType = { id: "ID", statement_kind: "Uri ng Pahayag", created: "Nilikha", + created_min: "Nilikha Min", + created_max: "Nilikha Max", statement_timestamp: "Tatak ng Panahon ng Pahayag", + statement_timestamp_min: "Tatak ng Panahon ng Pahayag Min", + statement_timestamp_max: "Tatak ng Panahon ng Pahayag Max", message: "Mensahe", user_id: "ID ng User", username: "Username", diff --git a/packages/ballot-verifier/graphql.schema.json b/packages/ballot-verifier/graphql.schema.json index ab1b071bacd..00a513e11ec 100644 --- a/packages/ballot-verifier/graphql.schema.json +++ b/packages/ballot-verifier/graphql.schema.json @@ -2308,7 +2308,19 @@ "fields": null, "inputFields": [ { - "name": "created", + "name": "created_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_min", "description": null, "type": { "kind": "SCALAR", @@ -2344,7 +2356,19 @@ "deprecationReason": null }, { - "name": "statement_timestamp", + "name": "statement_timestamp_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "statement_timestamp_min", "description": null, "type": { "kind": "SCALAR", diff --git a/packages/ballot-verifier/src/gql/graphql.ts b/packages/ballot-verifier/src/gql/graphql.ts index 404cd4553e5..dc2d5cd8d7f 100644 --- a/packages/ballot-verifier/src/gql/graphql.ts +++ b/packages/ballot-verifier/src/gql/graphql.ts @@ -278,10 +278,12 @@ export type ElectionStatsOutput = { } export type ElectoralLogFilter = { - created?: InputMaybe + created_max?: InputMaybe + created_min?: InputMaybe id?: InputMaybe statement_kind?: InputMaybe - statement_timestamp?: InputMaybe + statement_timestamp_max?: InputMaybe + statement_timestamp_min?: InputMaybe user_id?: InputMaybe username?: InputMaybe } diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index e9d20f72c1f..121c41ca2dd 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -205,15 +205,15 @@ impl BoardClient { .to_where_clause(); // Min and max clauses will go in the end of where_clause let (min_clause, min_clause_value) = if let Some(min_ts) = min_ts { - ("statement_timestamp >= @min_ts", min_ts) + ("statement_timestamp >= @min_ts", Some(min_ts)) } else { - ("", 0) + ("", None) }; let (max_clause, max_clause_value) = if let Some(max_ts) = max_ts { - ("statement_timestamp <= @max_ts", max_ts) + ("statement_timestamp <= @max_ts", Some(max_ts)) } else { - ("", 0) + ("", None) }; let order_by_clauses = if let Some(order_by) = order_by { @@ -226,7 +226,7 @@ impl BoardClient { format!("ORDER BY id desc") }; - if min_clause_value != 0 { + if let Some(min_clause_value) = min_clause_value { params.push(NamedParam { name: String::from("min_ts"), value: Some(SqlValue { @@ -234,7 +234,7 @@ impl BoardClient { }), }) } - if max_clause_value != 0 { + if let Some(max_clause_value) = max_clause_value { params.push(NamedParam { name: String::from("max_ts"), value: Some(SqlValue { @@ -910,11 +910,11 @@ pub(crate) mod tests { let cols_match = WhereClauseOrdMap::from(&[ ( ElectoralLogVarCharColumn::StatementKind, - (SqlCompOperators::Equal, "".to_string()), + (SqlCompOperators::Equal("".to_string())), ), ( ElectoralLogVarCharColumn::SenderPk, - (SqlCompOperators::Equal, "".to_string()), + (SqlCompOperators::Equal("".to_string())), ), ]); let ret = b diff --git a/packages/electoral-log/src/client/types.rs b/packages/electoral-log/src/client/types.rs index f1a3096d278..654810bc527 100644 --- a/packages/electoral-log/src/client/types.rs +++ b/packages/electoral-log/src/client/types.rs @@ -199,13 +199,35 @@ pub enum OrderDirection { Desc, } +// Enumeration for the valid filter fields with min/max support for timestamps +#[derive(Debug, Deserialize, Hash, PartialEq, Eq, EnumString, Display, Clone)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum FilterField { + Id, + CreatedMin, + CreatedMax, + StatementTimestampMin, + StatementTimestampMax, + StatementKind, + Message, + UserId, + Username, + BallotId, + SenderPk, + LogType, + EventType, + Description, + Version, +} + #[derive(Deserialize, Debug, Default, Clone)] pub struct GetElectoralLogBody { pub tenant_id: String, pub election_event_id: String, pub limit: Option, pub offset: Option, - pub filter: Option>, + pub filter: Option>, pub order_by: Option>, pub election_id: Option, pub area_ids: Option>, @@ -221,14 +243,19 @@ impl GetElectoralLogBody { if let Some(filters_map) = &self.filter { for (field, value) in filters_map.iter() { match field { - OrderField::Created | OrderField::StatementTimestamp => { + FilterField::CreatedMin | FilterField::StatementTimestampMin => { let date_time_utc = DateTime::parse_from_rfc3339(&value) .map_err(|err| anyhow!("Error parsing timestamp: {err:?}"))?; let datetime = date_time_utc.with_timezone(&Utc); let ts: i64 = datetime.timestamp(); - let ts_end: i64 = ts + 90_000; // Search along that day. min_ts = Some(ts); - max_ts = Some(ts_end); + } + FilterField::CreatedMax | FilterField::StatementTimestampMax => { + let date_time_utc = DateTime::parse_from_rfc3339(&value) + .map_err(|err| anyhow!("Error parsing timestamp: {err:?}"))?; + let datetime = date_time_utc.with_timezone(&Utc); + let ts: i64 = datetime.timestamp(); + max_ts = Some(ts); } _ => {} } @@ -244,15 +271,15 @@ impl GetElectoralLogBody { if let Some(filters_map) = &self.filter { for (field, value) in filters_map.iter() { match field { - OrderField::Id => {} // Why would someone filter the electoral log by id? - OrderField::SenderPk | OrderField::Username | OrderField::BallotId | OrderField::StatementKind | OrderField::Version => { // sql VARCHAR type + FilterField::Id => {} // Why would someone filter the electoral log by id? + FilterField::SenderPk | FilterField::Username | FilterField::BallotId | FilterField::StatementKind | FilterField::Version => { // sql VARCHAR type let variant = ElectoralLogVarCharColumn::from_str(field.to_string().as_str()).map_err(|_| anyhow!("Field not found"))?; cols_match_select.insert( variant, SqlCompOperators::Equal(value.clone()), // Using 'Like' here would not scale for millions of entries, causing no response from immudb is some cases. ); } - OrderField::UserId => { + FilterField::UserId => { // insert user_id_mod cols_match_select.insert( ElectoralLogVarCharColumn::UserIdKey, @@ -264,9 +291,10 @@ impl GetElectoralLogBody { SqlCompOperators::Equal(value.clone()), ); } - OrderField::StatementTimestamp | OrderField::Created => {} // handled by `get_min_max_ts` - OrderField::EventType | OrderField::LogType | OrderField::Description // these have no column but are inside of Message - | OrderField::Message => {} // Message column is sql BLOB type and it´s encrypted so we can't filter it without expensive operations + FilterField::StatementTimestampMin | FilterField::StatementTimestampMax + | FilterField::CreatedMin | FilterField::CreatedMax => {} // handled by `get_min_max_ts` + FilterField::EventType | FilterField::LogType | FilterField::Description // these have no column but are inside of Message + | FilterField::Message => {} // Message column is sql BLOB type and it´s encrypted so we can't filter it without expensive operations } } } diff --git a/packages/graphql.schema.json b/packages/graphql.schema.json index db5363a23c9..ec2441d16f6 100644 --- a/packages/graphql.schema.json +++ b/packages/graphql.schema.json @@ -2090,7 +2090,17 @@ "fields": null, "inputFields": [ { - "name": "created", + "name": "created_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "created_min", "description": null, "type": { "kind": "SCALAR", @@ -2120,7 +2130,17 @@ "defaultValue": null }, { - "name": "statement_timestamp", + "name": "statement_timestamp_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "statement_timestamp_min", "description": null, "type": { "kind": "SCALAR", diff --git a/packages/harvest/src/routes/electoral_log.rs b/packages/harvest/src/routes/electoral_log.rs index 9d36af8d80b..0ca539f0ca2 100644 --- a/packages/harvest/src/routes/electoral_log.rs +++ b/packages/harvest/src/routes/electoral_log.rs @@ -39,8 +39,8 @@ pub async fn list_electoral_log( // inprove performance. if let Some(filter) = &mut input.filter { if let (Some(username), None) = ( - filter.get(&OrderField::Username), - filter.get(&OrderField::UserId), + filter.get(&FilterField::Username), + filter.get(&FilterField::UserId), ) { match get_user_id( &input.tenant_id, @@ -50,7 +50,7 @@ pub async fn list_electoral_log( .await { Ok(Some(user_id)) => { - filter.insert(OrderField::UserId, user_id); + filter.insert(FilterField::UserId, user_id); } Ok(None) => { return Ok(Json(DataList::default())); diff --git a/packages/step-cli/src/graphql/schema.json b/packages/step-cli/src/graphql/schema.json index db5363a23c9..ec2441d16f6 100644 --- a/packages/step-cli/src/graphql/schema.json +++ b/packages/step-cli/src/graphql/schema.json @@ -2090,7 +2090,17 @@ "fields": null, "inputFields": [ { - "name": "created", + "name": "created_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "created_min", "description": null, "type": { "kind": "SCALAR", @@ -2120,7 +2130,17 @@ "defaultValue": null }, { - "name": "statement_timestamp", + "name": "statement_timestamp_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null + }, + { + "name": "statement_timestamp_min", "description": null, "type": { "kind": "SCALAR", diff --git a/packages/voting-portal/graphql.schema.json b/packages/voting-portal/graphql.schema.json index ab1b071bacd..00a513e11ec 100644 --- a/packages/voting-portal/graphql.schema.json +++ b/packages/voting-portal/graphql.schema.json @@ -2308,7 +2308,19 @@ "fields": null, "inputFields": [ { - "name": "created", + "name": "created_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_min", "description": null, "type": { "kind": "SCALAR", @@ -2344,7 +2356,19 @@ "deprecationReason": null }, { - "name": "statement_timestamp", + "name": "statement_timestamp_max", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "statement_timestamp_min", "description": null, "type": { "kind": "SCALAR", diff --git a/packages/voting-portal/src/gql/graphql.ts b/packages/voting-portal/src/gql/graphql.ts index 56354ce4270..29ec094d932 100644 --- a/packages/voting-portal/src/gql/graphql.ts +++ b/packages/voting-portal/src/gql/graphql.ts @@ -277,10 +277,12 @@ export type ElectionStatsOutput = { }; export type ElectoralLogFilter = { - created?: InputMaybe; + created_max?: InputMaybe; + created_min?: InputMaybe; id?: InputMaybe; statement_kind?: InputMaybe; - statement_timestamp?: InputMaybe; + statement_timestamp_max?: InputMaybe; + statement_timestamp_min?: InputMaybe; user_id?: InputMaybe; username?: InputMaybe; }; From 24f1c5b4c99f68adc523d70ed3e5471594a11198 Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Thu, 5 Feb 2026 08:24:43 +0000 Subject: [PATCH 22/25] Fix --- packages/admin-portal/src/components/ElectoralLogList.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index a0d91958d1e..db134cb0fb9 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -209,6 +209,7 @@ export const ElectoralLogList: React.FC = ({ step: 1, // Adds seconds field to the datetime input }, }} + format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, = ({ step: 1, // Adds seconds field to the datetime input }, }} + format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, = ({ step: 1, // Adds seconds field to the datetime input }, }} + format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, = ({ step: 1, // Adds seconds field to the datetime input }, }} + format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, Date: Thu, 5 Feb 2026 08:26:39 +0000 Subject: [PATCH 23/25] Format --- packages/admin-portal/src/components/ElectoralLogList.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/admin-portal/src/components/ElectoralLogList.tsx b/packages/admin-portal/src/components/ElectoralLogList.tsx index db134cb0fb9..96e134e4a9a 100644 --- a/packages/admin-portal/src/components/ElectoralLogList.tsx +++ b/packages/admin-portal/src/components/ElectoralLogList.tsx @@ -209,7 +209,7 @@ export const ElectoralLogList: React.FC = ({ step: 1, // Adds seconds field to the datetime input }, }} - format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} + format={(value) => (value ? value.replace("Z", "").replace(".000", "") : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, = ({ step: 1, // Adds seconds field to the datetime input }, }} - format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} + format={(value) => (value ? value.replace("Z", "").replace(".000", "") : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, = ({ step: 1, // Adds seconds field to the datetime input }, }} - format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} + format={(value) => (value ? value.replace("Z", "").replace(".000", "") : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, = ({ step: 1, // Adds seconds field to the datetime input }, }} - format={(value) => (value ? value.replace('Z', '').replace('.000', '') : value)} + format={(value) => (value ? value.replace("Z", "").replace(".000", "") : value)} parse={(value) => (value ? new Date(`${value}Z`).toISOString() : value)} />, Date: Fri, 17 Apr 2026 13:37:34 +0000 Subject: [PATCH 24/25] Fix conflicts --- .../electoral-log/src/client/board_client.rs | 256 +++++------------- .../src/services/reports/activity_log.rs | 98 ------- 2 files changed, 64 insertions(+), 290 deletions(-) diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index fce829bbb5e..b34fa6e7084 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -53,87 +53,6 @@ impl BoardClient { Ok(BoardClient { client: client }) } - /// Get all electoral log messages whose id is bigger than `last_id` - pub async fn get_electoral_log_messages( - &mut self, - board_db: &str, - ) -> Result> { - let mut offset: usize = 0; - let mut last_batch = self - .get_electoral_log_messages_from_db( - board_db, - 0, - Some(IMMUDB_DEFAULT_LIMIT), - Some(offset), - ) - .await?; - let mut messages = last_batch.clone(); - while IMMUDB_DEFAULT_LIMIT == last_batch.len() { - offset += last_batch.len(); - last_batch = self - .get_electoral_log_messages_from_db( - board_db, - 0, - Some(IMMUDB_DEFAULT_LIMIT), - Some(offset), - ) - .await?; - messages.extend(last_batch.clone()); - } - Ok(messages) - } - - async fn get_electoral_log_messages_from_db( - &mut self, - board_db: &str, - last_id: i64, - limit: Option, - offset: Option, - ) -> Result> { - self.client.use_database(board_db).await?; - let sql = format!( - r#" - SELECT - id, - created, - sender_pk, - statement_timestamp, - statement_kind, - message, - version, - user_id, - area_id, - ballot_id, - username - FROM {} - WHERE id > @last_id - ORDER BY id - LIMIT {} - OFFSET {}; - "#, - ELECTORAL_LOG_TABLE, - limit.unwrap_or(IMMUDB_DEFAULT_LIMIT), - offset.unwrap_or(IMMUDB_DEFAULT_OFFSET), - ); - - let params = vec![NamedParam { - name: String::from("last_id"), - value: Some(SqlValue { - value: Some(Value::N(last_id)), - }), - }]; - - let sql_query_response = self.client.sql_query(&sql, params).await?; - let messages = sql_query_response - .get_ref() - .rows - .iter() - .map(ElectoralLogMessage::try_from) - .collect::>>()?; - - Ok(messages) - } - /// columns_matcher represents the columns that will be used to filter the messages, /// The order as defined ElectoralLogVarCharColumn is important for preformance to match the indexes. /// BTreeMap ensures the order is preserved no matter the insertion sequence. @@ -164,8 +83,12 @@ impl BoardClient { .await } - #[instrument(skip_all, err)] - pub async fn get_electoral_log_messages_batch( + /// Returns a batch of electoral log messages at a given row offset, ordered by id. + /// Prefer `get_electoral_log_messages_batch` (cursor-based) when iterating sequentially. + /// This offset-based variant exists for parallel batch processing where offsets are + /// pre-computed and batches run concurrently. + #[instrument(skip(self), err)] + pub async fn get_electoral_log_messages_at_offset( &mut self, board_db: &str, limit: i64, @@ -362,6 +285,60 @@ impl BoardClient { Ok(messages) } + /// Returns a batch of electoral log messages with id > `after_id`, ordered by id, + /// using streaming to handle large result sets. Pass `after_id = 0` to start + /// from the beginning. Use the `id` of the last returned message as `after_id` + /// for the next page. + #[instrument(skip(self), err)] + pub async fn get_electoral_log_messages_batch( + &mut self, + board_db: &str, + limit: i64, + after_id: i64, + ) -> Result> { + self.client.use_database(board_db).await?; + let sql = format!( + r#" + SELECT + id, + created, + sender_pk, + statement_timestamp, + statement_kind, + message, + version, + user_id, + username + FROM {ELECTORAL_LOG_TABLE} + WHERE id > {after_id} + ORDER BY id + LIMIT {limit}; + "# + ); + let response_stream = self.client.streaming_sql_query(&sql, vec![]).await?; + let mut stream = response_stream.into_inner(); + let mut messages: Vec = vec![]; + while let Some(batch_result) = stream.next().await { + match batch_result { + Ok(batch) => { + for row in &batch.rows { + match ElectoralLogMessage::try_from(row) { + Ok(msg) => messages.push(msg), + Err(e) => { + warn!("Failed to parse row: {e}"); + } + } + } + } + Err(e) => { + error!("Error receiving batch from stream: {e}"); + break; + } + } + } + Ok(messages) + } + #[instrument(skip(self, board_db), err)] pub async fn count_electoral_log_messages( &mut self, @@ -452,114 +429,6 @@ impl BoardClient { Ok(count as i64) } - /// Returns a batch of electoral log messages at a given row offset, ordered by id. - /// Prefer `get_electoral_log_messages_batch` (cursor-based) when iterating sequentially. - /// This offset-based variant exists for parallel batch processing where offsets are - /// pre-computed and batches run concurrently. - #[instrument(skip(self), err)] - pub async fn get_electoral_log_messages_at_offset( - &mut self, - board_db: &str, - limit: i64, - offset: i64, - ) -> Result> { - self.client.use_database(board_db).await?; - let sql = format!( - r#" - SELECT - id, - created, - sender_pk, - statement_timestamp, - statement_kind, - message, - version, - user_id, - username - FROM {ELECTORAL_LOG_TABLE} - ORDER BY id - LIMIT {limit} - OFFSET {offset}; - "# - ); - let response_stream = self.client.streaming_sql_query(&sql, vec![]).await?; - let mut stream = response_stream.into_inner(); - let mut messages: Vec = vec![]; - while let Some(batch_result) = stream.next().await { - match batch_result { - Ok(batch) => { - for row in &batch.rows { - match ElectoralLogMessage::try_from(row) { - Ok(msg) => messages.push(msg), - Err(e) => { - warn!("Failed to parse row: {e}"); - } - } - } - } - Err(e) => { - error!("Error receiving batch from stream: {e}"); - break; - } - } - } - Ok(messages) - } - - /// Returns a batch of electoral log messages with id > `after_id`, ordered by id, - /// using streaming to handle large result sets. Pass `after_id = 0` to start - /// from the beginning. Use the `id` of the last returned message as `after_id` - /// for the next page. - #[instrument(skip(self), err)] - pub async fn get_electoral_log_messages_batch( - &mut self, - board_db: &str, - limit: i64, - after_id: i64, - ) -> Result> { - self.client.use_database(board_db).await?; - let sql = format!( - r#" - SELECT - id, - created, - sender_pk, - statement_timestamp, - statement_kind, - message, - version, - user_id, - username - FROM {ELECTORAL_LOG_TABLE} - WHERE id > {after_id} - ORDER BY id - LIMIT {limit}; - "# - ); - let response_stream = self.client.streaming_sql_query(&sql, vec![]).await?; - let mut stream = response_stream.into_inner(); - let mut messages: Vec = vec![]; - while let Some(batch_result) = stream.next().await { - match batch_result { - Ok(batch) => { - for row in &batch.rows { - match ElectoralLogMessage::try_from(row) { - Ok(msg) => messages.push(msg), - Err(e) => { - warn!("Failed to parse row: {e}"); - } - } - } - } - Err(e) => { - error!("Error receiving batch from stream: {e}"); - break; - } - } - } - Ok(messages) - } - pub async fn open_session(&mut self, database_name: &str) -> Result<()> { self.client.open_session(database_name).await } @@ -1012,7 +881,10 @@ pub(crate) mod tests { .await .unwrap(); - let ret = b.get_electoral_log_messages(BOARD_DB).await.unwrap(); + let ret = b + .get_electoral_log_messages_batch(BOARD_DB, 100, 0) + .await + .unwrap(); assert_eq!(messages, ret); let cols_match = WhereClauseOrdMap::from(&[ diff --git a/packages/windmill/src/services/reports/activity_log.rs b/packages/windmill/src/services/reports/activity_log.rs index 344af11b371..2c61f7cc39e 100644 --- a/packages/windmill/src/services/reports/activity_log.rs +++ b/packages/windmill/src/services/reports/activity_log.rs @@ -14,8 +14,6 @@ use csv::WriterBuilder; use deadpool_postgres::Transaction; use electoral_log::client::types::*; use electoral_log::messages::message::{Message, SigningData}; -use electoral_log::messages::message::Message; -use electoral_log::ElectoralLogMessage; use sequent_core::services::date::ISO8601; use sequent_core::services::s3::get_minio_url; use sequent_core::types::hasura::core::TasksExecution; @@ -28,9 +26,6 @@ use strum_macros::EnumString; use tempfile::NamedTempFile; use tracing::{debug, info, instrument, warn}; -const KB: f64 = 1024.0; -const MB: f64 = 1024.0 * KB; - #[derive(Serialize, Deserialize, Debug, Clone, EnumString, PartialEq, Copy)] pub enum ReportFormat { CSV, @@ -182,99 +177,6 @@ impl ActivityLogsTemplate { } } -impl TryFrom for ActivityLogRow { - type Error = anyhow::Error; - - fn try_from(electoral_log: ElectoralLogRow) -> Result { - let user_id = match electoral_log.user_id() { - Some(user_id) => user_id.to_string(), - None => "-".to_string(), - }; - - let statement_timestamp: String = if let Ok(datetime_parsed) = - ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.statement_timestamp()) - { - datetime_parsed.to_rfc3339() - } else { - return Err(anyhow::anyhow!("Error parsing statement_timestamp")); - }; - - let created: String = if let Ok(datetime_parsed) = - ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.created()) - { - datetime_parsed.to_rfc3339() - } else { - return Err(anyhow::anyhow!("Error parsing created")); - }; - - let head_data = electoral_log - .statement_head_data() - .with_context(|| "Error to get head data.")?; - let event_type = head_data.event_type; - let log_type = head_data.log_type; - let description = head_data.description; - - Ok(ActivityLogRow { - id: electoral_log.id(), - user_id, - created, - statement_timestamp, - statement_kind: electoral_log.statement_kind().to_string(), - event_type, - log_type, - description, - message: electoral_log.message().to_string(), - }) - } -} - -impl TryFrom for ActivityLogRow { - type Error = anyhow::Error; - - fn try_from(electoral_log: ElectoralLogMessage) -> Result { - let user_id = match electoral_log.user_id { - Some(user_id) => user_id.to_string(), - None => "-".to_string(), - }; - - let statement_timestamp: String = if let Ok(datetime_parsed) = - ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.statement_timestamp) - { - datetime_parsed.to_rfc3339() - } else { - return Err(anyhow::anyhow!("Error parsing statement_timestamp")); - }; - - let created: String = if let Ok(datetime_parsed) = - ISO8601::timestamp_secs_utc_to_date_opt(electoral_log.created) - { - datetime_parsed.to_rfc3339() - } else { - return Err(anyhow::anyhow!("Error parsing created")); - }; - - let deserialized_message = Message::strand_deserialize(&electoral_log.message) - .map_err(|e| anyhow!("Error deserializing message: {e:?}"))?; - - let head_data = deserialized_message.statement.head.clone(); - let event_type = head_data.event_type.to_string(); - let log_type = head_data.log_type.to_string(); - let description = head_data.description; - - Ok(ActivityLogRow { - id: electoral_log.id, - user_id, - created, - statement_timestamp, - statement_kind: electoral_log.statement_kind, - event_type, - log_type, - description, - message: deserialized_message.to_string(), - }) - } -} - #[async_trait] impl TemplateRenderer for ActivityLogsTemplate { type UserData = UserData; From 60992e009042ecee49c102f8152b3e4a073b0ecb Mon Sep 17 00:00:00 2001 From: Beltran Rodriguez Date: Fri, 17 Apr 2026 13:54:28 +0000 Subject: [PATCH 25/25] wip --- packages/electoral-log/src/client/board_client.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/electoral-log/src/client/board_client.rs b/packages/electoral-log/src/client/board_client.rs index b34fa6e7084..9e5be4471f6 100644 --- a/packages/electoral-log/src/client/board_client.rs +++ b/packages/electoral-log/src/client/board_client.rs @@ -4,22 +4,17 @@ use crate::client::types::*; use anyhow::{anyhow, Context, Result}; -use chrono::format; -use immudb_rs::{sql_value::Value, Client, CommittedSqlTx, NamedParam, Row, SqlValue, TxMode}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use immudb_rs::{sql_value::Value, Client, CommittedSqlTx, NamedParam, SqlValue, TxMode}; use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Display; -use std::time::Duration; use std::time::Instant; use tokio_stream::StreamExt; // Added for streaming use tracing::{error, info, instrument, warn}; const IMMUDB_DEFAULT_LIMIT: usize = 900; -const IMMUDB_DEFAULT_ENTRIES_TX_LIMIT: usize = 50; const IMMUDB_DEFAULT_OFFSET: usize = 0; -const ELECTORAL_LOG_TABLE: &'static str = "electoral_log_messages"; +const ELECTORAL_LOG_TABLE: &str = "electoral_log_messages"; /// 36 chars + EOL + some padding const ID_VARCHAR_LENGTH: usize = 40; const ID_KEY_VARCHAR_LENGTH: usize = 4; @@ -33,7 +28,7 @@ const BALLOT_ID_VARCHAR_LENGTH: usize = 70; /// /// Other columns that have no length constraint are not indexable. /// 'create' is not indexed, we use statement_timestamp intead. -pub const MULTI_COLUMN_INDEXES: [&'static str; 3] = [ +pub const MULTI_COLUMN_INDEXES: [&str; 3] = [ "(statement_kind, election_id, user_id_key, user_id, ballot_id)", // COUNT or SELECT cast_vote_messages and filter by ballot_id "(statement_kind, user_id_key, user_id)", // Filters in Admin portal LOGS tab. "(user_id_key, user_id)", // Filters in Admin portal LOGS tab and for the User´s logs.