From b94ac3891dccfddcad7f61978ca2479a5b07791a Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:40:41 +0000 Subject: [PATCH 1/2] use ro_pool for search indexing again --- apps/labrinth/src/background_task.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs index 32b32551aa..3936356840 100644 --- a/apps/labrinth/src/background_task.rs +++ b/apps/labrinth/src/background_task.rs @@ -100,7 +100,7 @@ impl BackgroundTask { } IncrementalIndexSearch => { crate::search::incremental::consume::run( - pool, + ro_pool, redis_pool, search_backend, kafka_client, From 62c128ac6ffb70fa61ed3001792c2370f7a63b69 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:07:11 +0000 Subject: [PATCH 2/2] adjust incremental indexing operations --- apps/labrinth/src/models/v2/search.rs | 4 +- apps/labrinth/src/routes/v3/projects.rs | 47 ++- .../src/search/backend/elasticsearch/mod.rs | 1 + apps/labrinth/src/search/incremental.rs | 56 ++- .../src/search/incremental/consume.rs | 55 ++- apps/labrinth/src/search/indexing.rs | 398 +++++++++--------- apps/labrinth/src/search/mod.rs | 9 +- docker-compose.yml | 2 + .../api-client/src/modules/labrinth/types.ts | 2 +- 9 files changed, 343 insertions(+), 231 deletions(-) diff --git a/apps/labrinth/src/models/v2/search.rs b/apps/labrinth/src/models/v2/search.rs index 5f617153bc..2edfef56fe 100644 --- a/apps/labrinth/src/models/v2/search.rs +++ b/apps/labrinth/src/models/v2/search.rs @@ -152,7 +152,9 @@ impl LegacyResultSearchProject { server_side, environment: environments, versions, - latest_version: result_search_project.version_id, + latest_version: result_search_project + .version_id + .unwrap_or_default(), categories, project_id: result_search_project.project_id, diff --git a/apps/labrinth/src/routes/v3/projects.rs b/apps/labrinth/src/routes/v3/projects.rs index f996bb6058..59d027c672 100644 --- a/apps/labrinth/src/routes/v3/projects.rs +++ b/apps/labrinth/src/routes/v3/projects.rs @@ -1207,6 +1207,14 @@ pub async fn project_edit_internal( let mut reindex_versions = new_project.categories.is_some() || new_project.additional_categories.is_some(); + let became_searchable = !project_item.inner.status.is_searchable() + && new_project + .status + .is_some_and(|status| status.is_searchable()); + let became_unsearchable = project_item.inner.status.is_searchable() + && new_project + .status + .is_some_and(|status| !status.is_searchable()); reindex_versions |= update( &mut transaction, @@ -1284,7 +1292,7 @@ pub async fn project_edit_internal( .await .wrap_internal_err("committing database transaction")?; - if reindex_versions { + if became_unsearchable { db_models::DBProject::clear_cache( project_item.inner.id, project_item.inner.slug, @@ -1295,10 +1303,20 @@ pub async fn project_edit_internal( .wrap_internal_err("clearing cached data from Redis")?; search_state .queue - .push_version_changes( - project_item.inner.id.into(), - project_item.versions.iter().copied().map(VersionId::from), - ) + .push_project_removal(project_item.inner.id.into()) + .await; + } else if reindex_versions || became_searchable { + db_models::DBProject::clear_cache( + project_item.inner.id, + project_item.inner.slug, + None, + &redis, + ) + .await + .wrap_internal_err("clearing cached data from Redis")?; + search_state + .queue + .push_project_with_all_versions_change(project_item.inner.id.into()) .await; } else { clear_project_cache_and_queue_search( @@ -1312,17 +1330,6 @@ pub async fn project_edit_internal( .wrap_api_err("executing `clear_project_cache_and_queue_search`")?; } - // Remove no longer searchable projects from search index - if let (true, Some(false)) = ( - project_item.inner.status.is_searchable(), - new_project.status.map(|status| status.is_searchable()), - ) { - search_state - .queue - .push_project_removal(project_item.inner.id.into()) - .await; - } - Ok(HttpResponse::NoContent().body("")) } @@ -1890,7 +1897,6 @@ pub async fn projects_edit( changed_projects.push(( project.inner.id, project.inner.slug, - project.versions, reindex_versions, )); } @@ -1900,17 +1906,14 @@ pub async fn projects_edit( .await .wrap_internal_err("committing database transaction")?; - for (project_id, slug, versions, reindex_versions) in changed_projects { + for (project_id, slug, reindex_versions) in changed_projects { if reindex_versions { db_models::DBProject::clear_cache(project_id, slug, None, &redis) .await .wrap_internal_err("clearing cached data from Redis")?; search_state .queue - .push_version_changes( - project_id.into(), - versions.into_iter().map(VersionId::from), - ) + .push_project_with_all_versions_change(project_id.into()) .await; } else { clear_project_cache_and_queue_search( diff --git a/apps/labrinth/src/search/backend/elasticsearch/mod.rs b/apps/labrinth/src/search/backend/elasticsearch/mod.rs index 29026364a4..acf44e4389 100644 --- a/apps/labrinth/src/search/backend/elasticsearch/mod.rs +++ b/apps/labrinth/src/search/backend/elasticsearch/mod.rs @@ -910,6 +910,7 @@ impl Elasticsearch { let mut document = hit["_source"].clone(); let object = document.as_object_mut()?; object.remove("document_type"); + object.remove("_search_tokens"); if filter .as_ref() .is_some_and(|filter| filter.has_version_filter) diff --git a/apps/labrinth/src/search/incremental.rs b/apps/labrinth/src/search/incremental.rs index f96c63f508..c8ac8c536b 100644 --- a/apps/labrinth/src/search/incremental.rs +++ b/apps/labrinth/src/search/incremental.rs @@ -17,7 +17,7 @@ use crate::{ }; pub const SEARCH_PROJECT_INDEX_QUEUE_TOPIC: &str = - "public.labrinth.search-project-index-queue.v1"; + "public.labrinth.search-project-index-queue.v2"; const QUEUE_FLUSH_INTERVAL: Duration = Duration::from_secs(10); #[derive(Clone)] @@ -40,6 +40,16 @@ impl IncrementalSearchQueue { self.operations.lock().await.push_project_change(project_id); } + pub async fn push_project_with_all_versions_change( + &self, + project_id: ProjectId, + ) { + self.operations + .lock() + .await + .push_project_with_all_versions_change(project_id); + } + pub async fn push_version_changes( &self, project_id: ProjectId, @@ -116,6 +126,7 @@ impl IncrementalSearchQueue { #[derive(Default)] struct PendingSearchIndexOperations { changed_project_ids: HashSet, + changed_project_ids_with_all_versions: HashSet, changed_project_versions: HashMap>, removed_project_ids: HashSet, } @@ -123,25 +134,47 @@ struct PendingSearchIndexOperations { impl PendingSearchIndexOperations { fn is_empty(&self) -> bool { self.changed_project_ids.is_empty() + && self.changed_project_ids_with_all_versions.is_empty() && self.changed_project_versions.is_empty() && self.removed_project_ids.is_empty() } fn push_project_change(&mut self, project_id: ProjectId) { - if !self.removed_project_ids.contains(&project_id) { + if !self.removed_project_ids.contains(&project_id) + && !self + .changed_project_ids_with_all_versions + .contains(&project_id) + && !self.changed_project_versions.contains_key(&project_id) + { self.changed_project_ids.insert(project_id); } } + fn push_project_with_all_versions_change(&mut self, project_id: ProjectId) { + if self.removed_project_ids.contains(&project_id) { + return; + } + + self.changed_project_ids.remove(&project_id); + self.changed_project_versions.remove(&project_id); + self.changed_project_ids_with_all_versions + .insert(project_id); + } + fn push_version_change( &mut self, project_id: ProjectId, version_ids: impl IntoIterator, ) { - if self.removed_project_ids.contains(&project_id) { + if self.removed_project_ids.contains(&project_id) + || self + .changed_project_ids_with_all_versions + .contains(&project_id) + { return; } + self.changed_project_ids.remove(&project_id); let version_ids = version_ids.into_iter().collect::>(); if !version_ids.is_empty() { self.changed_project_versions @@ -153,6 +186,8 @@ impl PendingSearchIndexOperations { fn push_project_removal(&mut self, project_id: ProjectId) { self.changed_project_ids.remove(&project_id); + self.changed_project_ids_with_all_versions + .remove(&project_id); self.changed_project_versions.remove(&project_id); self.removed_project_ids.insert(project_id); } @@ -162,6 +197,9 @@ impl PendingSearchIndexOperations { SearchProjectIndexQueueEventData::Change { project_id } => { self.push_project_change(project_id) } + SearchProjectIndexQueueEventData::ChangeWithAllVersions { + project_id, + } => self.push_project_with_all_versions_change(project_id), SearchProjectIndexQueueEventData::VersionChange { project_id, version_ids, @@ -175,6 +213,7 @@ impl PendingSearchIndexOperations { fn into_events(self) -> Vec { let mut events = Vec::with_capacity( self.changed_project_ids.len() + + self.changed_project_ids_with_all_versions.len() + self.changed_project_versions.len() + self.removed_project_ids.len(), ); @@ -185,6 +224,15 @@ impl PendingSearchIndexOperations { events.extend(self.changed_project_ids.into_iter().map(|project_id| { SearchProjectIndexQueueEventData::Change { project_id } })); + events.extend( + self.changed_project_ids_with_all_versions.into_iter().map( + |project_id| { + SearchProjectIndexQueueEventData::ChangeWithAllVersions { + project_id, + } + }, + ), + ); events.extend(self.changed_project_versions.into_iter().map( |(project_id, version_ids)| { SearchProjectIndexQueueEventData::VersionChange { @@ -202,6 +250,8 @@ impl PendingSearchIndexOperations { pub enum SearchProjectIndexQueueEventData { #[serde(rename = "project_change")] Change { project_id: ProjectId }, + #[serde(rename = "project_change_with_all_versions")] + ChangeWithAllVersions { project_id: ProjectId }, #[serde(rename = "project_version_change")] VersionChange { project_id: ProjectId, diff --git a/apps/labrinth/src/search/incremental/consume.rs b/apps/labrinth/src/search/incremental/consume.rs index 2533e15f14..0872dfea77 100644 --- a/apps/labrinth/src/search/incremental/consume.rs +++ b/apps/labrinth/src/search/incremental/consume.rs @@ -21,7 +21,10 @@ use crate::{ SearchBackend, SearchDocumentBatch, SearchIndexUpdate, UploadSearchProject, incremental::SEARCH_PROJECT_INDEX_QUEUE_TOPIC, - indexing::{build_project_documents, build_version_change_documents}, + indexing::{ + build_project_and_all_version_documents, build_project_documents, + build_version_change_documents, + }, }, util::kafka::{ INCREMENTAL_INDEX_SEARCH_TASK, KAFKA_OPERATION_INTERVAL, @@ -132,6 +135,7 @@ async fn consume_batch( let start = Instant::now(); let mut project_ids_to_change = HashSet::new(); + let mut project_ids_with_all_versions_to_change = HashSet::new(); let mut project_ids_with_version_changes = HashSet::new(); let mut project_ids_to_remove = HashSet::new(); let mut version_ids_to_change = HashSet::new(); @@ -180,6 +184,11 @@ async fn consume_batch( SearchProjectIndexQueueEventData::Change { project_id } => { project_ids_to_change.insert(project_id); } + SearchProjectIndexQueueEventData::ChangeWithAllVersions { + project_id, + } => { + project_ids_with_all_versions_to_change.insert(project_id); + } SearchProjectIndexQueueEventData::VersionChange { project_id, version_ids, @@ -198,16 +207,26 @@ async fn consume_batch( project_ids_to_change .retain(|project_id| !project_ids_to_remove.contains(project_id)); + project_ids_with_all_versions_to_change + .retain(|project_id| !project_ids_to_remove.contains(project_id)); project_ids_with_version_changes .retain(|project_id| !project_ids_to_remove.contains(project_id)); + project_ids_with_version_changes.retain(|project_id| { + !project_ids_with_all_versions_to_change.contains(project_id) + }); project_ids_to_change.retain(|project_id| { !project_ids_with_version_changes.contains(project_id) + && !project_ids_with_all_versions_to_change.contains(project_id) }); let project_ids_to_change = project_ids_to_change.into_iter().collect::>(); let project_ids_with_version_changes = project_ids_with_version_changes .into_iter() .collect::>(); + let project_ids_with_all_versions_to_change = + project_ids_with_all_versions_to_change + .into_iter() + .collect::>(); let mut project_ids_to_remove = project_ids_to_remove.into_iter().collect::>(); let version_ids_to_change = @@ -215,9 +234,10 @@ async fn consume_batch( info!( kafka.message_count = messages_to_commit.len(), - "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with {} version changes, and {} projects to remove", + "Read all Kafka messages in {:.2?}, found {} projects to change, {} projects with all versions to change, {} projects with {} version changes, and {} projects to remove", start.elapsed(), project_ids_to_change.len(), + project_ids_with_all_versions_to_change.len(), project_ids_with_version_changes.len(), version_ids_to_change.len(), project_ids_to_remove.len(), @@ -225,6 +245,35 @@ async fn consume_batch( let start = Instant::now(); let mut documents = SearchDocumentBatch::default(); + if !project_ids_with_all_versions_to_change.is_empty() { + let operation_start = Instant::now(); + let changed_documents = build_project_and_all_version_documents( + ro_pool, + redis_pool, + &project_ids_with_all_versions_to_change, + ) + .instrument(info_span!( + "index", + batch_size = project_ids_with_all_versions_to_change.len() + )) + .await + .wrap_err_with(|| { + format!( + "failed to build search documents for {} projects and all their versions", + project_ids_with_all_versions_to_change.len() + ) + })?; + project_ids_to_remove + .extend(project_ids_with_all_versions_to_change.iter().copied()); + documents.projects.extend(changed_documents.projects); + documents.versions.extend(changed_documents.versions); + info!( + project_count = project_ids_with_all_versions_to_change.len(), + "Built changed projects and all their versions in {:.2?}", + operation_start.elapsed() + ); + } + if !project_ids_with_version_changes.is_empty() { let operation_start = Instant::now(); let changed_documents = build_version_change_documents( @@ -402,6 +451,8 @@ enum SearchProjectIndexQueueEvent { enum SearchProjectIndexQueueEventData { #[serde(rename = "project_change")] Change { project_id: ProjectId }, + #[serde(rename = "project_change_with_all_versions")] + ChangeWithAllVersions { project_id: ProjectId }, #[serde(rename = "project_version_change")] VersionChange { project_id: ProjectId, diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index 658db2eacb..1f5319633d 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -129,11 +129,22 @@ pub async fn build_project_documents( project_ids: &[ProjectId], ) -> eyre::Result> { let version_ids = HashSet::new(); - Ok( - build_search_document_batch(pool, redis, project_ids, &version_ids) - .await? - .projects, + Ok(build_search_document_batch( + pool, + redis, + project_ids, + Some(&version_ids), ) + .await? + .projects) +} + +pub async fn build_project_and_all_version_documents( + pool: &PgPool, + redis: &RedisPool, + project_ids: &[ProjectId], +) -> eyre::Result { + build_search_document_batch(pool, redis, project_ids, None).await } pub async fn build_version_change_documents( @@ -147,14 +158,15 @@ pub async fn build_version_change_documents( .copied() .map(DBVersionId::from) .collect::>(); - build_search_document_batch(pool, redis, project_ids, &version_ids).await + build_search_document_batch(pool, redis, project_ids, Some(&version_ids)) + .await } async fn build_search_document_batch( pool: &PgPool, redis: &RedisPool, project_ids: &[ProjectId], - version_ids: &HashSet, + version_ids: Option<&HashSet>, ) -> eyre::Result { let searchable_statuses = searchable_statuses(); let project_ids = project_ids @@ -194,7 +206,7 @@ async fn build_search_document_batch( .await .wrap_err("failed to fetch project")?; - build_search_documents(pool, redis, db_projects, Some(version_ids)).await + build_search_documents(pool, redis, db_projects, version_ids).await } async fn build_search_documents( @@ -612,231 +624,219 @@ async fn build_search_documents( .map(|dependency| dependency.project_id.clone()) .collect::>(); - if let Some(versions) = versions.remove(&project.id) { - let Some(latest_version) = versions.iter().max_by(|a, b| { - a.date_published - .cmp(&b.date_published) - .then_with(|| a.id.0.cmp(&b.id.0)) - }) else { - continue; - }; + let versions = versions.remove(&project.id).unwrap_or_default(); + let latest_version = versions.iter().max_by(|a, b| { + a.date_published + .cmp(&b.date_published) + .then_with(|| a.id.0.cmp(&b.id.0)) + }); - let project_version_fields = versions - .iter() - .flat_map(|x| x.version_fields.clone()) - .collect::>(); - let aggregated_version_fields = VersionField::from_query_json( - project_version_fields, + let project_version_fields = versions + .iter() + .flat_map(|x| x.version_fields.clone()) + .collect::>(); + let aggregated_version_fields = VersionField::from_query_json( + project_version_fields, + &loader_field_definitions, + &loader_field_enum_values, + true, + ); + let unvectorized_loader_fields = aggregated_version_fields + .iter() + .map(|field| { + (field.field_name.clone(), field.value.serialize_internal()) + }) + .collect(); + let mut loader_fields = + from_duplicate_version_fields(aggregated_version_fields); + let project_loader_fields = loader_fields.clone(); + + let mut project_loaders = versions + .iter() + .flat_map(|x| x.loaders.clone()) + .collect::>(); + project_loaders.sort(); + project_loaders.dedup(); + + let mut project_types = versions + .iter() + .flat_map(|x| x.project_types.clone()) + .collect::>(); + project_types.sort(); + project_types.dedup(); + exp::compat::correct_project_types( + &project.components, + &mut project_types, + ); + + let project_id = ProjectId::from(project.id).to_string(); + version_uploads.extend(versions.iter().filter_map(|version| { + if version_ids + .is_some_and(|version_ids| !version_ids.contains(&version.id)) + { + return None; + } + + let version_fields = VersionField::from_query_json( + version.version_fields.clone(), &loader_field_definitions, &loader_field_enum_values, - true, + false, ); - let unvectorized_loader_fields = aggregated_version_fields + let unvectorized_loader_fields = version_fields .iter() .map(|field| { (field.field_name.clone(), field.value.serialize_internal()) }) .collect(); - let mut loader_fields = - from_duplicate_version_fields(aggregated_version_fields); - let project_loader_fields = loader_fields.clone(); - - let mut project_loaders = versions - .iter() - .flat_map(|x| x.loaders.clone()) - .collect::>(); - project_loaders.sort(); - project_loaders.dedup(); - - let mut project_types = versions - .iter() - .flat_map(|x| x.project_types.clone()) - .collect::>(); - project_types.sort(); - project_types.dedup(); + let mut fields = from_duplicate_version_fields(version_fields); + let mut version_project_types = version.project_types.clone(); exp::compat::correct_project_types( &project.components, - &mut project_types, + &mut version_project_types, ); - let project_id = ProjectId::from(project.id).to_string(); - version_uploads.extend(versions.iter().filter_map(|version| { - if version_ids.is_some_and(|version_ids| { - !version_ids.contains(&version.id) - }) { - return None; - } - - let version_fields = VersionField::from_query_json( - version.version_fields.clone(), - &loader_field_definitions, - &loader_field_enum_values, - false, - ); - let unvectorized_loader_fields = version_fields - .iter() - .map(|field| { - ( - field.field_name.clone(), - field.value.serialize_internal(), - ) - }) - .collect(); - let mut fields = from_duplicate_version_fields(version_fields); - let mut version_project_types = version.project_types.clone(); - exp::compat::correct_project_types( - &project.components, - &mut version_project_types, - ); - - // SPECIAL BEHAVIOUR - // Todo: revisit. - // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category. - // These were previously considered the loader, and in v2, the loader is a category for searching. - // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories. - // The loaders are kept in the project document's aggregated loader fields as well, so that no information is lost on retrieval. - let mut version_categories = project_categories.clone(); - version_categories.extend(version.loaders.iter().cloned()); - let mrpack_loaders = fields - .get("mrpack_loaders") - .into_iter() - .flatten() - .filter_map(|value| value.as_str().map(String::from)) - .collect::>(); - version_categories.extend(mrpack_loaders); - if fields.contains_key("mrpack_loaders") { - version_categories.retain(|category| category != "mrpack"); - } - version_categories.sort(); - version_categories.dedup(); - - let (_, v2_og_project_type) = - LegacyProject::get_project_type(&version_project_types); - let (client_side, server_side) = - v2_reroute::convert_v3_side_types_to_v2_side_types( - &unvectorized_loader_fields, - Some(&v2_og_project_type), - ); - if let Ok(client_side) = serde_json::to_value(client_side) { - fields.insert("client_side".to_string(), vec![client_side]); - } - if let Ok(server_side) = serde_json::to_value(server_side) { - fields.insert("server_side".to_string(), vec![server_side]); - } - fields.retain(|field, _| { - matches!( - field.as_str(), - "environment" - | "game_versions" - | "client_side" - | "server_side" - ) - }); - - Some(UploadSearchVersion { - version_id: VersionId::from(version.id).to_string(), - project_id: project_id.clone(), - categories: version_categories, - project_types: version_project_types, - version_published_timestamp: version - .date_published - .timestamp(), - loader_fields: fields, - }) - })); - - let mut categories = project_categories.clone(); - categories.extend(project_loaders.iter().cloned()); - - let mrpack_loaders = loader_fields + // SPECIAL BEHAVIOUR + // Todo: revisit. + // For consistency with v2 searching, we consider the loader field 'mrpack_loaders' to be a category. + // These were previously considered the loader, and in v2, the loader is a category for searching. + // So to avoid breakage or awkward conversions, we just consider those loader_fields to be categories. + // The loaders are kept in the project document's aggregated loader fields as well, so that no information is lost on retrieval. + let mut version_categories = project_categories.clone(); + version_categories.extend(version.loaders.iter().cloned()); + let mrpack_loaders = fields .get("mrpack_loaders") .into_iter() .flatten() .filter_map(|value| value.as_str().map(String::from)) .collect::>(); - categories.extend(mrpack_loaders); - if loader_fields.contains_key("mrpack_loaders") { - categories.retain(|category| category != "mrpack"); + version_categories.extend(mrpack_loaders); + if fields.contains_key("mrpack_loaders") { + version_categories.retain(|category| category != "mrpack"); } - categories.sort(); - categories.dedup(); + version_categories.sort(); + version_categories.dedup(); let (_, v2_og_project_type) = - LegacyProject::get_project_type(&project_types); + LegacyProject::get_project_type(&version_project_types); let (client_side, server_side) = v2_reroute::convert_v3_side_types_to_v2_side_types( &unvectorized_loader_fields, Some(&v2_og_project_type), ); - if let Ok(client_side) = serde_json::to_value(client_side) { - loader_fields - .insert("client_side".to_string(), vec![client_side]); + fields.insert("client_side".to_string(), vec![client_side]); } if let Ok(server_side) = serde_json::to_value(server_side) { - loader_fields - .insert("server_side".to_string(), vec![server_side]); + fields.insert("server_side".to_string(), vec![server_side]); } - - let components = project - .components - .clone() - .into_query(ProjectId::from(project.id), &project_query_context) - .wrap_err("failed to populate query components")?; - let indexed_name = normalize_for_search(&project.name); - - project_uploads.push(UploadSearchProject { - version_id: crate::models::ids::VersionId::from( - latest_version.id, + fields.retain(|field, _| { + matches!( + field.as_str(), + "environment" + | "game_versions" + | "client_side" + | "server_side" ) - .to_string(), - project_id, - name: project.name, - indexed_name, - summary: project.summary, - categories, - project_categories, - display_categories, - follows: project.follows, - downloads: project.downloads, - log_downloads: (project.downloads.max(1) as f64).ln(), - icon_url: project.icon_url, - author: username.clone(), - author_id: ariadne::ids::UserId::from(user_id).to_string(), - organization: org_name, - organization_id: org_id.map(|id| { - crate::models::ids::OrganizationId::from(id).to_string() - }), - indexed_author: normalize_for_search(&username), - date_created: project.approved, - created_timestamp: project.approved.timestamp(), - date_modified: project.updated, - modified_timestamp: project.updated.timestamp(), - version_published_timestamp: latest_version - .date_published - .timestamp(), - license, - slug: project.slug, - project_types: project_types.clone(), - all_project_types: project_types, - gallery, - featured_gallery, - open_source, - color: project.color.map(|x| x as u32), - dependency_project_ids, - compatible_dependency_project_ids, - required_dependency_project_ids, - optional_dependency_project_ids, - embedded_dependency_project_ids, - incompatible_dependency_project_ids, - dependencies, - disclosure_types, - project_loader_fields, - loader_fields, - loaders: project_loaders, - components, }); + + Some(UploadSearchVersion { + version_id: VersionId::from(version.id).to_string(), + project_id: project_id.clone(), + categories: version_categories, + project_types: version_project_types, + version_published_timestamp: version.date_published.timestamp(), + loader_fields: fields, + }) + })); + + let mut categories = project_categories.clone(); + categories.extend(project_loaders.iter().cloned()); + + let mrpack_loaders = loader_fields + .get("mrpack_loaders") + .into_iter() + .flatten() + .filter_map(|value| value.as_str().map(String::from)) + .collect::>(); + categories.extend(mrpack_loaders); + if loader_fields.contains_key("mrpack_loaders") { + categories.retain(|category| category != "mrpack"); } + categories.sort(); + categories.dedup(); + + let (_, v2_og_project_type) = + LegacyProject::get_project_type(&project_types); + let (client_side, server_side) = + v2_reroute::convert_v3_side_types_to_v2_side_types( + &unvectorized_loader_fields, + Some(&v2_og_project_type), + ); + + if let Ok(client_side) = serde_json::to_value(client_side) { + loader_fields.insert("client_side".to_string(), vec![client_side]); + } + if let Ok(server_side) = serde_json::to_value(server_side) { + loader_fields.insert("server_side".to_string(), vec![server_side]); + } + + let components = project + .components + .clone() + .into_query(ProjectId::from(project.id), &project_query_context) + .wrap_err("failed to populate query components")?; + let indexed_name = normalize_for_search(&project.name); + + project_uploads.push(UploadSearchProject { + version_id: latest_version.map(|version| { + crate::models::ids::VersionId::from(version.id).to_string() + }), + project_id, + name: project.name, + indexed_name, + summary: project.summary, + categories, + project_categories, + display_categories, + follows: project.follows, + downloads: project.downloads, + log_downloads: (project.downloads.max(1) as f64).ln(), + icon_url: project.icon_url, + author: username.clone(), + author_id: ariadne::ids::UserId::from(user_id).to_string(), + organization: org_name, + organization_id: org_id.map(|id| { + crate::models::ids::OrganizationId::from(id).to_string() + }), + indexed_author: normalize_for_search(&username), + date_created: project.approved, + created_timestamp: project.approved.timestamp(), + date_modified: project.updated, + modified_timestamp: project.updated.timestamp(), + version_published_timestamp: latest_version + .map(|version| version.date_published.timestamp()), + license, + slug: project.slug, + project_types: project_types.clone(), + all_project_types: project_types, + gallery, + featured_gallery, + open_source, + color: project.color.map(|x| x as u32), + dependency_project_ids, + compatible_dependency_project_ids, + required_dependency_project_ids, + optional_dependency_project_ids, + embedded_dependency_project_ids, + incompatible_dependency_project_ids, + dependencies, + disclosure_types, + project_loader_fields, + loader_fields, + loaders: project_loaders, + components, + }); } Ok(SearchDocumentBatch { diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 7c5254cb73..0f203ded0b 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -247,7 +247,8 @@ impl FromStr for SearchBackendKind { #[derive(Serialize, Deserialize, Debug, Clone)] pub struct UploadSearchProject { /// ID of the most recently published version. - pub version_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub version_id: Option, pub project_id: String, // pub project_types: Vec, @@ -285,7 +286,8 @@ pub struct UploadSearchProject { /// Unix timestamp of the last major modification pub modified_timestamp: i64, /// Unix timestamp of the most recently published version. - pub version_published_timestamp: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub version_published_timestamp: Option, pub open_source: bool, #[serde(skip_serializing_if = "Option::is_none")] pub color: Option, @@ -369,7 +371,8 @@ pub struct SearchResults { #[derive(Serialize, Deserialize, Debug, Clone, ToSchema)] pub struct ResultSearchProject { /// ID of the most recently published version. - pub version_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version_id: Option, pub project_id: String, pub project_types: Vec, #[serde(default)] diff --git a/docker-compose.yml b/docker-compose.yml index a488c7fee0..d33ff19224 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -578,6 +578,8 @@ services: volumes: - ./apps/labrinth/nginx/meili-lb.conf:/etc/nginx/conf.d/default.conf:ro networks: + default: + driver: bridge elasticsearch-mesh: driver: bridge meilisearch-mesh: diff --git a/packages/api-client/src/modules/labrinth/types.ts b/packages/api-client/src/modules/labrinth/types.ts index 071e5b5edc..d127526687 100644 --- a/packages/api-client/src/modules/labrinth/types.ts +++ b/packages/api-client/src/modules/labrinth/types.ts @@ -1899,7 +1899,7 @@ export namespace Labrinth { export namespace v3 { export interface ResultSearchProject { - version_id: string + version_id?: string project_id: string project_types: string[] all_project_types: string[]