Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public class DateContext {
* The generation of the contexts happens for each resolution with their mapped alignment.
* The returned list is primarily sorted in the order of the given resolutions and secondarily by the temporal
* succession of the contexts, e.g.: with resolutions YEARS, QUARTERS given the list would first contain the
* ascending year ranges and than the quarter ranges. The alignment references always the lower bound of the
* ascending year ranges and then the quarter ranges. The alignment references always the lower bound of the
* dateRangeMask.
* @param dateRangeMask The mask in which the contexts are generated
* @param resolutionAndAlignment The resolutions to produce and their alignment
Expand Down Expand Up @@ -125,7 +125,7 @@ public static Function<CDateRange,List<CDateRange>> getDateRangeSubdivider(Align

if (alignedSubdivisionCount % alignedPerResolution != 1) {
// The loop did not fullfill the resolution-sized subdivision it begun
result.add(alignRef.makeMergedRange(alignedSubdivisions.get(alignedSubdivisions.size() - 1), interestingDate));
result.add(alignRef.makeMergedRange(alignedSubdivisions.getLast(), interestingDate));
}

return alignRef.getAlignedIterationDirection(result);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,18 +124,12 @@ public FrontendPreviewConfig getEntityPreviewFrontendConfig(DatasetId dataset) {
* The user will upload a file and expect only well-corresponding resolutions.
*/
public ResolvedFilterValues resolveFilterValues(FilterId filterId, List<String> searchTerms) {
SelectFilter<?> filter = (SelectFilter<?>) filterId.resolve();


final SelectFilter<?> filter = (SelectFilter<?>) filterId.resolve();
final Namespace namespace = namespaces.get(filter.getDataset());

SearchProcessor filterSearch = namespace.getFilterSearch();

final ExactFilterValueResult exactResult = filterSearch.findExact(filter, searchTerms);

final ConnectorId connectorId = filter.getConnector().getId();
final ExactFilterValueResult exactResult = namespace.getFilterSearch().findExact(filter, searchTerms);

return new ResolvedFilterValues(new ResolvedFilterResult(connectorId, filter.getId().toString(), exactResult.resolved), exactResult.unresolved);
return new ResolvedFilterValues(new ResolvedFilterResult(filterId.getConnector(), filter.getId().toString(), exactResult.resolved), exactResult.unresolved);
}

public AutoCompleteResult autocompleteTextFilter(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,8 @@ static List<String> extractKeywords(FrontendValue value) {


/**
* Query for an exact matching {@link FrontendValue}.
* Query for an exact matching {@link FrontendValue}, per search term return only the first match by priority of search sources.
*
* Matches {@link FrontendValue#getValue()} or {@link FrontendValue#getLabel()} but case-insensitive.
* @param filter The filter to the resulting value must correspond to (domain of the {@link FrontendValue})
* @param searchTerms The exact terms to match
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,13 +292,13 @@ public List<FrontendValue> findExact(SelectFilter<?> filter, String searchTerm)
return out;
}


@Override
public ConceptsProcessor.ExactFilterValueResult findExact(SelectFilter<?> filter, List<String> searchTerms) {
final List<FrontendValue> out = new ArrayList<>();
// search in the full text engine
final Set<String> openSearchTerms = new HashSet<>(searchTerms);


for (final Iterator<String> iterator = openSearchTerms.iterator(); iterator.hasNext(); ) {

final String searchTerm = iterator.next();
Expand All @@ -309,7 +309,7 @@ public ConceptsProcessor.ExactFilterValueResult findExact(SelectFilter<?> filter
}

iterator.remove();
out.addAll(results);
out.add(results.getFirst());
}

return new ConceptsProcessor.ExactFilterValueResult(out, openSearchTerms);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,9 @@ We chunk the values for resolving here so that the request does not bust any URI
return new ConceptsProcessor.ExactFilterValueResult(List.of(), terms);
}

// We are matching on label and value.
// So if for reason a value is present in multiple sources (map, template, ...) but has different labels
// both can be found.
String collect = Stream.of(
SolrFrontendValue.Fields.value_s,
SolrFrontendValue.Fields.label_t
Expand All @@ -217,7 +220,8 @@ We chunk the values for resolving here so that the request does not bust any URI
try {
List<FrontendValue> resolvedValues = new ArrayList<>();

SolrQuery solrQuery = buildSolrQuery(collect, 0, batchSize, false, false, false);
// We sort to return value with the highest source priority and get the best description
SolrQuery solrQuery = buildSolrQuery(collect, 0, batchSize, true, false, false);

String decodedQuery = URLDecoder.decode(String.valueOf(solrQuery), StandardCharsets.UTF_8);
int queryHash = decodedQuery.hashCode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,20 +73,32 @@ public class SolrProcessor implements SearchProcessor, Managed {
private final FilterValueConfig filterValueConfig;

private final Map<String, FilterValueIndexer> indexers = new ConcurrentHashMap<>();

private SolrClient solrSearchClient;

private SolrClient solrIndexClient;

/**
* Single threaded runtime for the chunk submitter.
* This is mainly used to decouple mina threads from the solr client in order to prevent blocking and to convert between {@link com.bakdata.conquery.models.messages.namespaces.specific.RegisterColumnValues}'s
* and solr chunk sizes.
*/
private final ExecutorService chunkDecoupleExecutor = Executors.newSingleThreadExecutor(new ThreadFactoryBuilder()
.setNameFormat("solr-submitter-%d")
.setDaemon(true)
.build());
.setNameFormat("solr-submitter-%d")
.setDaemon(true)
.build());
private SolrClient solrSearchClient;
private SolrClient solrIndexClient;

/**
* Higher priority for lower number
* @param searchable The searchable entity that is map to a priority
* @return The priority
*/
static int getFilterValueSourcePriority(Searchable searchable) {
return switch (searchable) {
case SolrEmptySeachable ignore -> 0;
case LabelMap ignore -> 1;
case FilterTemplate ignore -> 2;
case Column ignore -> 3;
default -> Integer.MAX_VALUE;
};
}

@Override
public void start() throws Exception {
Expand All @@ -102,14 +114,16 @@ private synchronized void refreshClients() {
if (solrSearchClient != null) {
try {
solrSearchClient.close();
} catch (Exception e) {
}
catch (Exception e) {
log.warn("Failed to close solr search client", e);
}
}
if (solrIndexClient != null) {
try {
solrIndexClient.close();
} catch (Exception e) {
}
catch (Exception e) {
log.warn("Failed to close solr index client", e);
}
}
Expand All @@ -134,7 +148,8 @@ public void clearSearch() {
try (SolrClient solrClient = solrSearchClientFactory.get()) {
log.info("Clearing collection: {}", solrClient.getDefaultCollection());
solrClient.deleteByQuery("*:*");
} catch (SolrServerException | IOException e) {
}
catch (SolrServerException | IOException e) {
throw new RuntimeException(e);
}
}
Expand Down Expand Up @@ -180,21 +195,6 @@ private String getNameFromColumn(Column column) {
return "shared_column_" + columnGroup;
}

/**
* Higher priority for lower number
* @param searchable The searchable entity that is map to a priority
* @return The priority
*/
static int getFilterValueSourcePriority(Searchable searchable) {
return switch (searchable) {
case SolrEmptySeachable ignore -> 0;
case LabelMap ignore -> 1;
case FilterTemplate ignore -> 2;
case Column ignore -> 3;
default -> Integer.MAX_VALUE;
};
}

/*package*/ FilterValueIndexer getIndexerFor(Searchable searchable) {
String nameForSearchable = buildNameForSearchable(searchable);
int sourcePriority = getFilterValueSourcePriority(searchable);
Expand Down Expand Up @@ -231,15 +231,16 @@ public AutoCompleteResult topItems(SelectFilter<?> filter, String text, Integer
}

@Override
public void indexManagerResidingSearches(Set<Searchable> managerSearchables, AtomicBoolean cancelledState, ProgressReporter progressReporter) throws InterruptedException {
public void indexManagerResidingSearches(Set<Searchable> managerSearchables, AtomicBoolean cancelledState, ProgressReporter progressReporter)
throws InterruptedException {

// Index an empty result for all searches
indexEmptyLabel();

progressReporter.setMax(managerSearchables.size());
progressReporter.start();
// Most computations are cheap but data intensive: we fork here to use as many cores as possible.
try(final ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)) {
try (final ExecutorService service = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)) {

final Map<Searchable, Search<FrontendValue>> searchCache = new ConcurrentHashMap<>();
for (Searchable searchable : managerSearchables) {
Expand Down Expand Up @@ -357,10 +358,6 @@ private void indexFilterTemplate(FilterValueIndexer search, FilterTemplate temp)
}
}

public Collection<FrontendValue> findExact(SelectFilter<?> filter, String searchTerm) {

return findExact(filter, List.of(searchTerm)).resolved();
}

@Override
public ConceptsProcessor.ExactFilterValueResult findExact(SelectFilter<?> filter, List<String> searchTerms) {
Expand All @@ -371,7 +368,6 @@ public ConceptsProcessor.ExactFilterValueResult findExact(SelectFilter<?> filter

@Override
public AutoCompleteResult query(SelectFilter<?> filter, String maybeText, int itemsPerPage, int pageNumber) {

int start = itemsPerPage * pageNumber;
return topItems(filter, maybeText, start, itemsPerPage);
}
Expand Down Expand Up @@ -406,7 +402,7 @@ public void execute() throws Exception {
try {

Stopwatch stopwatch = Stopwatch.createStarted();
try(ExecutorService executorService = Executors.newFixedThreadPool(4)) {
try (ExecutorService executorService = Executors.newFixedThreadPool(4)) {
for (ColumnId columnId : columns) {
executorService.submit(() -> {
solrProcessor.finalizeSearch(columnId.resolve());
Expand All @@ -417,7 +413,8 @@ public void execute() throws Exception {
solrProcessor.explicitCommit();
getProgressReporter().report(1);
log.info("Finished commit on collection {} in {}", solrProcessor.solrIndexClient.getDefaultCollection(), stopwatch);
} catch (Exception e) {
}
catch (Exception e) {
log.error("Unable to issue explicit commit on collection {}", solrProcessor.solrIndexClient.getDefaultCollection(), e);
}
}
Expand Down
Loading
Loading