Commit aeea2067 authored by Jean-Baptiste Nizet's avatar Jean-Baptiste Nizet
Browse files

feat: implement service to list pillars

parent 8e1d8d70
......@@ -5,6 +5,7 @@ import java.util.List;
import fr.inra.urgi.rare.domain.GeneticResource;
import fr.inra.urgi.rare.domain.IndexedGeneticResource;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
......@@ -14,6 +15,9 @@ import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
*/
public interface GeneticResourceDaoCustom {
String DATABASE_SOURCE_AGGREGATION_NAME = "databaseSource";
String PORTAL_URL_AGGREGATION_NAME = "portalURL";
/**
* Searches for the given text anywhere (except in identifier, URL and numeric fields) in the genetic resources,
* and returns the requested page (results are sorted by score, in descending order).
......@@ -41,4 +45,6 @@ public interface GeneticResourceDaoCustom {
* a specific DAO would do.
*/
void saveAll(Collection<IndexedGeneticResource> indexedGeneticResources);
Terms findPillars();
}
package fr.inra.urgi.rare.dao;
import static org.elasticsearch.index.query.QueryBuilders.boolQuery;
import static org.elasticsearch.index.query.QueryBuilders.multiMatchQuery;
import static org.elasticsearch.index.query.QueryBuilders.termsQuery;
import static org.elasticsearch.index.query.QueryBuilders.*;
import java.util.Collection;
import java.util.Collections;
......@@ -12,16 +10,20 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import fr.inra.urgi.rare.domain.GeneticResource;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import fr.inra.urgi.rare.domain.IndexedGeneticResource;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestBuilders;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.aggregation.AggregatedPage;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder;
......@@ -143,10 +145,88 @@ public class GeneticResourceDaoImpl implements GeneticResourceDaoCustom {
elasticsearchTemplate.refresh(elasticsearchTemplate.getPersistentEntityFor(GeneticResource.class).getIndexName());
}
@Override
public Terms findPillars() {
String pillarAggregationName = "pillar";
TermsAggregationBuilder pillar =
AggregationBuilders.terms(pillarAggregationName).field("pillarName.keyword").size(100);
TermsAggregationBuilder databaseSource =
AggregationBuilders.terms(DATABASE_SOURCE_AGGREGATION_NAME).field("databaseSource.keyword").size(100);
TermsAggregationBuilder portalURL =
AggregationBuilders.terms(PORTAL_URL_AGGREGATION_NAME).field("portalURL.keyword").size(2);
databaseSource.subAggregation(portalURL);
pillar.subAggregation(databaseSource);
NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder()
.withQuery(new MatchAllQueryBuilder())
.addAggregation(pillar)
.withPageable(NoPage.INSTANCE);
AggregatedPage<GeneticResource> geneticResources = elasticsearchTemplate.queryForPage(builder.build(),
GeneticResource.class);
return geneticResources.getAggregations().get(pillarAggregationName);
}
private IndexQuery createIndexQuery(IndexedGeneticResource entity) {
IndexQuery query = new IndexQuery();
query.setObject(entity);
query.setId(entity.getGeneticResource().getId());
return query;
}
/**
* A Pageable implementation allowing to avoid loading any page (i.e. with a size equal to 0), because we
* are not interested in loading search results, but only the aggregations
* (see https://www.elastic.co/guide/en/elasticsearch/reference/6.3/returning-only-agg-results.html).
*
* We would normally use a {@link org.springframework.data.domain.PageRequest} as an implementation of
* {@link Pageable}, but PageRequest considers 0 as an invalid size. Hence this implementation.
*/
private static final class NoPage implements Pageable {
public static final NoPage INSTANCE = new NoPage();
private NoPage() {
}
@Override
public int getPageNumber() {
return 0;
}
@Override
public int getPageSize() {
return 0;
}
@Override
public long getOffset() {
return 0;
}
@Override
public Sort getSort() {
return null;
}
@Override
public Pageable next() {
throw new UnsupportedOperationException();
}
@Override
public Pageable previousOrFirst() {
throw new UnsupportedOperationException();
}
@Override
public Pageable first() {
throw new UnsupportedOperationException();
}
@Override
public boolean hasPrevious() {
return false;
}
}
}
package fr.inra.urgi.rare.pillar;
import java.util.Objects;
/**
* A database source (name, URL and document count), part of a {@link PillarDTO}
* @author JB Nizet
*/
public final class DatabaseSourceDTO {
private final String name;
private final String url;
private final long documentCount;
public DatabaseSourceDTO(String name, String url, long documentCount) {
this.name = name;
this.url = url;
this.documentCount = documentCount;
}
public String getName() {
return name;
}
public String getUrl() {
return url;
}
public long getDocumentCount() {
return documentCount;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
DatabaseSourceDTO that = (DatabaseSourceDTO) o;
return documentCount == that.documentCount &&
Objects.equals(name, that.name) &&
Objects.equals(url, that.url);
}
@Override
public int hashCode() {
return Objects.hash(name, url, documentCount);
}
@Override
public String toString() {
return "DatabaseSourceDTO{" +
"name='" + name + '\'' +
", url='" + url + '\'' +
", documentCount=" + documentCount +
'}';
}
}
package fr.inra.urgi.rare.pillar;
import java.util.List;
import java.util.stream.Collectors;
import fr.inra.urgi.rare.dao.GeneticResourceDao;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* REST Controller used to get the information, displayed in the home page, about pillars.
* @author JB Nizet
*/
@RestController
@RequestMapping("/api/pillars")
public class PillarController {
private final GeneticResourceDao geneticResourceDao;
public PillarController(GeneticResourceDao geneticResourceDao) {
this.geneticResourceDao = geneticResourceDao;
}
@GetMapping
public List<PillarDTO> list() {
Terms pillars = geneticResourceDao.findPillars();
return pillars.getBuckets()
.stream()
.map(this::toPillarDTO)
.collect(Collectors.toList());
}
private PillarDTO toPillarDTO(Bucket bucket) {
String name = bucket.getKeyAsString();
Terms databaseSourceAggregation =
bucket.getAggregations().get(GeneticResourceDao.DATABASE_SOURCE_AGGREGATION_NAME);
List<DatabaseSourceDTO> databaseSources =
databaseSourceAggregation.getBuckets()
.stream()
.map(this::toDatabaseSourceDTO)
.collect(Collectors.toList());
return new PillarDTO(name, databaseSources);
}
private DatabaseSourceDTO toDatabaseSourceDTO(Bucket bucket) {
String name = bucket.getKeyAsString();
Terms portalURLAggregation =
bucket.getAggregations().get(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME);
List<? extends Bucket> buckets = portalURLAggregation.getBuckets();
// there should be 0 bucket (if the database source has no portal URL), or 1 if it has one.
// if there are more, we only take the first one, which has the most documents: it probably means
// that the other buckets have a wrong URL
String url = buckets.size() > 0 ? buckets.get(0).getKeyAsString() : null;
return new DatabaseSourceDTO(name, url, bucket.getDocCount());
}
}
package fr.inra.urgi.rare.pillar;
import java.util.List;
import java.util.Objects;
import fr.inra.urgi.rare.util.Utils;
/**
* A pillar, with its list of database sources
* @author JB Nizet
*/
public final class PillarDTO {
/**
* The name of the pillar
*/
private final String name;
/**
* The database sources of this pillar
*/
private final List<DatabaseSourceDTO> databaseSources;
public PillarDTO(String name, List<DatabaseSourceDTO> databaseSources) {
this.name = name;
this.databaseSources = Utils.nullSafeUnmodifiableCopy(databaseSources);
}
public String getName() {
return name;
}
public List<DatabaseSourceDTO> getDatabaseSources() {
return databaseSources;
}
/**
* Returns the number of documents in this pillar. We could use the docCount retrieved by Elasticsearch,
* but since these counts are approximate, just summing the counts from database resources avoids
* wondering why the sum of the database sources document count is not equal to the pillar document count.
*/
public long getDocumentCount() {
return this.databaseSources.stream().mapToLong(DatabaseSourceDTO::getDocumentCount).sum();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
PillarDTO pillarDTO = (PillarDTO) o;
return Objects.equals(name, pillarDTO.name) &&
Objects.equals(databaseSources, pillarDTO.databaseSources);
}
@Override
public int hashCode() {
return Objects.hash(name, databaseSources);
}
@Override
public String toString() {
return "PillarDTO{" +
"name='" + name + '\'' +
", databaseSources=" + databaseSources +
'}';
}
}
......@@ -43,7 +43,13 @@
},
"portalURL": {
"type": "text",
"index": false
"index": false,
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"dataURL": {
"type": "text",
......
......@@ -348,6 +348,82 @@ class GeneticResourceDaoTest {
assertThat(taxon.getBuckets()).extracting(Bucket::getDocCount).containsOnly(1L);
}
@Test
public void shouldFindPillars() {
GeneticResource resource1 = new GeneticResourceBuilder()
.withId("r1")
.withPillarName("P1")
.withDatabaseSource("D11")
.withPortalURL("D11Url")
.build();
GeneticResource resource2 = new GeneticResourceBuilder()
.withId("r2")
.withPillarName("P1")
.withDatabaseSource("D11")
.withPortalURL("D11Url")
.build();
GeneticResource resource3 = new GeneticResourceBuilder()
.withId("r3")
.withPillarName("P1")
.withDatabaseSource("D12")
.withPortalURL("D12Url")
.build();
GeneticResource resource4 = new GeneticResourceBuilder()
.withId("r4")
.withPillarName("P2")
.withDatabaseSource("D21")
.withPortalURL("D21Url")
.build();
GeneticResource resource5 = new GeneticResourceBuilder()
.withId("r5")
.withPillarName("P2")
.withDatabaseSource("D22")
.build();
geneticResourceDao.saveAll(Arrays.asList(resource1, resource2, resource3, resource4, resource5));
Terms pillars = geneticResourceDao.findPillars();
assertThat(pillars.getBuckets()).hasSize(2);
Bucket p1 = pillars.getBucketByKey("P1");
Terms databaseSource = p1.getAggregations().get(GeneticResourceDao.DATABASE_SOURCE_AGGREGATION_NAME);
assertThat(databaseSource.getBuckets()).hasSize(2);
Bucket d11 = databaseSource.getBucketByKey("D11");
assertThat(d11.getDocCount()).isEqualTo(2);
Terms d11Url = d11.getAggregations().get(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME);
assertThat(d11Url.getBuckets()).hasSize(1);
assertThat(d11Url.getBuckets().get(0).getKeyAsString()).isEqualTo("D11Url");
Bucket d12 = databaseSource.getBucketByKey("D12");
assertThat(d12.getDocCount()).isEqualTo(1);
Terms d12Url = d12.getAggregations().get(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME);
assertThat(d12Url.getBuckets()).hasSize(1);
assertThat(d12Url.getBuckets().get(0).getKeyAsString()).isEqualTo("D12Url");
Bucket p2 = pillars.getBucketByKey("P2");
databaseSource = p2.getAggregations().get(GeneticResourceDao.DATABASE_SOURCE_AGGREGATION_NAME);
assertThat(databaseSource.getBuckets()).hasSize(2);
Bucket d21 = databaseSource.getBucketByKey("D21");
assertThat(d21.getDocCount()).isEqualTo(1);
Terms d21Url = d21.getAggregations().get(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME);
assertThat(d21Url.getBuckets()).hasSize(1);
assertThat(d21Url.getBuckets().get(0).getKeyAsString()).isEqualTo("D21Url");
Bucket d22 = databaseSource.getBucketByKey("D22");
assertThat(d22.getDocCount()).isEqualTo(1);
Terms d22Url = d22.getAggregations().get(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME);
assertThat(d22Url.getBuckets()).isEmpty();
}
@Nested
class RefinementTest {
@BeforeEach
......
package fr.inra.urgi.rare.pillar;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import java.util.Arrays;
import java.util.Collections;
import fr.inra.urgi.rare.config.SecurityConfig;
import fr.inra.urgi.rare.dao.GeneticResourceDao;
import fr.inra.urgi.rare.search.MockBucket;
import fr.inra.urgi.rare.search.MockTermsAggregation;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.hamcrest.CoreMatchers;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.context.annotation.Import;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.web.servlet.MockMvc;
/**
* Unit tests for {@link PillarController}
* @author JB Nizet
*/
@ExtendWith(SpringExtension.class)
@Import(SecurityConfig.class)
@WebMvcTest(controllers = PillarController.class)
class PillarControllerTest {
@MockBean
private GeneticResourceDao mockGeneticResourceDao;
@Autowired
private MockMvc mockMvc;
@Test
void shouldListPillars() throws Exception {
Terms pillarTerms = new MockTermsAggregation("pillar", Arrays.asList(
new MockBucket("Plant", 999, new Aggregations(Arrays.asList( // wrong approximate count
new MockTermsAggregation(GeneticResourceDao.DATABASE_SOURCE_AGGREGATION_NAME, Arrays.asList(
new MockBucket("Florilège", 800, new Aggregations(Arrays.asList(
new MockTermsAggregation(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME, Collections.singletonList(
new MockBucket("http://florilege.arcad-project.org/fr/collections", 800)
))
))),
new MockBucket("CNRGV", 200, new Aggregations(Arrays.asList(
new MockTermsAggregation(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME, Arrays.asList(
new MockBucket("https://cnrgv.toulouse.inra.fr/library/genomic_resource/ Aha-B-H25", 799),
new MockBucket("https://cnrgv.toulouse.inra.fr/library/genomic_resource/ Aha-B-H26", 1) // 2 URLS instead of 1
))
)))
))
))),
new MockBucket("Forest", 500, new Aggregations(Arrays.asList(
new MockTermsAggregation(GeneticResourceDao.DATABASE_SOURCE_AGGREGATION_NAME, Arrays.asList(
new MockBucket("GnpIS", 500, new Aggregations(Arrays.asList(
new MockTermsAggregation(GeneticResourceDao.PORTAL_URL_AGGREGATION_NAME, Collections.emptyList()) // no URL
)))
))
)))
));
when(mockGeneticResourceDao.findPillars()).thenReturn(pillarTerms);
mockMvc.perform(get("/api/pillars"))
.andExpect(status().isOk())
.andExpect(jsonPath("$").isArray())
.andExpect(jsonPath("$[0].name").value("Plant"))
.andExpect(jsonPath("$[0].documentCount").value(1000))
.andExpect(jsonPath("$[0].databaseSources").isArray())
.andExpect(jsonPath("$[0].databaseSources[0].name").value("Florilège"))
.andExpect(jsonPath("$[0].databaseSources[0].documentCount").value(800))
.andExpect(jsonPath("$[0].databaseSources[0].url")
.value("http://florilege.arcad-project.org/fr/collections"))
.andExpect(jsonPath("$[0].databaseSources[1].name").value("CNRGV"))
.andExpect(jsonPath("$[0].databaseSources[1].documentCount").value(200))
.andExpect(jsonPath("$[0].databaseSources[1].url")
.value("https://cnrgv.toulouse.inra.fr/library/genomic_resource/ Aha-B-H25"))
.andExpect(jsonPath("$[1].name").value("Forest"))
.andExpect(jsonPath("$[1].documentCount").value(500))
.andExpect(jsonPath("$[1].databaseSources").isArray())
.andExpect(jsonPath("$[1].databaseSources[0].name").value("GnpIS"))
.andExpect(jsonPath("$[1].databaseSources[0].documentCount").value(500))
.andExpect(jsonPath("$[1].databaseSources[0].url").value(CoreMatchers.nullValue()));
}
}
......@@ -15,10 +15,16 @@ public final class MockBucket implements Bucket {
private final String key;
private final long docCount;
private final Aggregations aggregations;
public MockBucket(String key, long docCount) {
this(key, docCount, null);
}
public MockBucket(String key, long docCount, Aggregations subAggregations) {
this.key = key;
this.docCount = docCount;
this.aggregations = subAggregations;
}
@Override
......@@ -48,7 +54,7 @@ public final class MockBucket implements Bucket {
@Override
public Aggregations getAggregations() {
throw new UnsupportedOperationException();
return this.aggregations;
}
@Override
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment