Commit abefa05c authored by Jean-Baptiste Nizet's avatar Jean-Baptiste Nizet Committed by Cyril Pommier
Browse files

chore: upgrade to elastic 7.13.2

parent 5eca149a
......@@ -22,7 +22,7 @@ variables:
JAR_PATH: "backend/build/libs/${APP_NAME}.jar"
GIT_DEPTH: 0
IMAGE_TAG: $CI_COMMIT_REF_SLUG
ELASTIC_VERSION: "6.6.2"
ELASTIC_VERSION: "7.13.2"
DOCKER_OPTS: "--mtu=1450"
# Gradle cache for all jobs
......@@ -50,6 +50,7 @@ build-loader-docker-image:
- if [ "master" == "${CI_COMMIT_REF_SLUG}" ] ; then docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest ; fi;
only:
changes:
- Dockerfile
- scripts/*
- backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json
- backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json
......
......@@ -3,7 +3,7 @@
"files": "frontend/package-lock.json|^.secrets.baseline$",
"lines": null
},
"generated_at": "2021-08-02T15:23:45Z",
"generated_at": "2021-10-01T17:08:35Z",
"plugins_used": [
{
"name": "AWSKeyDetector"
......@@ -58,14 +58,14 @@
"hashed_secret": "dd447c7c799dd4ebaacca8f0ad3da45a097d7211",
"is_secret": false,
"is_verified": false,
"line_number": 228,
"line_number": 204,
"type": "Base64 High Entropy String"
},
{
"hashed_secret": "8074db38f8a8acec1a147bc5daf2799ff6693fff",
"is_secret": false,
"is_verified": false,
"line_number": 247,
"line_number": 248,
"type": "Base64 High Entropy String"
}
],
......@@ -412,15 +412,6 @@
"type": "Base64 High Entropy String"
}
],
"backend/src/main/java/fr/inra/urgi/faidare/domain/xref/XRefDocumentSearchCriteria.java": [
{
"hashed_secret": "115a9779753641744ba0e30c3ff8586c823b6f7e",
"is_secret": false,
"is_verified": false,
"line_number": 18,
"type": "Base64 High Entropy String"
}
],
"backend/src/main/java/fr/inra/urgi/faidare/elasticsearch/query/impl/ESGenericQueryFactory.java": [
{
"hashed_secret": "084c833a01fb1d770f994aaec6a07f282e41827a",
......@@ -2552,6 +2543,22 @@
"type": "Base64 High Entropy String"
}
],
"gradle.properties": [
{
"hashed_secret": "93220646d2e029229894cf6f9e5d170c8fc59dbb",
"is_secret": true,
"is_verified": false,
"line_number": 5,
"type": "Base64 High Entropy String"
},
{
"hashed_secret": "93220646d2e029229894cf6f9e5d170c8fc59dbb",
"is_secret": true,
"is_verified": false,
"line_number": 5,
"type": "Hex High Entropy String"
}
],
"scripts/harvest.sh": [
{
"hashed_secret": "9b4ea0964706f977148ec989e7373d9622613547",
......
......@@ -2,6 +2,7 @@ FROM alpine
LABEL Author="Raphaël FLORES <raphael.flores@inrae.fr>"
COPY scripts/harvest.sh /opt/scripts/
COPY scripts/to_bulk.jq /opt/scripts/
# COPY dao settings
COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json
......
......@@ -110,8 +110,8 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-thymeleaf")
// Elasticsearch
implementation("org.elasticsearch:elasticsearch:6.6.2")
implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.6.2")
implementation("org.elasticsearch:elasticsearch:7.13.2")
implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:7.13.2")
// Swagger
implementation("io.swagger:swagger-annotations:1.5.21")
......
......@@ -31,12 +31,12 @@ public class XRefDocumentController {
@ApiOperation("Find xref documents")
@GetMapping(value = "/faidare/v1/xref/documentbyfulltextid")
public PaginatedList<XRefDocumentVO> documentByFullTextId(
@RequestParam(required = false, value = "entry_type") String entryType,
@RequestParam(required = false) List<String> linkedRessourcesID
@RequestParam(required = false, value = "entryType") String entryType,
@RequestParam(required = false) List<String> linkedResourcesID
) {
XRefDocumentSearchCriteria criteria = new XRefDocumentSearchCriteria();
criteria.setEntryType(entryType);
criteria.setLinkedRessourcesID(linkedRessourcesID);
criteria.setLinkedResourcesID(linkedResourcesID);
return repository.find(criteria);
}
......
......@@ -5,7 +5,7 @@ package fr.inra.urgi.faidare.domain.xref;
*/
public enum DocumentFields {
ENTRY_TYPE("entry_type"), LINKEDRESSOURCESID("linkedRessourcesID");
ENTRY_TYPE("entryType"), LINKED_RESOURCES_ID("linkedResourcesID");
private String field;
......
......@@ -13,15 +13,15 @@ import java.util.List;
@CriteriaForDocument(XRefDocumentVO.class)
public class XRefDocumentSearchCriteria extends PaginationCriteriaImpl {
@DocumentPath("entry_type")
@DocumentPath("entryType")
private String entryType;
@DocumentPath("linkedRessourcesID")
private List<String> linkedRessourcesID;
@DocumentPath("linkedResourcesID") // pragma: allowlist secret
private List<String> linkedResourcesID;
public static XRefDocumentSearchCriteria forXRefId(String resourceId) {
XRefDocumentSearchCriteria criteria = new XRefDocumentSearchCriteria();
criteria.setLinkedRessourcesID(Collections.singletonList(resourceId));
criteria.setLinkedResourcesID(Collections.singletonList(resourceId));
return criteria;
}
......@@ -33,11 +33,11 @@ public class XRefDocumentSearchCriteria extends PaginationCriteriaImpl {
this.entryType = entryType;
}
public List<String> getLinkedRessourcesID() {
return linkedRessourcesID;
public List<String> getLinkedResourcesID() {
return linkedResourcesID;
}
public void setLinkedRessourcesID(List<String> linkedRessourcesID) {
this.linkedRessourcesID = linkedRessourcesID;
public void setLinkedResourcesID(List<String> linkedResourcesID) {
this.linkedResourcesID = linkedResourcesID;
}
}
......@@ -11,23 +11,20 @@ import java.util.List;
*
* @author fphilippe
*/
@Document(type = "transplant")
@Document(type = "xref")
public class XRefDocumentVO {
@JsonProperty("group_id")
@JsonProperty("groupId")
private String groupId;
@JsonProperty("entry_type")
@JsonProperty("entryType")
private String entryType;
@JsonProperty("database_name")
@JsonProperty("databaseName")
private String databaseName;
@JsonProperty("db_id")
private String dbId;
@JsonProperty("db_version")
private String dbVersion;
@JsonProperty("identifier")
private String identifier;
@JsonProperty("name")
private String name;
......@@ -41,58 +38,7 @@ public class XRefDocumentVO {
@JsonProperty("species")
private String species;
@JsonProperty("xref")
private String xref;
@JsonProperty("feature_type")
private String featureType;
@JsonProperty("sequence_id")
private String sequenceId;
@JsonProperty("sequence_version")
private String sequence_version;
@JsonProperty("start_position")
private String startPosition;
@JsonProperty("end_position")
private String endPosition;
@JsonProperty("map")
private String map;
@JsonProperty("map_position")
private String mapPosition;
@JsonProperty("authority")
private String authority;
@JsonProperty("trait")
private String trait;
@JsonProperty("trait_id")
private String traitId;
@JsonProperty("environment")
private String environment;
@JsonProperty("environment_id")
private String environmentId;
@JsonProperty("statistic")
private String statistic;
@JsonProperty("unit")
private String unit;
@JsonProperty("genotype")
private String genotype;
@JsonProperty("experiment_type")
private String experimentType;
private List<String> linkedRessourcesID;
private List<String> linkedResourcesID;
public String getGroupId() {
return groupId;
......@@ -118,20 +64,12 @@ public class XRefDocumentVO {
this.databaseName = databaseName;
}
public String getDbId() {
return dbId;
public String getIdentifier() {
return identifier;
}
public void setDbId(String dbId) {
this.dbId = dbId;
}
public String getDbVersion() {
return dbVersion;
}
public void setDbVersion(String dbVersion) {
this.dbVersion = dbVersion;
public void setIdentifier(String identifier) {
this.identifier = identifier;
}
public String getName() {
......@@ -166,148 +104,12 @@ public class XRefDocumentVO {
this.species = species;
}
public String getXref() {
return xref;
}
public void setXref(String xref) {
this.xref = xref;
}
public String getFeatureType() {
return featureType;
}
public void setFeatureType(String featureType) {
this.featureType = featureType;
}
public String getSequenceId() {
return sequenceId;
}
public void setSequenceId(String sequenceId) {
this.sequenceId = sequenceId;
}
public String getSequence_version() {
return sequence_version;
}
public void setSequence_version(String sequence_version) {
this.sequence_version = sequence_version;
}
public String getStartPosition() {
return startPosition;
}
public void setStartPosition(String startPosition) {
this.startPosition = startPosition;
}
public String getEndPosition() {
return endPosition;
}
public void setEndPosition(String endPosition) {
this.endPosition = endPosition;
}
public String getMap() {
return map;
}
public void setMap(String map) {
this.map = map;
}
public String getMapPosition() {
return mapPosition;
}
public void setMapPosition(String mapPosition) {
this.mapPosition = mapPosition;
}
public String getAuthority() {
return authority;
}
public void setAuthority(String authority) {
this.authority = authority;
}
public String getTrait() {
return trait;
}
public void setTrait(String trait) {
this.trait = trait;
}
public String getTraitId() {
return traitId;
}
public void setTraitId(String traitId) {
this.traitId = traitId;
}
public String getEnvironment() {
return environment;
}
public void setEnvironment(String environment) {
this.environment = environment;
}
public String getEnvironmentId() {
return environmentId;
}
public void setEnvironmentId(String environmentId) {
this.environmentId = environmentId;
}
public String getStatistic() {
return statistic;
}
public void setStatistic(String statistic) {
this.statistic = statistic;
}
public String getUnit() {
return unit;
}
public void setUnit(String unit) {
this.unit = unit;
}
public String getGenotype() {
return genotype;
}
public void setGenotype(String genotype) {
this.genotype = genotype;
}
public String getExperimentType() {
return experimentType;
}
public void setExperimentType(String experimentType) {
this.experimentType = experimentType;
}
public List<String> getLinkedRessourcesID() {
return linkedRessourcesID;
public List<String> getLinkedResourcesID() {
return linkedResourcesID;
}
public void setLinkedRessourcesID(List<String> linkedRessourcesID) {
this.linkedRessourcesID = linkedRessourcesID;
public void setLinkedResourcesID(List<String> linkedResourcesID) {
this.linkedResourcesID = linkedResourcesID;
}
}
......@@ -58,10 +58,6 @@ public class ESRequestFactory {
request.source(new SearchSourceBuilder());
request.indicesOptions(indicesOptions);
if (documentType != null) {
request.types(documentType);
}
if (query != null) {
request.source().query(query);
}
......
......@@ -44,7 +44,7 @@ public class ESResponseParser {
if (response == null) return null;
SearchHits hits = response.getHits();
if (hits == null) return null;
return hits.totalHits;
return hits.getTotalHits().value;
}
/**
......
......@@ -83,7 +83,7 @@ public class ESScrollIterator<T> implements Iterator<T> {
}
this.scrollId = response.getScrollId();
this.totalHits = response.getHits().getTotalHits();
this.totalHits = response.getHits().getTotalHits().value;
this.hitIndex = 0;
this.currentIterator = parseIterator(response);
}
......
......@@ -182,7 +182,7 @@ public class GermplasmRepositoryImpl implements GermplasmRepository {
LOGGER.debug("\n\nQuery from findPedigree :\n" + termQuery.toString() + "\n\n");
if (hits.totalHits == 1) {
if (hits.getTotalHits().value == 1) {
// result found! \o/
SearchHit hit = hits.getAt(0);
String source = hit.getSourceAsString();
......@@ -191,7 +191,7 @@ public class GermplasmRepositoryImpl implements GermplasmRepository {
} catch (IOException e) {
LOGGER.error("Error occured when converting ES response to PedigreeVO: " + e.getMessage(), e);
}
} else if (hits.totalHits > 1) {
} else if (hits.getTotalHits().value > 1) {
throw new IllegalStateException("Expected only 1 result for pedigree with germplasmDbId: " + germplasmDbId);
}
return pedigreeVO;
......@@ -211,7 +211,7 @@ public class GermplasmRepositoryImpl implements GermplasmRepository {
final SearchHits hits = response.getHits();
LOGGER.debug("\n\nQuery from findProgeny :\n" + termQuery.toString() + "\n\n");
if (hits.totalHits == 1) {
if (hits.getTotalHits().value == 1) {
SearchHit hit = hits.getAt(0);
String source = hit.getSourceAsString();
try {
......@@ -219,7 +219,7 @@ public class GermplasmRepositoryImpl implements GermplasmRepository {
} catch (IOException e) {
LOGGER.error("Error occured when converting ES response to ProgenyVO: " + e.getMessage(), e);
}
} else if (hits.totalHits > 1) {
} else if (hits.getTotalHits().value > 1) {
throw new IllegalStateException("Expected only 1 result for progeny with germplasmDbId: " + germplasmDbId);
}
return progenyVO;
......
......@@ -2,6 +2,7 @@ package fr.inra.urgi.faidare.elasticsearch;
import com.fasterxml.jackson.databind.ObjectMapper;
import fr.inra.urgi.faidare.elasticsearch.fixture.DocumentObject;
import org.apache.lucene.search.TotalHits;
import org.assertj.core.util.Lists;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.bytes.BytesReference;
......@@ -50,7 +51,7 @@ class ESResponseParserTest {
float maxScore = 100;
// Can't mock SearchHits since it is a final class
SearchHits hits = new SearchHits(hitsArray, expectedTotalHits, maxScore);
SearchHits hits = new SearchHits(hitsArray, new TotalHits(expectedTotalHits, TotalHits.Relation.EQUAL_TO), maxScore);
SearchResponse response = mock(SearchResponse.class);
when(response.getHits()).thenReturn(hits);
......@@ -98,7 +99,7 @@ class ESResponseParserTest {
SearchHit hit3 = mockSearchHit(object3);
// Can't mock SearchHits since it is a final class
SearchHits hits = new SearchHits(new SearchHit[]{hit1, hit2, hit3}, 3, 100);
SearchHits hits = new SearchHits(new SearchHit[]{hit1, hit2, hit3}, new TotalHits(3L, TotalHits.Relation.EQUAL_TO), 100);
SearchResponse response = mock(SearchResponse.class);
when(response.getHits()).thenReturn(hits);
......@@ -132,7 +133,7 @@ class ESResponseParserTest {
assertThat(result).isNull();
// Return null if no hits in hits
SearchHits hits = new SearchHits(null, 0, 100);
SearchHits hits = new SearchHits(null, new TotalHits(0, TotalHits.Relation.EQUAL_TO), 100);
when(response.getHits()).thenReturn(hits);
List<DocumentObject> result2 = parser.parseHits(response, DocumentObject.class);
......
......@@ -62,12 +62,12 @@ class XRefDocumentRepositoryTest {
@Test
void should_Find_By_Linked_Id() {
String id = "ID2";
List<String> linkedRessourcesID = Collections.singletonList(id);
List<String> linkedResourcesID = Collections.singletonList(id);
XRefDocumentSearchCriteria criteria = new XRefDocumentSearchCriteria();
criteria.setLinkedRessourcesID(linkedRessourcesID);
criteria.setLinkedResourcesID(linkedResourcesID);
PaginatedList<XRefDocumentVO> documents = repository.find(criteria);
assertThat(documents).isNotNull().hasSize(2)
.flatExtracting("linkedRessourcesID")
.flatExtracting("linkedResourcesID")
.contains(id);
}
......
......@@ -7,10 +7,7 @@ import fr.inra.urgi.faidare.config.FaidareProperties;
import fr.inra.urgi.faidare.domain.xref.XRefDocumentVO;
import fr.inra.urgi.faidare.elasticsearch.document.DocumentAnnotationUtil;
import fr.inra.urgi.faidare.repository.es.XRefDocumentRepositoryImpl;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.bulk.BulkRequest;
......@@ -19,8 +16,12 @@ import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
......@@ -47,8 +48,7 @@ public class ESSetUp {
* Delete index/alias if it exists
*/
private void deleteIndex(String indexName) throws IOException {
GetIndexRequest existsRequest = new GetIndexRequest();
existsRequest.indices(indexName);
GetIndexRequest existsRequest = new GetIndexRequest(indexName);
boolean exists = client.indices().exists(existsRequest, RequestOptions.DEFAULT);
if (!exists) {
// Do not delete non existing index
......@@ -76,11 +76,11 @@ public class ESSetUp {
// with document mappings
String mapping = readResource("./index/" + documentType + "_mappin