Commit d075900e authored by Exbrayat Cédric's avatar Exbrayat Cédric
Browse files

chore: add elasticsearch

parent ecb15d79
# Image source available on https://github.com/Ninja-Squad/docker-rare
# It contains a JDK 8 and a Chrome browser
# Node, NPM and Yarn are installed by Gradle
image: ninjasquad/docker-rare
# Disable the Gradle daemon for Continuous Integration servers as correctness
......@@ -13,6 +15,23 @@ before_script:
test:
stage: test
# the backend tests need an elasticsearch instance
services:
# even if that would be ideal
# we can't just launch the service with just elasticsearch:6.3.1
# because we need to pass some variables, but they are passed to _all_ containers
# so they fail the start of other docker images like ninjasquad/docker-rare
# the only solution is to override the entrypoint of the service and pass the arguments manually
- name: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
alias: elasticsearch
# discovery.type=single-node
# single-node is necessary to start in development mode
# so there will be no bootstrap checks that would fail on CI
# especially the error regarding
# `max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]`
# cluster.name=es-rare
# the cluster name used in tests
command: ["bin/elasticsearch", "-Ediscovery.type=single-node", "-Ecluster.name=es-rare"]
script: ./gradlew build
cache:
key: "$CI_COMMIT_REF_NAME"
......
......@@ -11,9 +11,21 @@ You need to install:
- a recent enough JDK8
The application expects to connect on an ElasticSearch instance running on `http://127.0.0.1:9300`,
in a cluster named `es-rare`.
To have such an instance, simply run:
docker-compose up
And this will start ElasticSearch and a Kibana instance (allowing to explore the data on http://localhost:5601).
Then at the root of the application, run `./gradlew build` to download the dependencies.
Then run `./gradlew bootRun` to start the app.
You can stop the Elastic Search and Kibana instances by running:
docker-compose stop
### Frontend
The project uses Angular (6.x) for the frontend,
......
......@@ -23,6 +23,7 @@ java {
repositories {
mavenCentral()
maven("https://repo.spring.io/libs-milestone")
}
tasks {
......@@ -71,6 +72,10 @@ tasks {
dependencies {
implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("org.springframework.data:spring-data-elasticsearch:3.1.0.M3")
implementation("org.elasticsearch:elasticsearch:6.3.1")
implementation("org.elasticsearch.client:transport:6.3.1")
implementation("org.elasticsearch.plugin:transport-netty4-client:6.3.1")
testImplementation("org.springframework.boot:spring-boot-starter-test") {
exclude(module = "junit")
......
package fr.inra.urgi.rare.config;
import java.io.IOException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.data.elasticsearch.core.EntityMapper;
/**
* Re-implements the ElasticSearch entity mapper as the default one
* doesn't allow to use the {@link ObjectMapper} we have in our Spring Boot application.
* This avoids to annotate every parameter of the constructor of our documents with JsonProperty.
* See https://github.com/spring-projects/spring-data-elasticsearch/wiki/Custom-ObjectMapper
*/
public class CustomElasticSearchEntityMapper implements EntityMapper {
private ObjectMapper objectMapper;
CustomElasticSearchEntityMapper(ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}
@Override
public String mapToString(Object object) throws IOException {
return objectMapper.writeValueAsString(object);
}
@Override
public <T> T mapToObject(String source, Class<T> clazz) throws IOException {
return objectMapper.readValue(source, clazz);
}
}
package fr.inra.urgi.rare.config;
import java.net.InetAddress;
import java.net.UnknownHostException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
@Configuration
@EnableElasticsearchRepositories(basePackages = "fr.inra.urgi.rare.dao")
public class ElasticSearchConfig {
@Value("${spring.data.elasticsearch.cluster.name}")
private String esClusterName;
@Value("${spring.data.elasticsearch.host}")
private String esHost;
@Value("${spring.data.elasticsearch.port}")
private Integer esPort;
@Bean
public RareProperties rareProperties() {
return new RareProperties();
}
/**
* Creates a custom entity mapper for ES with the Jackson {@link ObjectMapper}
* This avoids to annotate every parameter of the constructor of our documents with JsonProperty
*
* @param objectMapper - the Jackson {@link ObjectMapper}
*/
@Bean
public EntityMapper customElasticSearchEntityMapper(ObjectMapper objectMapper) {
return new CustomElasticSearchEntityMapper(objectMapper);
}
/**
* Creates an Elasticsearch instance using the configuration provided.
* It relies on {@link TransportClient }.
* In the future it might be interesting to switch to HighLevelRestClient
* when Spring Data Elasticsearch supports it (see https://github.com/spring-projects/spring-data-elasticsearch/pull/216)
*/
@Bean
public Client client() throws UnknownHostException {
Settings settings = Settings.builder()
.put("cluster.name", esClusterName)
.build();
// if we are on CI, we use a hardcoded host, else we use the injected value
String host = System.getenv("CI") != null ? "elasticsearch" : esHost;
return new PreBuiltTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), esPort));
}
@Bean
public ElasticsearchTemplate elasticsearchTemplate(Client client, EntityMapper customElasticSearchEntityMapper) {
return new ElasticsearchTemplate(client, customElasticSearchEntityMapper);
}
}
......@@ -6,6 +6,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
/**
* Properties class holding the rare-specific properties of the application (typically stored in application.yml)
*
* @author JB Nizet
*/
@ConfigurationProperties(prefix = "rare")
......@@ -16,6 +17,14 @@ public class RareProperties {
*/
private Path resourceDir;
/**
* The ES prefix used to store the resources.
* Allows for a different index and type name between dev and tests ('resource' and 'test-resource' for example).
* Used in the {@link org.springframework.data.elasticsearch.annotations.Document} annotation
* on our domain entities.
*/
private String elasticsearchPrefix;
public Path getResourceDir() {
return resourceDir;
}
......@@ -24,10 +33,19 @@ public class RareProperties {
this.resourceDir = resourceDir;
}
public String getElasticsearchPrefix() {
return elasticsearchPrefix;
}
public void setElasticsearchPrefix(String elasticsearchPrefix) {
this.elasticsearchPrefix = elasticsearchPrefix;
}
@Override
public String toString() {
return "RareProperties{" +
"resourceDir=" + resourceDir +
'}';
"resourceDir=" + resourceDir +
", elasticsearchPrefix='" + elasticsearchPrefix + '\'' +
'}';
}
}
package fr.inra.urgi.rare.dao;
import fr.inra.urgi.rare.domain.GeneticResource;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
public interface GeneticResourceDao extends ElasticsearchRepository<GeneticResource, String> {
}
......@@ -5,11 +5,16 @@ import java.util.Objects;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.springframework.data.elasticsearch.annotations.Document;
/**
* A genetic resource, as loaded from a JSON file, and stored in ElasticSearch
* @author JB Nizet
*/
@Document(
indexName = "#{@rareProperties.getElasticsearchPrefix()}resource-index",
type = "#{@rareProperties.getElasticsearchPrefix()}resource"
)
public final class GeneticResource {
@JsonProperty("identifier")
private final String id;
......@@ -168,74 +173,74 @@ public final class GeneticResource {
}
GeneticResource that = (GeneticResource) o;
return Objects.equals(id, that.id) &&
Objects.equals(name, that.name) &&
Objects.equals(description, that.description) &&
Objects.equals(pillarName, that.pillarName) &&
Objects.equals(databaseSource, that.databaseSource) &&
Objects.equals(portalURL, that.portalURL) &&
Objects.equals(dataURL, that.dataURL) &&
Objects.equals(domain, that.domain) &&
Objects.equals(taxon, that.taxon) &&
Objects.equals(family, that.family) &&
Objects.equals(genus, that.genus) &&
Objects.equals(species, that.species) &&
Objects.equals(materialType, that.materialType) &&
Objects.equals(biotopeType, that.biotopeType) &&
Objects.equals(countryOfOrigin, that.countryOfOrigin) &&
Objects.equals(originLatitude, that.originLatitude) &&
Objects.equals(originLongitude, that.originLongitude) &&
Objects.equals(countryOfCollect, that.countryOfCollect) &&
Objects.equals(collectLatitude, that.collectLatitude) &&
Objects.equals(collectLongitude, that.collectLongitude);
Objects.equals(name, that.name) &&
Objects.equals(description, that.description) &&
Objects.equals(pillarName, that.pillarName) &&
Objects.equals(databaseSource, that.databaseSource) &&
Objects.equals(portalURL, that.portalURL) &&
Objects.equals(dataURL, that.dataURL) &&
Objects.equals(domain, that.domain) &&
Objects.equals(taxon, that.taxon) &&
Objects.equals(family, that.family) &&
Objects.equals(genus, that.genus) &&
Objects.equals(species, that.species) &&
Objects.equals(materialType, that.materialType) &&
Objects.equals(biotopeType, that.biotopeType) &&
Objects.equals(countryOfOrigin, that.countryOfOrigin) &&
Objects.equals(originLatitude, that.originLatitude) &&
Objects.equals(originLongitude, that.originLongitude) &&
Objects.equals(countryOfCollect, that.countryOfCollect) &&
Objects.equals(collectLatitude, that.collectLatitude) &&
Objects.equals(collectLongitude, that.collectLongitude);
}
@Override
public int hashCode() {
return Objects.hash(id,
name,
description,
pillarName,
databaseSource,
portalURL,
dataURL,
domain,
taxon,
family,
genus,
species,
materialType,
biotopeType,
countryOfOrigin,
originLatitude,
originLongitude,
countryOfCollect,
collectLatitude,
collectLongitude);
name,
description,
pillarName,
databaseSource,
portalURL,
dataURL,
domain,
taxon,
family,
genus,
species,
materialType,
biotopeType,
countryOfOrigin,
originLatitude,
originLongitude,
countryOfCollect,
collectLatitude,
collectLongitude);
}
@Override
public String toString() {
return "GeneticResource{" +
"id='" + id + '\'' +
", name='" + name + '\'' +
", description='" + description + '\'' +
", pillarName='" + pillarName + '\'' +
", databaseSource='" + databaseSource + '\'' +
", portalURL='" + portalURL + '\'' +
", dataURL='" + dataURL + '\'' +
", domain='" + domain + '\'' +
", taxon='" + taxon + '\'' +
", family='" + family + '\'' +
", genus='" + genus + '\'' +
", species='" + species + '\'' +
", materialType='" + materialType + '\'' +
", biotopeType='" + biotopeType + '\'' +
", countryOfOrigin='" + countryOfOrigin + '\'' +
", originLatitude=" + originLatitude +
", originLongitude=" + originLongitude +
", countryOfCollect='" + countryOfCollect + '\'' +
", collectLatitude=" + collectLatitude +
", collectLongitude=" + collectLongitude +
'}';
"id='" + id + '\'' +
", name='" + name + '\'' +
", description='" + description + '\'' +
", pillarName='" + pillarName + '\'' +
", databaseSource='" + databaseSource + '\'' +
", portalURL='" + portalURL + '\'' +
", dataURL='" + dataURL + '\'' +
", domain='" + domain + '\'' +
", taxon='" + taxon + '\'' +
", family='" + family + '\'' +
", genus='" + genus + '\'' +
", species='" + species + '\'' +
", materialType='" + materialType + '\'' +
", biotopeType='" + biotopeType + '\'' +
", countryOfOrigin='" + countryOfOrigin + '\'' +
", originLatitude=" + originLatitude +
", originLongitude=" + originLongitude +
", countryOfCollect='" + countryOfCollect + '\'' +
", collectLatitude=" + collectLatitude +
", collectLongitude=" + collectLongitude +
'}';
}
}
rare:
resource-dir: /tmp/rare/resources
elasticsearch-prefix: ''
spring:
data:
elasticsearch:
cluster:
name: es-rare
host: 127.0.0.1
port: 9300
package fr.inra.urgi.rare.dao;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Collections;
import fr.inra.urgi.rare.config.ElasticSearchConfig;
import fr.inra.urgi.rare.domain.GeneticResource;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.json.JsonTest;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
@ExtendWith(SpringExtension.class)
@TestPropertySource("/test.properties")
@SpringBootTest(classes = ElasticSearchConfig.class)
@JsonTest
class GeneticResourceDaoTest {
@Autowired
private GeneticResourceDao geneticResourceDao;
@Test
void shouldSaveAndList() {
GeneticResource geneticResource =
new GeneticResource(
"doi:10.15454/1.492178535151698E12",
"Grecanico dorato",
"Grecanico dorato is a Vitis vinifera subsp vinifera cv. Garganega accession (number: "
+ "1310Mtp1, doi:10.15454/1.492178535151698E12) maintained by the GRAPEVINE (managed by INRA) and held "
+ "by INRA. It is a maintained/maintenu accession of biological status traditional cultivar/cultivar "
+ "traditionnel",
"Plant",
"Florilège",
"http://florilege.arcad-project.org/fr/collections",
"https://urgi.versailles.inra.fr/gnpis-core/#accessionCard/id=ZG9pOjEwLjE1NDU0LzEuNDkyMTc4NTM1MTUxNjk4RTEy",
"Plantae",
Collections.singletonList("Vitis vinifera"),
Collections.singletonList("Vitaceae"),
Collections.singletonList("Vitis"),
Collections.singletonList("Vitis vinifera"),
Collections.singletonList("testMaterialType"),
Collections.singletonList("testBiotopeType"),
"France",
0.1,
0.2,
"Italy",
37.5,
15.099722);
geneticResourceDao.save(geneticResource);
assertThat(geneticResourceDao.findAll())
.extracting(GeneticResource::getName)
.containsExactly("Grecanico dorato");
}
}
spring.data.elasticsearch.cluster.name=es-rare
spring.data.elasticsearch.host=localhost
spring.data.elasticsearch.port=9300
rare.elasticsearch-prefix=test-
\ No newline at end of file
version: '3.3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
container_name: elasticsearch
environment:
- cluster.name=es-rare
- discovery.type=single-node
ports:
- 9200:9200
- 9300:9300
kibana:
image: docker.elastic.co/kibana/kibana:6.3.1
container_name: kibana
environment:
- "ELASTICSEARCH_URL=http://elasticsearch:9200"
depends_on:
- elasticsearch
ports:
- 5601:5601
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment