chore: add elasticsearch

.gitlab-ci.yml

 # Image source available on https://github.com/Ninja-Squad/docker-rare
+# It contains a JDK 8 and a Chrome browser
+# Node, NPM and Yarn are installed by Gradle
 image: ninjasquad/docker-rare
 
 # Disable the Gradle daemon for Continuous Integration servers as correctness
@@ -13,6 +15,23 @@ before_script:
 
 test:
  stage: test
+ # the backend tests need an elasticsearch instance
+ services:
+   # even if that would be ideal
+   # we can't just launch the service with just elasticsearch:6.3.1
+   # because we need to pass some variables, but they are passed to _all_ containers
+   # so they fail the start of other docker images like ninjasquad/docker-rare
+   # the only solution is to override the entrypoint of the service and pass the arguments manually
+   - name: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+     alias: elasticsearch
+     # discovery.type=single-node
+     # single-node is necessary to start in development mode
+     # so there will be no bootstrap checks that would fail on CI
+     # especially the error regarding
+     # `max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]`
+     # cluster.name=es-rare
+     # the cluster name used in tests
+     command: ["bin/elasticsearch", "-Ediscovery.type=single-node", "-Ecluster.name=es-rare"]
  script: ./gradlew build
  cache:
    key: "$CI_COMMIT_REF_NAME"

README.md

@@ -11,9 +11,21 @@ You need to install:
 
 - a recent enough JDK8
 
+The application expects to connect on an ElasticSearch instance running on `http://127.0.0.1:9300`,
+in a cluster named `es-rare`.
+To have such an instance, simply run:
+
+    docker-compose up
+
+And this will start ElasticSearch and a Kibana instance (allowing to explore the data on http://localhost:5601).
+
 Then at the root of the application, run `./gradlew build` to download the dependencies.
 Then run `./gradlew bootRun` to start the app.
 
+You can stop the Elastic Search and Kibana instances by running:
+
+    docker-compose stop
+
 ### Frontend
 
 The project uses Angular (6.x) for the frontend,

backend/build.gradle.kts

@@ -23,6 +23,7 @@ java {
 
 repositories {
     mavenCentral()
+    maven("https://repo.spring.io/libs-milestone")
 }
 
 tasks {
@@ -71,6 +72,10 @@ tasks {
 dependencies {
     implementation("org.springframework.boot:spring-boot-starter-web")
     implementation("org.springframework.boot:spring-boot-starter-actuator")
+    implementation("org.springframework.data:spring-data-elasticsearch:3.1.0.M3")
+    implementation("org.elasticsearch:elasticsearch:6.3.1")
+    implementation("org.elasticsearch.client:transport:6.3.1")
+    implementation("org.elasticsearch.plugin:transport-netty4-client:6.3.1")
 
     testImplementation("org.springframework.boot:spring-boot-starter-test") {
         exclude(module = "junit")

backend/src/main/java/fr/inra/urgi/rare/config/CustomElasticSearchEntityMapper.java

+package fr.inra.urgi.rare.config;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.springframework.data.elasticsearch.core.EntityMapper;
+
+/**
+ * Re-implements the ElasticSearch entity mapper as the default one
+ * doesn't allow to use the {@link ObjectMapper} we have in our Spring Boot application.
+ * This avoids to annotate every parameter of the constructor of our documents with JsonProperty.
+ * See https://github.com/spring-projects/spring-data-elasticsearch/wiki/Custom-ObjectMapper
+ */
+public class CustomElasticSearchEntityMapper implements EntityMapper {
+
+    private ObjectMapper objectMapper;
+
+    CustomElasticSearchEntityMapper(ObjectMapper objectMapper) {
+        this.objectMapper = objectMapper;
+    }
+
+    @Override
+    public String mapToString(Object object) throws IOException {
+        return objectMapper.writeValueAsString(object);
+    }
+
+    @Override
+    public <T> T mapToObject(String source, Class<T> clazz) throws IOException {
+        return objectMapper.readValue(source, clazz);
+    }
+}

backend/src/main/java/fr/inra/urgi/rare/config/ElasticSearchConfig.java

+package fr.inra.urgi.rare.config;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.TransportAddress;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
+import org.springframework.data.elasticsearch.core.EntityMapper;
+import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
+
+@Configuration
+@EnableElasticsearchRepositories(basePackages = "fr.inra.urgi.rare.dao")
+public class ElasticSearchConfig {
+
+    @Value("${spring.data.elasticsearch.cluster.name}")
+    private String esClusterName;
+
+    @Value("${spring.data.elasticsearch.host}")
+    private String esHost;
+
+    @Value("${spring.data.elasticsearch.port}")
+    private Integer esPort;
+
+    @Bean
+    public RareProperties rareProperties() {
+        return new RareProperties();
+    }
+
+    /**
+     * Creates a custom entity mapper for ES with the Jackson {@link ObjectMapper}
+     * This avoids to annotate every parameter of the constructor of our documents with JsonProperty
+     *
+     * @param objectMapper - the Jackson {@link ObjectMapper}
+     */
+    @Bean
+    public EntityMapper customElasticSearchEntityMapper(ObjectMapper objectMapper) {
+        return new CustomElasticSearchEntityMapper(objectMapper);
+    }
+
+    /**
+     * Creates an Elasticsearch instance using the configuration provided.
+     * It relies on {@link TransportClient }.
+     * In the future it might be interesting to switch to HighLevelRestClient
+     * when Spring Data Elasticsearch supports it (see https://github.com/spring-projects/spring-data-elasticsearch/pull/216)
+     */
+    @Bean
+    public Client client() throws UnknownHostException {
+        Settings settings = Settings.builder()
+                .put("cluster.name", esClusterName)
+                .build();
+        // if we are on CI, we use a hardcoded host, else we use the injected value
+        String host = System.getenv("CI") != null ? "elasticsearch" : esHost;
+        return new PreBuiltTransportClient(settings)
+                .addTransportAddress(new TransportAddress(InetAddress.getByName(host), esPort));
+    }
+
+    @Bean
+    public ElasticsearchTemplate elasticsearchTemplate(Client client, EntityMapper customElasticSearchEntityMapper) {
+        return new ElasticsearchTemplate(client, customElasticSearchEntityMapper);
+    }
+}
+

backend/src/main/java/fr/inra/urgi/rare/config/RareProperties.java

@@ -6,6 +6,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
 
 /**
  * Properties class holding the rare-specific properties of the application (typically stored in application.yml)
+ *
  * @author JB Nizet
  */
 @ConfigurationProperties(prefix = "rare")
@@ -16,6 +17,14 @@ public class RareProperties {
      */
     private Path resourceDir;
 
+    /**
+     * The ES prefix used to store the resources.
+     * Allows for a different index and type name between dev and tests ('resource' and 'test-resource' for example).
+     * Used in the {@link org.springframework.data.elasticsearch.annotations.Document} annotation
+     * on our domain entities.
+     */
+    private String elasticsearchPrefix;
+
     public Path getResourceDir() {
         return resourceDir;
     }
@@ -24,10 +33,19 @@ public class RareProperties {
         this.resourceDir = resourceDir;
     }
 
+    public String getElasticsearchPrefix() {
+        return elasticsearchPrefix;
+    }
+
+    public void setElasticsearchPrefix(String elasticsearchPrefix) {
+        this.elasticsearchPrefix = elasticsearchPrefix;
+    }
+
     @Override
     public String toString() {
         return "RareProperties{" +
-            "resourceDir=" + resourceDir +
-            '}';
+                "resourceDir=" + resourceDir +
+                ", elasticsearchPrefix='" + elasticsearchPrefix + '\'' +
+                '}';
     }
 }

backend/src/main/java/fr/inra/urgi/rare/dao/GeneticResourceDao.java

+package fr.inra.urgi.rare.dao;
+
+import fr.inra.urgi.rare.domain.GeneticResource;
+import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
+
+public interface GeneticResourceDao extends ElasticsearchRepository<GeneticResource, String> {
+}

backend/src/main/java/fr/inra/urgi/rare/domain/GeneticResource.java

@@ -5,11 +5,16 @@ import java.util.Objects;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import org.springframework.data.elasticsearch.annotations.Document;
 
 /**
  * A genetic resource, as loaded from a JSON file, and stored in ElasticSearch
  * @author JB Nizet
  */
+@Document(
+        indexName = "#{@rareProperties.getElasticsearchPrefix()}resource-index",
+        type = "#{@rareProperties.getElasticsearchPrefix()}resource"
+)
 public final class GeneticResource {
     @JsonProperty("identifier")
     private final String id;
@@ -168,74 +173,74 @@ public final class GeneticResource {
         }
         GeneticResource that = (GeneticResource) o;
         return Objects.equals(id, that.id) &&
-            Objects.equals(name, that.name) &&
-            Objects.equals(description, that.description) &&
-            Objects.equals(pillarName, that.pillarName) &&
-            Objects.equals(databaseSource, that.databaseSource) &&
-            Objects.equals(portalURL, that.portalURL) &&
-            Objects.equals(dataURL, that.dataURL) &&
-            Objects.equals(domain, that.domain) &&
-            Objects.equals(taxon, that.taxon) &&
-            Objects.equals(family, that.family) &&
-            Objects.equals(genus, that.genus) &&
-            Objects.equals(species, that.species) &&
-            Objects.equals(materialType, that.materialType) &&
-            Objects.equals(biotopeType, that.biotopeType) &&
-            Objects.equals(countryOfOrigin, that.countryOfOrigin) &&
-            Objects.equals(originLatitude, that.originLatitude) &&
-            Objects.equals(originLongitude, that.originLongitude) &&
-            Objects.equals(countryOfCollect, that.countryOfCollect) &&
-            Objects.equals(collectLatitude, that.collectLatitude) &&
-            Objects.equals(collectLongitude, that.collectLongitude);
+                Objects.equals(name, that.name) &&
+                Objects.equals(description, that.description) &&
+                Objects.equals(pillarName, that.pillarName) &&
+                Objects.equals(databaseSource, that.databaseSource) &&
+                Objects.equals(portalURL, that.portalURL) &&
+                Objects.equals(dataURL, that.dataURL) &&
+                Objects.equals(domain, that.domain) &&
+                Objects.equals(taxon, that.taxon) &&
+                Objects.equals(family, that.family) &&
+                Objects.equals(genus, that.genus) &&
+                Objects.equals(species, that.species) &&
+                Objects.equals(materialType, that.materialType) &&
+                Objects.equals(biotopeType, that.biotopeType) &&
+                Objects.equals(countryOfOrigin, that.countryOfOrigin) &&
+                Objects.equals(originLatitude, that.originLatitude) &&
+                Objects.equals(originLongitude, that.originLongitude) &&
+                Objects.equals(countryOfCollect, that.countryOfCollect) &&
+                Objects.equals(collectLatitude, that.collectLatitude) &&
+                Objects.equals(collectLongitude, that.collectLongitude);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(id,
-                            name,
-                            description,
-                            pillarName,
-                            databaseSource,
-                            portalURL,
-                            dataURL,
-                            domain,
-                            taxon,
-                            family,
-                            genus,
-                            species,
-                            materialType,
-                            biotopeType,
-                            countryOfOrigin,
-                            originLatitude,
-                            originLongitude,
-                            countryOfCollect,
-                            collectLatitude,
-                            collectLongitude);
+                name,
+                description,
+                pillarName,
+                databaseSource,
+                portalURL,
+                dataURL,
+                domain,
+                taxon,
+                family,
+                genus,
+                species,
+                materialType,
+                biotopeType,
+                countryOfOrigin,
+                originLatitude,
+                originLongitude,
+                countryOfCollect,
+                collectLatitude,
+                collectLongitude);
     }
 
     @Override
     public String toString() {
         return "GeneticResource{" +
-            "id='" + id + '\'' +
-            ", name='" + name + '\'' +
-            ", description='" + description + '\'' +
-            ", pillarName='" + pillarName + '\'' +
-            ", databaseSource='" + databaseSource + '\'' +
-            ", portalURL='" + portalURL + '\'' +
-            ", dataURL='" + dataURL + '\'' +
-            ", domain='" + domain + '\'' +
-            ", taxon='" + taxon + '\'' +
-            ", family='" + family + '\'' +
-            ", genus='" + genus + '\'' +
-            ", species='" + species + '\'' +
-            ", materialType='" + materialType + '\'' +
-            ", biotopeType='" + biotopeType + '\'' +
-            ", countryOfOrigin='" + countryOfOrigin + '\'' +
-            ", originLatitude=" + originLatitude +
-            ", originLongitude=" + originLongitude +
-            ", countryOfCollect='" + countryOfCollect + '\'' +
-            ", collectLatitude=" + collectLatitude +
-            ", collectLongitude=" + collectLongitude +
-            '}';
+                "id='" + id + '\'' +
+                ", name='" + name + '\'' +
+                ", description='" + description + '\'' +
+                ", pillarName='" + pillarName + '\'' +
+                ", databaseSource='" + databaseSource + '\'' +
+                ", portalURL='" + portalURL + '\'' +
+                ", dataURL='" + dataURL + '\'' +
+                ", domain='" + domain + '\'' +
+                ", taxon='" + taxon + '\'' +
+                ", family='" + family + '\'' +
+                ", genus='" + genus + '\'' +
+                ", species='" + species + '\'' +
+                ", materialType='" + materialType + '\'' +
+                ", biotopeType='" + biotopeType + '\'' +
+                ", countryOfOrigin='" + countryOfOrigin + '\'' +
+                ", originLatitude=" + originLatitude +
+                ", originLongitude=" + originLongitude +
+                ", countryOfCollect='" + countryOfCollect + '\'' +
+                ", collectLatitude=" + collectLatitude +
+                ", collectLongitude=" + collectLongitude +
+                '}';
     }
 }

backend/src/main/resources/application.yml

 rare:
   resource-dir: /tmp/rare/resources
+  elasticsearch-prefix: ''
+
+spring:
+  data:
+    elasticsearch:
+        cluster:
+          name: es-rare
+        host: 127.0.0.1
+        port: 9300

backend/src/test/java/fr/inra/urgi/rare/dao/GeneticResourceDaoTest.java

+package fr.inra.urgi.rare.dao;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Collections;
+
+import fr.inra.urgi.rare.config.ElasticSearchConfig;
+import fr.inra.urgi.rare.domain.GeneticResource;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.json.JsonTest;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+
+@ExtendWith(SpringExtension.class)
+@TestPropertySource("/test.properties")
+@SpringBootTest(classes = ElasticSearchConfig.class)
+@JsonTest
+class GeneticResourceDaoTest {
+
+    @Autowired
+    private GeneticResourceDao geneticResourceDao;
+
+    @Test
+    void shouldSaveAndList() {
+
+        GeneticResource geneticResource =
+                new GeneticResource(
+                        "doi:10.15454/1.492178535151698E12",
+                        "Grecanico dorato",
+                        "Grecanico dorato is a Vitis vinifera subsp vinifera cv. Garganega accession (number: "
+                                + "1310Mtp1, doi:10.15454/1.492178535151698E12) maintained by the GRAPEVINE (managed by INRA) and held "
+                                + "by INRA. It is a maintained/maintenu accession of biological status traditional cultivar/cultivar "
+                                + "traditionnel",
+                        "Plant",
+                        "Florilège",
+                        "http://florilege.arcad-project.org/fr/collections",
+                        "https://urgi.versailles.inra.fr/gnpis-core/#accessionCard/id=ZG9pOjEwLjE1NDU0LzEuNDkyMTc4NTM1MTUxNjk4RTEy",
+                        "Plantae",
+                        Collections.singletonList("Vitis vinifera"),
+                        Collections.singletonList("Vitaceae"),
+                        Collections.singletonList("Vitis"),
+                        Collections.singletonList("Vitis vinifera"),
+                        Collections.singletonList("testMaterialType"),
+                        Collections.singletonList("testBiotopeType"),
+                        "France",
+                        0.1,
+                        0.2,
+                        "Italy",
+                        37.5,
+                        15.099722);
+
+        geneticResourceDao.save(geneticResource);
+
+        assertThat(geneticResourceDao.findAll())
+                .extracting(GeneticResource::getName)
+                .containsExactly("Grecanico dorato");
+    }
+}
+

backend/src/test/resources/test.properties

+spring.data.elasticsearch.cluster.name=es-rare
+spring.data.elasticsearch.host=localhost
+spring.data.elasticsearch.port=9300
+
+rare.elasticsearch-prefix=test-
\ No newline at end of file

docker-compose.yml

+version: '3.3'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.3.1
+    container_name: elasticsearch
+    environment:
+      - cluster.name=es-rare
+      - discovery.type=single-node
+    ports:
+      - 9200:9200
+      - 9300:9300
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.3.1
+    container_name: kibana
+    environment:
+      - "ELASTICSEARCH_URL=http://elasticsearch:9200"
+    depends_on:
+      - elasticsearch
+    ports:
+      - 5601:5601
\ No newline at end of file