Commit 4755a0d4 authored by Jean-Baptiste Nizet's avatar Jean-Baptiste Nizet
Browse files

feat: add list of harvests

Note that this feature revealed that the version of spring-data-elasticsearch that we're using was not compatible with the spring-data-commons version used by Spring Boot. So I upgraded the spring-data-commons version.
parent a50d8d55
......@@ -151,3 +151,13 @@ It's only when the property `endInstant` of the returned JSON is non-null that t
"startInstant": "2018-07-24T12:56:27.322Z"
}
```
In case you lost the response to the POST request and thus don't know what the URL of the harvest is,
you can list the harvests, in descending order of their start instant, by sending a GET request to
`/api/harvests`:
http --auth rare:f01a7031fc17 GET http://localhost:8080/api/harvests
or
curl -u rare:f01a7031fc17 http://localhost:8080/api/harvests
......@@ -73,7 +73,7 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("org.springframework.boot:spring-boot-starter-security")
implementation("org.springframework.boot:spring-boot-starter-webflux")
implementation("org.springframework.data:spring-data-commons:2.1.0.M3")
implementation("org.springframework.data:spring-data-elasticsearch:3.1.0.M3")
implementation("org.elasticsearch:elasticsearch:6.3.1")
implementation("org.elasticsearch.client:transport:6.3.1")
......
......@@ -4,6 +4,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
import com.fasterxml.jackson.databind.ObjectMapper;
import fr.inra.urgi.rare.dao.GeneticResourceDao;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
......@@ -17,7 +18,7 @@ import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
@Configuration
@EnableElasticsearchRepositories(basePackages = "fr.inra.urgi.rare.dao")
@EnableElasticsearchRepositories(basePackageClasses = GeneticResourceDao.class)
public class ElasticSearchConfig {
@Value("${spring.data.elasticsearch.cluster.name}")
......
......@@ -6,5 +6,5 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository
/**
* DAO for {@link HarvestResult}
*/
public interface HarvestResultDao extends ElasticsearchRepository<HarvestResult, String> {
public interface HarvestResultDao extends ElasticsearchRepository<HarvestResult, String>, HarvestResultDaoCustom {
}
package fr.inra.urgi.rare.dao;
import fr.inra.urgi.rare.harvest.HarvestResult;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
/**
* Custom methods of {@link HarvestResultDao}
* @author JB Nizet
*/
public interface HarvestResultDaoCustom {
/**
* Lists the {@link HarvestResult} of the given page, sorted by start instant, in descending order (most recent
* first).
* @return a page of <strong>partial</strong> {@link HarvestResult} instances, containing only the ID, start instant
* and end instant.
*/
Page<HarvestResult> list(Pageable page);
}
package fr.inra.urgi.rare.dao;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import fr.inra.urgi.rare.harvest.HarvestResult;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
/**
* Implementation of {@link HarvestResultDaoCustom}
* @author JB Nizet
*/
public class HarvestResultDaoImpl implements HarvestResultDaoCustom {
private final ElasticsearchTemplate elasticsearchTemplate;
public HarvestResultDaoImpl(ElasticsearchTemplate elasticsearchTemplate) {
this.elasticsearchTemplate = elasticsearchTemplate;
}
@Override
public Page<HarvestResult> list(Pageable page) {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(matchAllQuery())
.withSourceFilter(new FetchSourceFilterBuilder().withIncludes("id", "startInstant", "endInstant").build())
.withSort(SortBuilders.fieldSort("startInstant").order(SortOrder.DESC))
.withPageable(page)
.build();
return this.elasticsearchTemplate.queryForPage(searchQuery, HarvestResult.class);
}
}
package fr.inra.urgi.rare.dto;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.data.domain.Page;
......@@ -51,6 +53,14 @@ public final class PageDTO<T> {
page.getTotalPages());
}
public static <T, R> PageDTO<R> fromPage(Page<T> page, Function<T, R> mapper) {
return new PageDTO<R>(page.getContent().stream().map(mapper).collect(Collectors.toList()),
page.getNumber(),
page.getSize(),
page.getTotalElements(),
page.getTotalPages());
}
public List<T> getContent() {
return content;
}
......
......@@ -53,8 +53,8 @@ public final class HarvestResult {
this.id = id;
this.startInstant = startInstant;
this.endInstant = endInstant;
this.globalErrors = Collections.unmodifiableList(new ArrayList<>(globalErrors));
this.files = Collections.unmodifiableList(new ArrayList<>(files));
this.globalErrors = globalErrors == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(globalErrors));
this.files = files == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(files));
}
public String getId() {
......@@ -118,7 +118,7 @@ public final class HarvestResult {
*/
public static final class HarvestResultBuilder {
private final String id;
private final Instant startInstant;
private Instant startInstant;
private Instant endInstant;
private final List<HarvestedFile> files = new ArrayList<>();
private final List<String> globalErrors = new ArrayList<>();
......@@ -128,6 +128,14 @@ public final class HarvestResult {
this.startInstant = Instant.now();
}
/**
* Sets the start instant (useful in tests)
*/
public HarvestResultBuilder withStartInstant(Instant startInstant) {
this.startInstant = startInstant;
return this;
}
/**
* Adds a global error (i.e. not specific to any given file)
*/
......
package fr.inra.urgi.rare.harvest;
import java.net.URI;
import java.util.Optional;
import fr.inra.urgi.rare.dao.HarvestResultDao;
import fr.inra.urgi.rare.dto.PageDTO;
import fr.inra.urgi.rare.exception.NotFoundException;
import fr.inra.urgi.rare.harvest.HarvestResult.HarvestResultBuilder;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
......@@ -23,6 +27,8 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
@RequestMapping("/api/harvests")
public class HarvesterController {
public static final int PAGE_SIZE = 10;
private final AsyncHarvester asyncHarvester;
private final HarvestResultDao harvestResultDao;
......@@ -40,17 +46,30 @@ public class HarvesterController {
harvestResultDao.save(temporaryHarvestResult);
asyncHarvester.harvest(resultBuilder);
URI location = ServletUriComponentsBuilder
.fromCurrentRequest()
.path("/{id}")
.buildAndExpand(temporaryHarvestResult.getId())
.toUri();
return ResponseEntity.created(location).build();
return ResponseEntity.created(toDetail(temporaryHarvestResult.getId())).build();
}
@GetMapping("/{id}")
public HarvestResult get(@PathVariable("id") String id) {
return harvestResultDao.findById(id).orElseThrow(NotFoundException::new);
}
@GetMapping
public PageDTO<LightHarvestResultDTO> list(@RequestParam(name = "page") Optional<Integer> page) {
return PageDTO.fromPage(
harvestResultDao.list(PageRequest.of(page.orElse(0), PAGE_SIZE)),
result -> new LightHarvestResultDTO(result.getId(),
toDetail(result.getId()).toString(),
result.getStartInstant(),
result.getEndInstant())
);
}
private URI toDetail(String id) {
return ServletUriComponentsBuilder
.fromCurrentRequest()
.path("/{id}")
.buildAndExpand(id)
.toUri();
}
}
package fr.inra.urgi.rare.harvest;
import java.time.Instant;
import java.util.Objects;
/**
* DTO used when listing the harvest results
* @author JB Nizet
*/
public final class LightHarvestResultDTO {
private final String id;
private final String url;
private final Instant startInstant;
private final Instant endInstant;
public LightHarvestResultDTO(String id, String url, Instant startInstant, Instant endInstant) {
this.id = id;
this.url = url;
this.startInstant = startInstant;
this.endInstant = endInstant;
}
public String getId() {
return id;
}
public String getUrl() {
return url;
}
public Instant getStartInstant() {
return startInstant;
}
public Instant getEndInstant() {
return endInstant;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LightHarvestResultDTO that = (LightHarvestResultDTO) o;
return Objects.equals(id, that.id) &&
Objects.equals(url, that.url) &&
Objects.equals(startInstant, that.startInstant) &&
Objects.equals(endInstant, that.endInstant);
}
@Override
public int hashCode() {
return Objects.hash(id, url, startInstant, endInstant);
}
@Override
public String toString() {
return "LightHarvestResultDTO{" +
"id='" + id + '\'' +
", url='" + url + '\'' +
", startInstant=" + startInstant +
", endInstant=" + endInstant +
'}';
}
}
......@@ -2,14 +2,23 @@ package fr.inra.urgi.rare.dao;
import static org.assertj.core.api.Assertions.assertThat;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import fr.inra.urgi.rare.config.ElasticSearchConfig;
import fr.inra.urgi.rare.harvest.HarvestResult;
import fr.inra.urgi.rare.harvest.HarvestedFile;
import org.assertj.core.groups.Tuple;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.json.JsonTest;
import org.springframework.context.annotation.Import;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
......@@ -25,6 +34,11 @@ class HarvestResultDaoTest {
@Autowired
private HarvestResultDao harvestResultDao;
@BeforeEach
public void prepare() {
harvestResultDao.deleteAll();
}
@Test
void shouldSaveAndGet() {
......@@ -41,4 +55,37 @@ class HarvestResultDaoTest {
assertThat(harvestResultDao.findById(harvestResult.getId()).get()).isEqualTo(harvestResult);
}
@Test
void shouldList() {
Instant now = Instant.now();
List<HarvestResult> harvestResults =
Arrays.asList(
HarvestResult.builder().withStartInstant(now.minus(Duration.ofDays(2))).end(),
HarvestResult.builder().withStartInstant(now.minus(Duration.ofDays(1))).end(),
HarvestResult.builder().withStartInstant(now).build());
harvestResultDao.saveAll(harvestResults);
PageRequest firstPageRequest = PageRequest.of(0, 2);
Page<HarvestResult> firstPage = harvestResultDao.list(firstPageRequest);
assertThat(firstPage.getTotalElements()).isEqualTo(3);
assertThat(firstPage.getTotalPages()).isEqualTo(2);
assertThat(firstPage.getContent())
.extracting(HarvestResult::getId,
HarvestResult::getStartInstant,
HarvestResult::getEndInstant)
.containsExactly(Tuple.tuple(harvestResults.get(2).getId(),
harvestResults.get(2).getStartInstant(),
harvestResults.get(2).getEndInstant()),
Tuple.tuple(harvestResults.get(1).getId(),
harvestResults.get(1).getStartInstant(),
harvestResults.get(1).getEndInstant()));
PageRequest secondPageRequest = PageRequest.of(1, 2);
Page<HarvestResult> secondPage = harvestResultDao.list(secondPageRequest);
assertThat(secondPage.getContent())
.extracting(HarvestResult::getId)
.containsExactly(harvestResults.get(0).getId());
}
}
package fr.inra.urgi.rare.harvest;
import static org.assertj.core.api.Java6Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.*;
import java.io.IOException;
import java.time.Instant;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
......@@ -56,4 +56,21 @@ class HarvestResultTest {
assertThat(firstFile.getErrorCount()).isEqualTo(1);
assertThat(firstFile.getErrors()).hasSize(1);
}
@Test
void shouldUnmarshallPartialResult() throws IOException {
String json = "{\n" +
" \"id\": \"abcd\",\n" +
" \"startInstant\": \"2018-07-25T13:31:00Z\",\n" +
" \"endInstant\": \"2018-07-25T13:31:20Z\"\n" +
"}";
HarvestResult unmarshalled = objectMapper.readValue(json, HarvestResult.class);
assertThat(unmarshalled.getId()).isEqualTo("abcd");
assertThat(unmarshalled.getStartInstant()).isEqualTo(Instant.parse("2018-07-25T13:31:00Z"));
assertThat(unmarshalled.getEndInstant()).isEqualTo(Instant.parse("2018-07-25T13:31:20Z"));
assertThat(unmarshalled.getFiles()).isEmpty();
assertThat(unmarshalled.getGlobalErrors()).isEmpty();
}
}
......@@ -5,6 +5,7 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.util.Arrays;
import java.util.Optional;
import fr.inra.urgi.rare.dao.HarvestResultDao;
......@@ -15,6 +16,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.HttpHeaders;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.web.servlet.MockMvc;
......@@ -59,6 +62,22 @@ class HarvesterControllerTest {
.andExpect(header().string(HttpHeaders.LOCATION, matches("^(.*)/api/harvests/(.+)$")));
}
@Test
void shouldList() throws Exception {
HarvestResult harvestResult = HarvestResult.builder().end();
PageRequest pageRequest = PageRequest.of(0, HarvesterController.PAGE_SIZE);
when(mockHarvestResultDao.list(pageRequest))
.thenReturn(new PageImpl<>(Arrays.asList(harvestResult), pageRequest, 1));
mockMvc.perform(get("/api/harvests", harvestResult.getId()))
.andExpect(status().isOk())
.andExpect(jsonPath("$.number").value(0))
.andExpect(jsonPath("$.content[0].id").value(harvestResult.getId()))
.andExpect(jsonPath("$.content[0].startInstant").value(harvestResult.getStartInstant().toString()))
.andExpect(jsonPath("$.content[0].endInstant").value(harvestResult.getEndInstant().toString()));
}
private static Matcher<String> matches(String regexp) {
return new CustomTypeSafeMatcher<String>("should match regexp " + regexp) {
@Override
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment