Commit 4755a0d4 authored by Jean-Baptiste Nizet's avatar Jean-Baptiste Nizet
Browse files

feat: add list of harvests

Note that this feature revealed that the version of spring-data-elasticsearch that we're using was not compatible with the spring-data-commons version used by Spring Boot. So I upgraded the spring-data-commons version.
parent a50d8d55
...@@ -151,3 +151,13 @@ It's only when the property `endInstant` of the returned JSON is non-null that t ...@@ -151,3 +151,13 @@ It's only when the property `endInstant` of the returned JSON is non-null that t
"startInstant": "2018-07-24T12:56:27.322Z" "startInstant": "2018-07-24T12:56:27.322Z"
} }
``` ```
In case you lost the response to the POST request and thus don't know what the URL of the harvest is,
you can list the harvests, in descending order of their start instant, by sending a GET request to
`/api/harvests`:
http --auth rare:f01a7031fc17 GET http://localhost:8080/api/harvests
or
curl -u rare:f01a7031fc17 http://localhost:8080/api/harvests
...@@ -73,7 +73,7 @@ dependencies { ...@@ -73,7 +73,7 @@ dependencies {
implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-web")
implementation("org.springframework.boot:spring-boot-starter-actuator") implementation("org.springframework.boot:spring-boot-starter-actuator")
implementation("org.springframework.boot:spring-boot-starter-security") implementation("org.springframework.boot:spring-boot-starter-security")
implementation("org.springframework.boot:spring-boot-starter-webflux") implementation("org.springframework.data:spring-data-commons:2.1.0.M3")
implementation("org.springframework.data:spring-data-elasticsearch:3.1.0.M3") implementation("org.springframework.data:spring-data-elasticsearch:3.1.0.M3")
implementation("org.elasticsearch:elasticsearch:6.3.1") implementation("org.elasticsearch:elasticsearch:6.3.1")
implementation("org.elasticsearch.client:transport:6.3.1") implementation("org.elasticsearch.client:transport:6.3.1")
......
...@@ -4,6 +4,7 @@ import java.net.InetAddress; ...@@ -4,6 +4,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import fr.inra.urgi.rare.dao.GeneticResourceDao;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
...@@ -17,7 +18,7 @@ import org.springframework.data.elasticsearch.core.EntityMapper; ...@@ -17,7 +18,7 @@ import org.springframework.data.elasticsearch.core.EntityMapper;
import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories; import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories;
@Configuration @Configuration
@EnableElasticsearchRepositories(basePackages = "fr.inra.urgi.rare.dao") @EnableElasticsearchRepositories(basePackageClasses = GeneticResourceDao.class)
public class ElasticSearchConfig { public class ElasticSearchConfig {
@Value("${spring.data.elasticsearch.cluster.name}") @Value("${spring.data.elasticsearch.cluster.name}")
......
...@@ -6,5 +6,5 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository ...@@ -6,5 +6,5 @@ import org.springframework.data.elasticsearch.repository.ElasticsearchRepository
/** /**
* DAO for {@link HarvestResult} * DAO for {@link HarvestResult}
*/ */
public interface HarvestResultDao extends ElasticsearchRepository<HarvestResult, String> { public interface HarvestResultDao extends ElasticsearchRepository<HarvestResult, String>, HarvestResultDaoCustom {
} }
package fr.inra.urgi.rare.dao;
import fr.inra.urgi.rare.harvest.HarvestResult;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
/**
* Custom methods of {@link HarvestResultDao}
* @author JB Nizet
*/
public interface HarvestResultDaoCustom {
/**
* Lists the {@link HarvestResult} of the given page, sorted by start instant, in descending order (most recent
* first).
* @return a page of <strong>partial</strong> {@link HarvestResult} instances, containing only the ID, start instant
* and end instant.
*/
Page<HarvestResult> list(Pageable page);
}
package fr.inra.urgi.rare.dao;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import fr.inra.urgi.rare.harvest.HarvestResult;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.FetchSourceFilterBuilder;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
/**
* Implementation of {@link HarvestResultDaoCustom}
* @author JB Nizet
*/
public class HarvestResultDaoImpl implements HarvestResultDaoCustom {
private final ElasticsearchTemplate elasticsearchTemplate;
public HarvestResultDaoImpl(ElasticsearchTemplate elasticsearchTemplate) {
this.elasticsearchTemplate = elasticsearchTemplate;
}
@Override
public Page<HarvestResult> list(Pageable page) {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder()
.withQuery(matchAllQuery())
.withSourceFilter(new FetchSourceFilterBuilder().withIncludes("id", "startInstant", "endInstant").build())
.withSort(SortBuilders.fieldSort("startInstant").order(SortOrder.DESC))
.withPageable(page)
.build();
return this.elasticsearchTemplate.queryForPage(searchQuery, HarvestResult.class);
}
}
package fr.inra.urgi.rare.dto; package fr.inra.urgi.rare.dto;
import java.util.List; import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.data.domain.Page; import org.springframework.data.domain.Page;
...@@ -51,6 +53,14 @@ public final class PageDTO<T> { ...@@ -51,6 +53,14 @@ public final class PageDTO<T> {
page.getTotalPages()); page.getTotalPages());
} }
public static <T, R> PageDTO<R> fromPage(Page<T> page, Function<T, R> mapper) {
return new PageDTO<R>(page.getContent().stream().map(mapper).collect(Collectors.toList()),
page.getNumber(),
page.getSize(),
page.getTotalElements(),
page.getTotalPages());
}
public List<T> getContent() { public List<T> getContent() {
return content; return content;
} }
......
...@@ -53,8 +53,8 @@ public final class HarvestResult { ...@@ -53,8 +53,8 @@ public final class HarvestResult {
this.id = id; this.id = id;
this.startInstant = startInstant; this.startInstant = startInstant;
this.endInstant = endInstant; this.endInstant = endInstant;
this.globalErrors = Collections.unmodifiableList(new ArrayList<>(globalErrors)); this.globalErrors = globalErrors == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(globalErrors));
this.files = Collections.unmodifiableList(new ArrayList<>(files)); this.files = files == null ? Collections.emptyList() : Collections.unmodifiableList(new ArrayList<>(files));
} }
public String getId() { public String getId() {
...@@ -118,7 +118,7 @@ public final class HarvestResult { ...@@ -118,7 +118,7 @@ public final class HarvestResult {
*/ */
public static final class HarvestResultBuilder { public static final class HarvestResultBuilder {
private final String id; private final String id;
private final Instant startInstant; private Instant startInstant;
private Instant endInstant; private Instant endInstant;
private final List<HarvestedFile> files = new ArrayList<>(); private final List<HarvestedFile> files = new ArrayList<>();
private final List<String> globalErrors = new ArrayList<>(); private final List<String> globalErrors = new ArrayList<>();
...@@ -128,6 +128,14 @@ public final class HarvestResult { ...@@ -128,6 +128,14 @@ public final class HarvestResult {
this.startInstant = Instant.now(); this.startInstant = Instant.now();
} }
/**
* Sets the start instant (useful in tests)
*/
public HarvestResultBuilder withStartInstant(Instant startInstant) {
this.startInstant = startInstant;
return this;
}
/** /**
* Adds a global error (i.e. not specific to any given file) * Adds a global error (i.e. not specific to any given file)
*/ */
......
package fr.inra.urgi.rare.harvest; package fr.inra.urgi.rare.harvest;
import java.net.URI; import java.net.URI;
import java.util.Optional;
import fr.inra.urgi.rare.dao.HarvestResultDao; import fr.inra.urgi.rare.dao.HarvestResultDao;
import fr.inra.urgi.rare.dto.PageDTO;
import fr.inra.urgi.rare.exception.NotFoundException; import fr.inra.urgi.rare.exception.NotFoundException;
import fr.inra.urgi.rare.harvest.HarvestResult.HarvestResultBuilder; import fr.inra.urgi.rare.harvest.HarvestResult.HarvestResultBuilder;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController; import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
...@@ -23,6 +27,8 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder; ...@@ -23,6 +27,8 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
@RequestMapping("/api/harvests") @RequestMapping("/api/harvests")
public class HarvesterController { public class HarvesterController {
public static final int PAGE_SIZE = 10;
private final AsyncHarvester asyncHarvester; private final AsyncHarvester asyncHarvester;
private final HarvestResultDao harvestResultDao; private final HarvestResultDao harvestResultDao;
...@@ -40,17 +46,30 @@ public class HarvesterController { ...@@ -40,17 +46,30 @@ public class HarvesterController {
harvestResultDao.save(temporaryHarvestResult); harvestResultDao.save(temporaryHarvestResult);
asyncHarvester.harvest(resultBuilder); asyncHarvester.harvest(resultBuilder);
URI location = ServletUriComponentsBuilder return ResponseEntity.created(toDetail(temporaryHarvestResult.getId())).build();
.fromCurrentRequest()
.path("/{id}")
.buildAndExpand(temporaryHarvestResult.getId())
.toUri();
return ResponseEntity.created(location).build();
} }
@GetMapping("/{id}") @GetMapping("/{id}")
public HarvestResult get(@PathVariable("id") String id) { public HarvestResult get(@PathVariable("id") String id) {
return harvestResultDao.findById(id).orElseThrow(NotFoundException::new); return harvestResultDao.findById(id).orElseThrow(NotFoundException::new);
} }
@GetMapping
public PageDTO<LightHarvestResultDTO> list(@RequestParam(name = "page") Optional<Integer> page) {
return PageDTO.fromPage(
harvestResultDao.list(PageRequest.of(page.orElse(0), PAGE_SIZE)),
result -> new LightHarvestResultDTO(result.getId(),
toDetail(result.getId()).toString(),
result.getStartInstant(),
result.getEndInstant())
);
}
private URI toDetail(String id) {
return ServletUriComponentsBuilder
.fromCurrentRequest()
.path("/{id}")
.buildAndExpand(id)
.toUri();
}
} }
package fr.inra.urgi.rare.harvest;
import java.time.Instant;
import java.util.Objects;
/**
* DTO used when listing the harvest results
* @author JB Nizet
*/
public final class LightHarvestResultDTO {
private final String id;
private final String url;
private final Instant startInstant;
private final Instant endInstant;
public LightHarvestResultDTO(String id, String url, Instant startInstant, Instant endInstant) {
this.id = id;
this.url = url;
this.startInstant = startInstant;
this.endInstant = endInstant;
}
public String getId() {
return id;
}
public String getUrl() {
return url;
}
public Instant getStartInstant() {
return startInstant;
}
public Instant getEndInstant() {
return endInstant;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
LightHarvestResultDTO that = (LightHarvestResultDTO) o;
return Objects.equals(id, that.id) &&
Objects.equals(url, that.url) &&
Objects.equals(startInstant, that.startInstant) &&
Objects.equals(endInstant, that.endInstant);
}
@Override
public int hashCode() {
return Objects.hash(id, url, startInstant, endInstant);
}
@Override
public String toString() {
return "LightHarvestResultDTO{" +
"id='" + id + '\'' +
", url='" + url + '\'' +
", startInstant=" + startInstant +
", endInstant=" + endInstant +
'}';
}
}
...@@ -2,14 +2,23 @@ package fr.inra.urgi.rare.dao; ...@@ -2,14 +2,23 @@ package fr.inra.urgi.rare.dao;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import fr.inra.urgi.rare.config.ElasticSearchConfig; import fr.inra.urgi.rare.config.ElasticSearchConfig;
import fr.inra.urgi.rare.harvest.HarvestResult; import fr.inra.urgi.rare.harvest.HarvestResult;
import fr.inra.urgi.rare.harvest.HarvestedFile; import fr.inra.urgi.rare.harvest.HarvestedFile;
import org.assertj.core.groups.Tuple;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.json.JsonTest; import org.springframework.boot.test.autoconfigure.json.JsonTest;
import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Import;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
...@@ -25,6 +34,11 @@ class HarvestResultDaoTest { ...@@ -25,6 +34,11 @@ class HarvestResultDaoTest {
@Autowired @Autowired
private HarvestResultDao harvestResultDao; private HarvestResultDao harvestResultDao;
@BeforeEach
public void prepare() {
harvestResultDao.deleteAll();
}
@Test @Test
void shouldSaveAndGet() { void shouldSaveAndGet() {
...@@ -41,4 +55,37 @@ class HarvestResultDaoTest { ...@@ -41,4 +55,37 @@ class HarvestResultDaoTest {
assertThat(harvestResultDao.findById(harvestResult.getId()).get()).isEqualTo(harvestResult); assertThat(harvestResultDao.findById(harvestResult.getId()).get()).isEqualTo(harvestResult);
} }
@Test
void shouldList() {
Instant now = Instant.now();
List<HarvestResult> harvestResults =
Arrays.asList(
HarvestResult.builder().withStartInstant(now.minus(Duration.ofDays(2))).end(),
HarvestResult.builder().withStartInstant(now.minus(Duration.ofDays(1))).end(),
HarvestResult.builder().withStartInstant(now).build());
harvestResultDao.saveAll(harvestResults);
PageRequest firstPageRequest = PageRequest.of(0, 2);
Page<HarvestResult> firstPage = harvestResultDao.list(firstPageRequest);
assertThat(firstPage.getTotalElements()).isEqualTo(3);
assertThat(firstPage.getTotalPages()).isEqualTo(2);
assertThat(firstPage.getContent())
.extracting(HarvestResult::getId,
HarvestResult::getStartInstant,
HarvestResult::getEndInstant)
.containsExactly(Tuple.tuple(harvestResults.get(2).getId(),
harvestResults.get(2).getStartInstant(),
harvestResults.get(2).getEndInstant()),
Tuple.tuple(harvestResults.get(1).getId(),
harvestResults.get(1).getStartInstant(),
harvestResults.get(1).getEndInstant()));
PageRequest secondPageRequest = PageRequest.of(1, 2);
Page<HarvestResult> secondPage = harvestResultDao.list(secondPageRequest);
assertThat(secondPage.getContent())
.extracting(HarvestResult::getId)
.containsExactly(harvestResults.get(0).getId());
}
} }
package fr.inra.urgi.rare.harvest; package fr.inra.urgi.rare.harvest;
import static org.assertj.core.api.Java6Assertions.assertThat; import static org.assertj.core.api.Java6Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.*;
import java.io.IOException; import java.io.IOException;
import java.time.Instant;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
...@@ -56,4 +56,21 @@ class HarvestResultTest { ...@@ -56,4 +56,21 @@ class HarvestResultTest {
assertThat(firstFile.getErrorCount()).isEqualTo(1); assertThat(firstFile.getErrorCount()).isEqualTo(1);
assertThat(firstFile.getErrors()).hasSize(1); assertThat(firstFile.getErrors()).hasSize(1);
} }
@Test
void shouldUnmarshallPartialResult() throws IOException {
String json = "{\n" +
" \"id\": \"abcd\",\n" +
" \"startInstant\": \"2018-07-25T13:31:00Z\",\n" +
" \"endInstant\": \"2018-07-25T13:31:20Z\"\n" +
"}";
HarvestResult unmarshalled = objectMapper.readValue(json, HarvestResult.class);
assertThat(unmarshalled.getId()).isEqualTo("abcd");
assertThat(unmarshalled.getStartInstant()).isEqualTo(Instant.parse("2018-07-25T13:31:00Z"));
assertThat(unmarshalled.getEndInstant()).isEqualTo(Instant.parse("2018-07-25T13:31:20Z"));
assertThat(unmarshalled.getFiles()).isEmpty();
assertThat(unmarshalled.getGlobalErrors()).isEmpty();
}
} }
...@@ -5,6 +5,7 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder ...@@ -5,6 +5,7 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.util.Arrays;
import java.util.Optional; import java.util.Optional;
import fr.inra.urgi.rare.dao.HarvestResultDao; import fr.inra.urgi.rare.dao.HarvestResultDao;
...@@ -15,6 +16,8 @@ import org.junit.jupiter.api.extension.ExtendWith; ...@@ -15,6 +16,8 @@ import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.domain.PageRequest;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.MockMvc;
...@@ -59,6 +62,22 @@ class HarvesterControllerTest { ...@@ -59,6 +62,22 @@ class HarvesterControllerTest {
.andExpect(header().string(HttpHeaders.LOCATION, matches("^(.*)/api/harvests/(.+)$"))); .andExpect(header().string(HttpHeaders.LOCATION, matches("^(.*)/api/harvests/(.+)$")));
} }
@Test
void shouldList() throws Exception {
HarvestResult harvestResult = HarvestResult.builder().end();
PageRequest pageRequest = PageRequest.of(0, HarvesterController.PAGE_SIZE);
when(mockHarvestResultDao.list(pageRequest))
.thenReturn(new PageImpl<>(Arrays.asList(harvestResult), pageRequest, 1));
mockMvc.perform(get("/api/harvests", harvestResult.getId()))
.andExpect(status().isOk())
.andExpect(jsonPath("$.number").value(0))
.andExpect(jsonPath("$.content[0].id").value(harvestResult.getId()))
.andExpect(jsonPath("$.content[0].startInstant").value(harvestResult.getStartInstant().toString()))
.andExpect(jsonPath("$.content[0].endInstant").value(harvestResult.getEndInstant().toString()));
}
private static Matcher<String> matches(String regexp) { private static Matcher<String> matches(String regexp) {
return new CustomTypeSafeMatcher<String>("should match regexp " + regexp) { return new CustomTypeSafeMatcher<String>("should match regexp " + regexp) {
@Override @Override
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment