Commit a3050d84 authored by Jean-Baptiste Nizet's avatar Jean-Baptiste Nizet
Browse files

feat: implement export of germplasms

parent b42b497f
......@@ -12,6 +12,7 @@ import fr.inra.urgi.faidare.domain.response.PaginatedList;
import fr.inra.urgi.faidare.elasticsearch.repository.ESFindRepository;
import java.util.Iterator;
import java.util.Set;
public interface GermplasmRepository
extends ESFindRepository<GermplasmSearchCriteria, GermplasmVO> {
......@@ -52,6 +53,11 @@ public interface GermplasmRepository
*/
Iterator<GermplasmMcpdVO> scrollAllGermplasmMcpd(FaidareGermplasmPOSTShearchCriteria criteria);
/**
* Scroll through all germplasmMcpd having one of the given IDs.
*/
Iterator<GermplasmMcpdVO> scrollGermplasmMcpdsByIds(Set<String> ids, int fetchSize);
/**
* Find pedigree for germplasm by id.
*/
......
......@@ -45,6 +45,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import static org.elasticsearch.search.aggregations.AggregationBuilders.filter;
import static org.elasticsearch.search.aggregations.AggregationBuilders.terms;
......@@ -118,6 +119,12 @@ public class GermplasmRepositoryImpl implements GermplasmRepository {
return new ESScrollIterator<>(client, requestFactory, parser, GermplasmMcpdVO.class, query, fetchSize);
}
@Override
public Iterator<GermplasmMcpdVO> scrollGermplasmMcpdsByIds(Set<String> ids, int fetchSize) {
QueryBuilder query = QueryBuilders.idsQuery().addIds(ids.toArray(new String[0]));
return new ESScrollIterator<>(client, requestFactory, parser, GermplasmMcpdVO.class, query, fetchSize);
}
@Override
public GermplasmVO getById(String germplasmDbId) {
return getByIdRepository.getById(germplasmDbId);
......
......@@ -6,14 +6,8 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Spliterators;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.servlet.http.HttpServletRequest;
import com.google.common.collect.Streams;
import fr.inra.urgi.faidare.api.NotFoundException;
import fr.inra.urgi.faidare.config.FaidareProperties;
import fr.inra.urgi.faidare.domain.brapi.v1.data.BrapiGermplasmAttributeValue;
......@@ -25,6 +19,7 @@ import fr.inra.urgi.faidare.domain.data.germplasm.DonorVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GenealogyVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmAttributeValueVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmInstituteVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmMcpdVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmSitemapVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmVO;
import fr.inra.urgi.faidare.domain.data.germplasm.InstituteVO;
......@@ -34,7 +29,6 @@ import fr.inra.urgi.faidare.domain.data.germplasm.PuiNameValueVO;
import fr.inra.urgi.faidare.domain.data.germplasm.SiblingVO;
import fr.inra.urgi.faidare.domain.data.germplasm.SiteVO;
import fr.inra.urgi.faidare.domain.data.germplasm.TaxonSourceVO;
import fr.inra.urgi.faidare.domain.xref.XRefDocumentSearchCriteria;
import fr.inra.urgi.faidare.domain.xref.XRefDocumentVO;
import fr.inra.urgi.faidare.repository.es.GermplasmAttributeRepository;
import fr.inra.urgi.faidare.repository.es.GermplasmRepository;
......@@ -43,8 +37,11 @@ import fr.inra.urgi.faidare.utils.Sitemaps;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
......@@ -63,15 +60,18 @@ public class GermplasmController {
private final FaidareProperties faidareProperties;
private final XRefDocumentRepository xRefDocumentRepository;
private final GermplasmAttributeRepository germplasmAttributeRepository;
private final GermplasmExportService germplasmExportService;
public GermplasmController(GermplasmRepository germplasmRepository,
FaidareProperties faidareProperties,
XRefDocumentRepository xRefDocumentRepository,
GermplasmAttributeRepository germplasmAttributeRepository) {
GermplasmAttributeRepository germplasmAttributeRepository,
GermplasmExportService germplasmExportService) {
this.germplasmRepository = germplasmRepository;
this.faidareProperties = faidareProperties;
this.xRefDocumentRepository = xRefDocumentRepository;
this.germplasmAttributeRepository = germplasmAttributeRepository;
this.germplasmExportService = germplasmExportService;
}
@GetMapping("/{germplasmId}")
......@@ -100,8 +100,19 @@ public class GermplasmController {
return toModelAndView(germplasms.get(0));
}
@PostMapping("/exports")
@ResponseBody
public ResponseEntity<StreamingResponseBody> export(@Validated @RequestBody GermplasmExportCommand command) {
List<GermplasmExportableField> fields = getFieldsToExport(command);
@GetMapping(value = "/sitemap-{index}.txt")
StreamingResponseBody body = out -> {
Iterator<GermplasmMcpdVO> iterator = germplasmRepository.scrollGermplasmMcpdsByIds(command.getIds(), 1000);
germplasmExportService.export(out, iterator, fields);
};
return ResponseEntity.ok().contentType(MediaType.parseMediaType("text/csv")).body(body);
}
@GetMapping("/sitemap-{index}.txt")
@ResponseBody
public ResponseEntity<StreamingResponseBody> sitemap(@PathVariable("index") int index) {
if (index < 0 || index >= Sitemaps.BUCKET_COUNT) {
......@@ -440,4 +451,12 @@ public class GermplasmController {
xref.setEntryType("type " + name);
return xref;
}
private List<GermplasmExportableField> getFieldsToExport(GermplasmExportCommand command) {
List<GermplasmExportableField> fields = command.getFields();
if (fields.isEmpty()) {
fields = Arrays.asList(GermplasmExportableField.values());
}
return fields;
}
}
package fr.inra.urgi.faidare.web.germplasm;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import javax.validation.constraints.NotEmpty;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
/**
* Command sent to export a list of germplasm IDs
* @author JB Nizet
*/
public class GermplasmExportCommand {
@NotEmpty
private final Set<String> ids;
/**
* The ordered list of fields to export. If empty, all fields are exported
*/
private final List<GermplasmExportableField> fields;
@JsonCreator
public GermplasmExportCommand(@JsonProperty("ids") Set<String> ids,
@JsonProperty("fields") List<GermplasmExportableField> fields) {
this.ids = ids;
this.fields = fields == null ? Collections.emptyList() : fields;
}
public Set<String> getIds() {
return ids;
}
public List<GermplasmExportableField> getFields() {
return fields;
}
}
package fr.inra.urgi.faidare.web.germplasm;
import static fr.inra.urgi.faidare.web.germplasm.GermplasmExportableField.*;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import com.opencsv.CSVWriter;
import fr.inra.urgi.faidare.domain.data.germplasm.DonorInfoVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmMcpdVO;
import fr.inra.urgi.faidare.domain.data.germplasm.InstituteVO;
import org.springframework.stereotype.Component;
/**
* Service allowing to export germplasms as CSV
* @author JB Nizet
*/
@Component
public class GermplasmExportService {
private final Map<GermplasmExportableField, GermplasmExportableFieldDescriptor> descriptors;
public GermplasmExportService() {
Map<GermplasmExportableField, GermplasmExportableFieldDescriptor> map = new HashMap<>();
map.put(PUID, withFieldAsHeader(PUID, vo -> vo.getGermplasmPUI()));
map.put(INSTCODE, withFieldAsHeader(INSTCODE, vo -> vo.getInstituteCode()));
map.put(ACCENUMB, withFieldAsHeader(ACCENUMB, vo -> vo.getAccessionNumber()));
map.put(COLLNUMB, withFieldAsHeader(COLLNUMB, vo -> vo.getCollectingInfo().getCollectingNumber()));
map.put(COLLCODE, withFieldAsHeader(COLLCODE, vo ->
vo.getCollectingInfo()
.getCollectingInstitutes()
.stream()
.map(InstituteVO::getInstituteCode).collect(Collectors.joining(";"))));
map.put(COLLNAME, withFieldAsHeader(COLLNAME, vo ->
vo.getCollectingInfo()
.getCollectingInstitutes()
.stream()
.map(InstituteVO::getInstituteName)
.collect(Collectors.joining(";"))));
map.put(COLLINSTADDRESS, withFieldAsHeader(COLLINSTADDRESS, vo ->
vo.getCollectingInfo()
.getCollectingInstitutes()
.stream()
.map(InstituteVO::getAddress)
.collect(Collectors.joining(";"))));
map.put(COLLMISSID, withFieldAsHeader(COLLMISSID, vo -> vo.getCollectingInfo().getCollectingMissionIdentifier()));
map.put(GENUS, withFieldAsHeader(GENUS, vo -> vo.getGenus()));
map.put(SPECIES, withFieldAsHeader(SPECIES, vo -> vo.getSpecies()));
map.put(SPAUTHOR, withFieldAsHeader(SPAUTHOR, vo -> vo.getSpeciesAuthority()));
map.put(SUBTAXA, withFieldAsHeader(SUBTAXA, vo -> vo.getSubtaxon()));
map.put(SUBTAUTHOR, withFieldAsHeader(SUBTAUTHOR, vo -> vo.getSubtaxonAuthority()));
map.put(CROPNAME, withFieldAsHeader(CROPNAME, vo -> vo.getCommonCropName()));
map.put(ACCENAME, withFieldAsHeader(ACCENAME, vo -> String.join(";", vo.getAccessionNames())));
map.put(ACQDATE, withFieldAsHeader(ACQDATE, vo -> vo.getAcquisitionDate()));
map.put(ORIGCTY, withFieldAsHeader(ORIGCTY, vo -> vo.getCountryOfOriginCode()));
map.put(COLLSITE, withFieldAsHeader(COLLSITE, vo -> vo.getCollectingInfo().getCollectingSite().getSiteName()));
map.put(DECLATITUDE, withFieldAsHeader(DECLATITUDE, vo -> vo.getCollectingInfo().getCollectingSite().getLatitudeDecimal()));
map.put(LATITUDE, withFieldAsHeader(LATITUDE, vo -> vo.getCollectingInfo().getCollectingSite().getLatitudeDegrees()));
map.put(DECLONGITUDE, withFieldAsHeader(DECLONGITUDE, vo -> vo.getCollectingInfo().getCollectingSite().getLongitudeDecimal()));
map.put(LONGITUDE, withFieldAsHeader(LONGITUDE, vo -> vo.getCollectingInfo().getCollectingSite().getLongitudeDegrees()));
map.put(COORDUNCERT, withFieldAsHeader(COORDUNCERT, vo -> vo.getCollectingInfo().getCollectingSite().getCoordinateUncertainty()));
map.put(COORDDATUM, withFieldAsHeader(COORDDATUM, vo -> vo.getCollectingInfo().getCollectingSite().getSpatialReferenceSystem()));
map.put(GEOREFMETH, withFieldAsHeader(GEOREFMETH, vo -> vo.getCollectingInfo().getCollectingSite().getGeoreferencingMethod()));
map.put(ELEVATION, withFieldAsHeader(ELEVATION, vo -> vo.getCollectingInfo().getCollectingSite().getElevation()));
map.put(COLLDATE, withFieldAsHeader(COLLDATE, vo -> vo.getCollectingInfo().getCollectingDate()));
map.put(BREDCODE, withFieldAsHeader(BREDCODE, vo ->
vo.getBreedingInstitutes()
.stream()
.map(InstituteVO::getInstituteCode)
.collect(Collectors.joining(";"))));
map.put(BREDNAME, withFieldAsHeader(BREDNAME, vo ->
vo.getBreedingInstitutes()
.stream()
.map(InstituteVO::getInstituteName)
.collect(Collectors.joining(";"))));
map.put(SAMPSTAT, withFieldAsHeader(SAMPSTAT, vo -> vo.getBiologicalStatusOfAccessionCode()));
map.put(ANCEST, withFieldAsHeader(ANCEST, vo -> vo.getAncestralData()));
map.put(COLLSRC, withFieldAsHeader(COLLSRC, vo -> vo.getAcquisitionSourceCode()));
map.put(DONORCODE, withFieldAsHeader(DONORCODE, vo ->
vo.getDonorInfo()
.stream()
.map(donorInfoVO -> donorInfoVO.getDonorInstitute().getInstituteCode())
.collect(Collectors.joining(";"))));
map.put(DONORNAME, withFieldAsHeader(DONORNAME, vo ->
vo.getDonorInfo()
.stream()
.map(donorInfoVO -> donorInfoVO.getDonorInstitute().getInstituteName())
.collect(Collectors.joining(";"))));
map.put(DONORNUMB, withFieldAsHeader(DONORNUMB, vo ->
vo.getDonorInfo()
.stream()
.map(DonorInfoVO::getDonorAccessionNumber)
.collect(Collectors.joining(";"))));
map.put(OTHERNUMB, withFieldAsHeader(OTHERNUMB, vo -> String.join(";", vo.getAlternateIDs())));
map.put(DUPLSITE, withFieldAsHeader(DUPLSITE, vo ->
vo.getSafetyDuplicateInstitutes()
.stream()
.map(InstituteVO::getInstituteCode)
.collect(Collectors.joining(";"))));
map.put(DUPLINSTNAME, withFieldAsHeader(DUPLINSTNAME, vo ->
vo.getSafetyDuplicateInstitutes()
.stream()
.map(InstituteVO::getInstituteName)
.collect(Collectors.joining(";"))));
map.put(STORAGE, withFieldAsHeader(STORAGE, vo -> String.join(";", vo.getStorageTypeCodes())));
map.put(MLSSTAT, withFieldAsHeader(MLSSTAT, vo -> vo.getMlsStatus()));
map.put(REMARKS, withFieldAsHeader(REMARKS, vo -> vo.getRemarks()));
this.descriptors = Collections.unmodifiableMap(map);
if (map.size() != GermplasmExportableField.values().length) {
throw new IllegalStateException("Missing field descriptor");
}
}
public void export(OutputStream out, Iterator<GermplasmMcpdVO> germplasms, List<GermplasmExportableField> fields) {
try {
CSVWriter csvWriter = new CSVWriter(new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8)), ';', '"', '\\', "\n");
String[] header = fields.stream()
.map(descriptors::get)
.map(GermplasmExportableFieldDescriptor::getHeader)
.toArray(String[]::new);
csvWriter.writeNext(header);
while (germplasms.hasNext()) {
GermplasmMcpdVO vo = germplasms.next();
String[] line =
fields.stream()
.map(descriptors::get)
.map(descriptor -> descriptor.export(vo))
.toArray(String[]::new);
csvWriter.writeNext(line);
}
csvWriter.flush();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
}
private GermplasmExportableFieldDescriptor withFieldAsHeader(GermplasmExportableField field,
Function<GermplasmMcpdVO, String> exporter) {
return new GermplasmExportableFieldDescriptor(field.name(), exporter);
}
private static class GermplasmExportableFieldDescriptor {
private final String header;
private final Function<GermplasmMcpdVO, String> exporter;
public GermplasmExportableFieldDescriptor(String header,
Function<GermplasmMcpdVO, String> exporter) {
this.header = header;
this.exporter = exporter;
}
public String getHeader() {
return this.header;
}
public String export(GermplasmMcpdVO germplasm) {
return this.exporter.apply(germplasm);
}
}
}
package fr.inra.urgi.faidare.web.germplasm;
import com.fasterxml.jackson.annotation.JsonValue;
/**
* The fields of a germplasm that can be exported
*
* @author JB Nizet
*/
public enum GermplasmExportableField {
PUID,
INSTCODE,
ACCENUMB,
COLLNUMB,
COLLCODE,
COLLNAME,
COLLINSTADDRESS,
COLLMISSID,
GENUS,
SPECIES,
SPAUTHOR,
SUBTAXA,
SUBTAUTHOR,
CROPNAME,
ACCENAME,
ACQDATE,
ORIGCTY,
COLLSITE,
DECLATITUDE,
LATITUDE,
DECLONGITUDE,
LONGITUDE,
COORDUNCERT,
COORDDATUM,
GEOREFMETH,
ELEVATION,
COLLDATE,
BREDCODE,
BREDNAME,
SAMPSTAT,
ANCEST,
COLLSRC,
DONORCODE,
DONORNAME,
DONORNUMB,
OTHERNUMB,
DUPLSITE,
DUPLINSTNAME,
STORAGE,
MLSSTAT,
REMARKS
}
......@@ -20,6 +20,7 @@ import fr.inra.urgi.faidare.domain.data.germplasm.GenealogyVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmAttributeValueListVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmAttributeValueVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmInstituteVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmMcpdVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmVO;
import fr.inra.urgi.faidare.domain.data.germplasm.InstituteVO;
import fr.inra.urgi.faidare.domain.data.germplasm.PedigreeVO;
......@@ -384,4 +385,11 @@ public class Fixtures {
variable.setTrait(trait);
return variable;
}
public static GermplasmMcpdVO createGermplasmMcpd() {
GermplasmMcpdVO result = new GermplasmMcpdVO();
result.setGermplasmPUI("PUI1");
result.setInstituteCode("Inst1");
return result;
}
}
package fr.inra.urgi.faidare.web.germplasm;
import static fr.inra.urgi.faidare.web.Fixtures.htmlContent;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import fr.inra.urgi.faidare.config.FaidareProperties;
import fr.inra.urgi.faidare.domain.brapi.v1.data.BrapiGermplasmAttributeValue;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmAttributeValueListVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmMcpdVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmSitemapVO;
import fr.inra.urgi.faidare.domain.data.germplasm.GermplasmVO;
import fr.inra.urgi.faidare.domain.data.study.StudySitemapVO;
import fr.inra.urgi.faidare.domain.datadiscovery.data.DataSource;
import fr.inra.urgi.faidare.domain.response.PaginatedList;
import fr.inra.urgi.faidare.domain.xref.XRefDocumentSearchCriteria;
import fr.inra.urgi.faidare.domain.xref.XRefDocumentVO;
import fr.inra.urgi.faidare.repository.es.GermplasmAttributeRepository;
import fr.inra.urgi.faidare.repository.es.GermplasmRepository;
import fr.inra.urgi.faidare.repository.es.XRefDocumentRepository;
import fr.inra.urgi.faidare.utils.Sitemaps;
import fr.inra.urgi.faidare.web.Fixtures;
import fr.inra.urgi.faidare.web.study.StudyController;
import fr.inra.urgi.faidare.web.thymeleaf.CoordinatesDialect;
import fr.inra.urgi.faidare.web.thymeleaf.FaidareDialect;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -45,11 +40,15 @@ import org.springframework.test.web.servlet.MvcResult;
* @author JB Nizet
*/
@WebMvcTest(GermplasmController.class)
@Import(GermplasmExportService.class)
public class GermplasmControllerTest {
@Autowired
private MockMvc mockMvc;
@Autowired
private ObjectMapper objectMapper;
@MockBean
private GermplasmRepository mockGermplasmRepository;
......@@ -131,6 +130,35 @@ public class GermplasmControllerTest {
.andExpect(status().isNotFound());
}
@Test
void shouldExportGermplasms() throws Exception {
List<GermplasmMcpdVO> germplasms = Arrays.asList(
Fixtures.createGermplasmMcpd(),
Fixtures.createGermplasmMcpd()
);
GermplasmExportCommand command = new GermplasmExportCommand(
Sets.newHashSet("g1", "g2"),
Arrays.asList(GermplasmExportableField.PUID, GermplasmExportableField.INSTCODE));
when(mockGermplasmRepository.scrollGermplasmMcpdsByIds(eq(command.getIds()), anyInt()))
.thenAnswer(invocation -> germplasms.iterator());
MvcResult mvcResult = mockMvc.perform(post("/germplasms/exports")
.contentType(MediaType.APPLICATION_JSON)
.content(objectMapper.writeValueAsBytes(
command)))
.andExpect(request().asyncStarted())
.andReturn();
this.mockMvc.perform(asyncDispatch(mvcResult))
.andExpect(status().isOk())
.andExpect(content().contentType("text/csv"))
.andExpect(content().string("\"PUID\";\"INSTCODE\"\n" +
"\"PUI1\";\"Inst1\"\n" +
"\"PUI1\";\"Inst1\"\n"));
}
private void testSitemap(int index, String expectedContent) throws Exception {
MvcResult mvcResult = mockMvc.perform(get("/faidare/germplasms/sitemap-" + index + ".txt")
.contextPath("/faidare"))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment