doc: generate REST API documentation of harvest service

README.md

@@ -82,82 +82,9 @@ environment variable for example).
 
 The files must have the extension `.json`, and must be stored in that directory (not in a sub-directory).
 Once the files are ready and the server is started, the harvest is triggered by sending a POST request
-to the endpoint `/api/harvests`, without any request body.
-This endpoint, as well as the actuator endpoints, is only accessible to an authenticated user. The user (`rare`) and its password (`f01a7031fc17`) are configured in the application.yml file (and can thus be overridden using environment variables for example).
+to the endpoint `/api/harvests`, as described in the API documentation that you can generate using the 
+build task `asciidoctor`, which executes tests and generates documentation based on snippets generated 
+by these tests. The documentation is generated in the folder `backend/build/asciidoc/html5/index.html`/
 
-Example with the `http` command ([HTTPie](https://httpie.org/)):
-
-    http --auth rare:f01a7031fc17 POST http://localhost:8080/api/harvests
-    
-Example with the `curl` command:
-
-    curl -i -X POST -u rare:f01a7031fc17 http://localhost:8080/api/harvests
-    
-The harvest job is executed asynchronously, and a response is immediately sent back, with the URL allowing
-to get the result of the job. For example:
-
-    HTTP/1.1 201 
-    Content-Length: 0
-    Date: Tue, 24 Jul 2018 12:58:04 GMT
-    Location: http://localhost:8080/api/harvests/abb5784d-3006-48fb-b5db-d3ff9583e8b9
-    
-To get the result of the job, you can then send a GET request to the returned URL:
-
-    http --auth rare:f01a7031fc17 GET http://localhost:8080/api/harvests/abb5784d-3006-48fb-b5db-d3ff9583e8b9
-
-or
-
-    curl -u rare:f01a7031fc17 http://localhost:8080/api/harvests/abb5784d-3006-48fb-b5db-d3ff9583e8b9
+    ./gradlew asciidoctor
     
-`http` has the advantage of nicely formetting the returned JSON.
-
-The response contains a detailed report containing the start instant, and the list of files
-that have been processed, with the number of successfully imported resources, and the errors
-that occurred, if any.
-
-It's only when the property `endInstant` of the returned JSON is non-null that the job is complete.
-```
-{
-    "endInstant": "2018-07-24T12:56:28.077Z",
-    "files": [
-        {
-            "errorCount": 0,
-            "errors": [],
-            "fileName": "rare_pilier_microbial.json",
-            "successCount": 10
-        },
-        {
-            "errorCount": 2,
-            "errors": [
-                {
-                    "column": 4,
-                    "error": "Error while parsing object: com.fasterxml.jackson.databind.exc.MismatchedInputException: Cannot deserialize instance of `java.lang.String` out of START_ARRAY token\n at [Source: UNKNOWN; line: -1, column: -1] (through reference chain: fr.inra.urgi.rare.domain.GeneticResource[\"name\"])",
-                    "index": 4790,
-                    "line": 105594
-                },
-                {
-                    "column": 4,
-                    "error": "Error while parsing object: com.fasterxml.jackson.databind.exc.MismatchedInputException: Cannot deserialize instance of `java.lang.String` out of START_ARRAY token\n at [Source: UNKNOWN; line: -1, column: -1] (through reference chain: fr.inra.urgi.rare.domain.GeneticResource[\"countryOfCollect\"])",
-                    "index": 5905,
-                    "line": 130127
-                }
-            ],
-            "fileName": "rare_pilier_plant.json",
-            "successCount": 14522
-        }
-    ],
-    "globalErrors": [],
-    "id": "55e70557-79e8-4e40-a44b-2ef4b3df076a",
-    "startInstant": "2018-07-24T12:56:27.322Z"
-}
-```
-
-In case you lost the response to the POST request and thus don't know what the URL of the harvest is, 
-you can list the harvests, in descending order of their start instant, by sending a GET request to
-`/api/harvests`:
-
-    http --auth rare:f01a7031fc17 GET http://localhost:8080/api/harvests
-
-or
-
-    curl -u rare:f01a7031fc17 http://localhost:8080/api/harvests

backend/build.gradle.kts

+import org.asciidoctor.gradle.AsciidoctorTask
 import org.gradle.api.tasks.testing.logging.TestExceptionFormat
 import org.springframework.boot.gradle.tasks.buildinfo.BuildInfo
 import org.springframework.boot.gradle.tasks.bundling.BootJar
@@ -14,6 +15,7 @@ plugins {
     jacoco
     id("org.springframework.boot") version "2.0.3.RELEASE"
     id("com.gorylenko.gradle-git-properties") version "1.4.21"
+    id("org.asciidoctor.convert") version "1.5.3"
 }
 
 apply(plugin = "io.spring.dependency-management")
@@ -27,6 +29,8 @@ repositories {
     maven("https://repo.spring.io/libs-milestone")
 }
 
+val snippetsDir = file("build/generated-snippets")
+
 tasks {
 
     withType(JavaCompile::class.java) {
@@ -63,6 +67,7 @@ tasks {
         testLogging {
             exceptionFormat = TestExceptionFormat.FULL
         }
+        outputs.dir(snippetsDir)
     }
 
     val jacocoTestReport by getting(JacocoReport::class) {
@@ -71,8 +76,16 @@ tasks {
             html.setEnabled(true)
         }
     }
+
+    val asciidoctor by getting(AsciidoctorTask::class) {
+        inputs.dir(snippetsDir)
+        dependsOn(test)
+        sourceDir("src/main/asciidoc")
+    }
 }
 
+
+
 dependencies {
     implementation("org.springframework.boot:spring-boot-starter-web")
     implementation("org.springframework.boot:spring-boot-starter-actuator")
@@ -90,5 +103,9 @@ dependencies {
     testImplementation("org.junit.jupiter:junit-jupiter-api")
     testImplementation("org.mockito:mockito-junit-jupiter:2.19.1")
     testImplementation("org.junit-pioneer:junit-pioneer:0.1.2")
+    testImplementation("org.springframework.restdocs:spring-restdocs-mockmvc")
+
     testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine")
+
+    asciidoctor("org.springframework.restdocs:spring-restdocs-asciidoctor:2.0.2.RELEASE")
 }

backend/src/main/asciidoc/index.adoc

+= RARe REST services documentation
+:toc: left
+:source-highlighter: highlightjs
+
+== Introduction
+
+This document describes the URLs, parameters, request and response bodies of the various web services of RARe.
+
+The request and response bodies are always JSON. The examples showed here are pretty-printed to make them easier
+to read, but they're not in reality. If you're using the services with the command-line, using HTTPie instead
+of curl allows automatically pretty-printing the returned JSON.
+
+All examples here use HTTP, and show example with `localhost` as the server, and 8080 as port. To use the
+actual deployed web services, you must of course use the actual protocol, host name and port.
+
+Some headers are also removed from the snippets here, to improve readability.
+
+== Harvests
+
+=== Trigger a harvest
+
+Harvesting, i.e. importing resources from JSON files into Elasticsearch, is simply done by copying the JSON
+files to the configured `rare.resource-dir` directory, and then sending a POST request, without any body,
+to trigger the harvest.
+
+Harvested resources which already exist are updated (they are identified by their `identifier` property).
+
+Note that, to avoid letting anyone trigger a harvest, the endpoints are secured using basic authentication. The
+user and the password are configured in the Spring configuration. The following snippets assume the user is
+`rare`, and the password is `f01a7031fc17`.
+
+.Request
+include::{snippets}/harvests/post/http-request.adoc[]
+
+.Curl
+include::{snippets}/harvests/post/curl-request.adoc[]
+
+.HTTPie
+include::{snippets}/harvests/post/httpie-request.adoc[]
+
+The response is immediate: the harvesting job is executed asynchronously. It only contains a Location header
+containing the URL of the endpoint that you can query to know the status of the harvest. Each triggered harvest
+has a unique auto-generated ID, found at the end of the URL.
+
+.Response
+include::{snippets}/harvests/post/http-response.adoc[]
+
+=== Get a harvest
+
+You can get a harvest to know if the harvest is finished or not, and to know which files have already been harvested,
+and which errors occurred during the harvest. The report is pretty detailed, and tries its best to provide indices,
+line and colum numbers as well as error messages allowing to identify what and where the errors are.
+
+Note that files are processed sequentially, and that resources are parsed one by one, using a the Jackson streaming
+parser. This allows harvesting enormous files if needed without fearing any memory problem. It also allows
+parsing a file even if one of its resources has an error and thus can't be parsed correctly.
+
+.Request
+include::{snippets}/harvests/get/http-request.adoc[]
+
+.Path parameters
+include::{snippets}/harvests/get/path-parameters.adoc[]
+
+.Curl
+include::{snippets}/harvests/get/curl-request.adoc[]
+
+.HTTPie
+include::{snippets}/harvests/get/httpie-request.adoc[]
+
+.Response
+include::{snippets}/harvests/get/http-response.adoc[]
+
+.Response fields
+include::{snippets}/harvests/get/response-fields.adoc[]
+
+=== List harvests
+
+If you lost or forgot the URL of the harvest you have triggered, and want to see how it went, don't panic. You can
+send a GET request to list the last 10 harvests that have been triggered.
+
+.Request
+include::{snippets}/harvests/list/http-request.adoc[]
+
+.Curl
+include::{snippets}/harvests/list/curl-request.adoc[]
+
+.HTTPie
+include::{snippets}/harvests/list/httpie-request.adoc[]
+
+.Response
+include::{snippets}/harvests/list/http-response.adoc[]
+
+.Response fields
+include::{snippets}/harvests/list/response-fields.adoc[]
+
+As you can see, you actually get back a page of results. In the unlikely case you want to know about old harvests,
+you can request other pages than the latest one by passing the page as request parameter:
+
+.Request
+include::{snippets}/harvests/list2/http-request.adoc[]
+
+.Request parameters
+include::{snippets}/harvests/list2/request-parameters.adoc[]
+
+.Curl
+include::{snippets}/harvests/list2/curl-request.adoc[]
+
+.HTTPie
+include::{snippets}/harvests/list2/httpie-request.adoc[]
+

backend/src/test/java/fr/inra/urgi/rare/doc/DocumentationConfig.java

+package fr.inra.urgi.rare.doc;
+
+import static org.springframework.restdocs.operation.preprocess.Preprocessors.prettyPrint;
+import static org.springframework.restdocs.operation.preprocess.Preprocessors.removeHeaders;
+
+import org.springframework.boot.test.autoconfigure.restdocs.RestDocsMockMvcConfigurationCustomizer;
+import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.http.HttpHeaders;
+import org.springframework.restdocs.mockmvc.MockMvcRestDocumentationConfigurer;
+
+/**
+ * Additional configuration class allowing to set defaults regarding the REST documentation
+ * @author JB Nizet
+ */
+@TestConfiguration
+public class DocumentationConfig implements RestDocsMockMvcConfigurationCustomizer {
+    @Override
+    public void customize(MockMvcRestDocumentationConfigurer configurer) {
+        configurer.operationPreprocessors()
+                  .withRequestDefaults(prettyPrint())
+                  .withResponseDefaults(prettyPrint(), removeHeaders(HttpHeaders.CONTENT_TYPE,
+                                                                     HttpHeaders.CONTENT_LENGTH));
+    }
+}

backend/src/test/java/fr/inra/urgi/rare/harvest/HarvesterControllerDocTest.java

+package fr.inra.urgi.rare.harvest;
+
+import static org.mockito.Mockito.when;
+import static org.springframework.restdocs.mockmvc.MockMvcRestDocumentation.document;
+import static org.springframework.restdocs.mockmvc.RestDocumentationRequestBuilders.get;
+import static org.springframework.restdocs.mockmvc.RestDocumentationRequestBuilders.post;
+import static org.springframework.restdocs.payload.PayloadDocumentation.fieldWithPath;
+import static org.springframework.restdocs.payload.PayloadDocumentation.responseFields;
+import static org.springframework.restdocs.request.RequestDocumentation.parameterWithName;
+import static org.springframework.restdocs.request.RequestDocumentation.pathParameters;
+import static org.springframework.restdocs.request.RequestDocumentation.requestParameters;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
+
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Optional;
+
+import fr.inra.urgi.rare.dao.HarvestResultDao;
+import fr.inra.urgi.rare.doc.DocumentationConfig;
+import org.hamcrest.CoreMatchers;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.restdocs.AutoConfigureRestDocs;
+import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.context.annotation.Import;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.http.HttpHeaders;
+import org.springframework.restdocs.RestDocumentationExtension;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.test.web.servlet.MockMvc;
+
+/**
+ * REST-Docs tests for {@link HarvesterController}
+ * @author JB Nizet
+ */
+@ExtendWith({SpringExtension.class, RestDocumentationExtension.class})
+@WebMvcTest(controllers = HarvesterController.class, secure = false)
+@AutoConfigureRestDocs
+@Import(DocumentationConfig.class)
+class HarvesterControllerDocTest {
+
+    private static final String USER = "rare";
+    private static final String PASSWORD = "f01a7031fc17";
+
+    @MockBean
+    private AsyncHarvester mockAsyncHarvester;
+
+    @MockBean
+    private HarvestResultDao mockHarvestResultDao;
+
+    @Autowired
+    private MockMvc mockMvc;
+
+    @Test
+    public void shouldHarvest() throws Exception {
+        mockMvc.perform(post("/api/harvests")
+                            .header("Authorization", basicAuth(USER, PASSWORD)))
+               .andExpect(status().isCreated())
+               .andExpect(header().string(HttpHeaders.LOCATION, CoreMatchers.containsString("/api/harvests")))
+               .andDo(document("harvests/post"));
+    }
+
+    @Test
+    void shouldGet() throws Exception {
+        HarvestResult harvestResult =
+            HarvestResult.builder()
+                         .withStartInstant(Instant.now().minus(50, ChronoUnit.SECONDS))
+                         .withFile(HarvestedFile.builder("plant.json")
+                                                .addSuccesses(4321)
+                                                .addError(4790,
+                                                          "Error while parsing object: com.fasterxml.jackson.databind.exc.MismatchedInputException: Cannot deserialize instance of `java.lang.String` out of START_ARRAY token\n at [Source: UNKNOWN; line: -1, column: -1] (through reference chain: fr.inra.urgi.rare.domain.GeneticResource[\"name\"])",
+                                                          105594,
+                                                          4)
+                                                .build())
+                         .withFile(HarvestedFile.builder("microbial.json")
+                                                .addSuccesses(25)
+                                                .build())
+                         .end();
+
+        when(mockHarvestResultDao.findById(harvestResult.getId())).thenReturn(Optional.of(harvestResult));
+
+        mockMvc.perform(get("/api/harvests/{id}", harvestResult.getId())
+                            .header("Authorization", basicAuth(USER, PASSWORD)))
+               .andExpect(status().isOk())
+               .andDo(document("harvests/get",
+                               pathParameters(parameterWithName("id").description("The ID of the harvest")),
+                               responseFields(
+                                   fieldWithPath("id").description("The unique ID of the harvest"),
+                                   fieldWithPath("startInstant").description("The instant when the harvest job started"),
+                                   fieldWithPath("endInstant").description("The instant when the harvest job finished. Null if it's not finished yet"),
+                                   fieldWithPath("globalErrors").description("An array of global errors. Such a global error would exist, for example, for each file that can't be read"),
+                                   fieldWithPath("files").description("An array of files that have been harvested. Files that have not been harvested yet are not listed."),
+                                   fieldWithPath("files[].fileName").description("The name of the harvested file"),
+                                   fieldWithPath("files[].successCount").description("The number of resources in the files that have been harvested successfully"),
+                                   fieldWithPath("files[].errorCount").description("The number of resources in the files that couldn't be harvested due to an error"),
+                                   fieldWithPath("files[].errors").description("The errors that occurred while harvesting the file (one per failed resource)"),
+                                   fieldWithPath("files[].errors[].index").description("The index, starting at 0, of the resource in the JSON file"),
+                                   fieldWithPath("files[].errors[].error").description("The text of the error. The example error here shows that the name property, which is supposed to be a string, couldn't be parsed because it was an array"),
+                                   fieldWithPath("files[].errors[].line").description("The line number, in the JSON file, of the error. It's actually the line of the end of the resource object"),
+                                   fieldWithPath("files[].errors[].column").description("The column number, in the JSON file, of the error. It's actually the column of the end of the resource object")
+                                   )));
+    }
+
+    @Test
+    void shouldList() throws Exception {
+        HarvestResult harvestResult =
+            HarvestResult.builder()
+                         .withStartInstant(Instant.now().minus(50, ChronoUnit.SECONDS))
+                         .end();
+
+        PageRequest pageRequest = PageRequest.of(0, HarvesterController.PAGE_SIZE);
+        when(mockHarvestResultDao.list(pageRequest))
+            .thenReturn(new PageImpl<>(Arrays.asList(harvestResult), pageRequest, 1));
+
+        mockMvc.perform(get("/api/harvests")
+                            .header("Authorization", basicAuth(USER, PASSWORD)))
+               .andExpect(status().isOk())
+               .andDo(document("harvests/list",
+                               responseFields(
+                                   fieldWithPath("number").description("The number of the page, starting at 0"),
+                                   fieldWithPath("size").description("The size of the page"),
+                                   fieldWithPath("totalElements").description("The total number of harvests"),
+                                   fieldWithPath("maxResults").ignored(),
+                                   fieldWithPath("totalPages").description("The total number of pages of harvests"),
+                                   fieldWithPath("content").description("The array of harvests contained in the requested page"),
+                                   fieldWithPath("content[].id").description("The unique ID of the harvest"),
+                                   fieldWithPath("content[].url").description("The URL of the harvest, that you can use to get the details of that harvest"),
+                                   fieldWithPath("content[].startInstant").description("The instant when the harvest job started"),
+                                   fieldWithPath("content[].endInstant").description("The instant when the harvest job finished. Null if it's not finished yet"))));
+    }
+
+    @Test
+    void shouldListSecondPage() throws Exception {
+        HarvestResult harvestResult =
+            HarvestResult.builder()
+                         .withStartInstant(Instant.now().minus(50, ChronoUnit.SECONDS))
+                         .end();
+
+        PageRequest pageRequest = PageRequest.of(1, HarvesterController.PAGE_SIZE);
+        when(mockHarvestResultDao.list(pageRequest))
+            .thenReturn(new PageImpl<>(Arrays.asList(harvestResult), pageRequest, HarvesterController.PAGE_SIZE + 1));
+
+        mockMvc.perform(get("/api/harvests")
+                            .header("Authorization", basicAuth(USER, PASSWORD))
+                            .param("page", "1"))
+               .andExpect(status().isOk())
+               .andDo(document("harvests/list2",
+                               requestParameters(
+                                   parameterWithName("page")
+                                       .optional()
+                                       .description("The requested page number, starting at 0"))));
+    }
+
+    private String basicAuth(String user, String password) {
+        return "Basic " + Base64.getEncoder().encodeToString((user + ":" + password).getBytes(StandardCharsets.UTF_8));
+    }
+}

backend/src/test/java/fr/inra/urgi/rare/harvest/HarvesterControllerTest.java

@@ -70,7 +70,7 @@ class HarvesterControllerTest {
         when(mockHarvestResultDao.list(pageRequest))
             .thenReturn(new PageImpl<>(Arrays.asList(harvestResult), pageRequest, 1));
 
-        mockMvc.perform(get("/api/harvests", harvestResult.getId()))
+        mockMvc.perform(get("/api/harvests"))
                .andExpect(status().isOk())
                .andExpect(jsonPath("$.number").value(0))
                .andExpect(jsonPath("$.content[0].id").value(harvestResult.getId()))