Create Docker image for FAIDARE data loader.

.dockerignore

+.git/
+frontend/node_modules/
+backend/out
+frontend/.gradle
+.gradle/

.gitlab-ci.yml

 ---
 stages:
+  - pre-build
   - test
   - build
   - deploy-beta
@@ -20,7 +21,9 @@ variables:
   APP_NAME: faidare
   JAR_PATH: "backend/build/libs/${APP_NAME}.jar"
   GIT_DEPTH: 0
-
+  IMAGE_TAG: $CI_COMMIT_REF_SLUG
+  ELASTIC_VERSION: "6.6.2"
+  DOCKER_OPTS: "--mtu=1450"
 
 # Gradle cache for all jobs
 cache:
@@ -30,6 +33,31 @@ cache:
     - "frontend/.gradle/"
     - "frontend/node_modules/"
 
+# PRE-BUILD
+
+build-loader-docker-image:
+ image: registry.forgemia.inra.fr/urgi-is/docker-rare/docker-git:latest
+ stage: pre-build
+ services:
+  - docker:20.10.6-dind
+ script:
+  # build the image
+  - docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:${IMAGE_TAG} .
+  - docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest .
+  # Login before pushing the image
+  - docker login registry.forgemia.inra.fr -u $CONTAINER_REGISTRY_USERNAME -p $CONTAINER_REGISTRY_TOKEN
+  # push the built image
+  - docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:${IMAGE_TAG}
+  # only push latest tag on master branch
+  - if [ "master" == "${CI_COMMIT_REF_SLUG}" ] ; then docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest ; fi;
+ only:
+  changes:
+   - scripts/*
+   - backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json
+   - backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json
+   - .gitlab-ci.yml
+ allow_failure: true
+
 
 # TESTS
 
@@ -39,6 +67,13 @@ lint:
   tags:
    - openstack
   script: "./gradlew lint"
+  cache:
+    key: "$CI_COMMIT_REF_NAME"
+    policy: pull
+    paths:
+      - ".gradle"
+      - "frontend/.gradle/"
+      - "frontend/node_modules/"
 
 
 test-and-sonarqube:
@@ -52,7 +87,7 @@ test-and-sonarqube:
     # because we need to pass some variables, but they are passed to _all_ containers
     # so they fail the start of other docker images like urgi/docker-browsers
     # the only solution is to override the entrypoint of the service and pass the arguments manually
-    - name: docker.elastic.co/elasticsearch/elasticsearch:6.5.4
+    - name: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
       alias: elasticsearch
       # discovery.type=single-node
       # single-node is necessary to start in development mode
@@ -65,11 +100,15 @@ test-and-sonarqube:
     SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache
     GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task
   cache:
-    key: "${CI_JOB_NAME}"
+    key: "${CI_COMMIT_REF_NAME}"
+    policy: pull-push
     paths:
+      - ".gradle"
+      - "frontend/.gradle/"
+      - "frontend/node_modules/"
       - .sonar/cache
   script:
-    - ./gradlew :frontend:test --parallel
+    - ./gradlew :frontend:assemble --parallel
     - ./gradlew :backend:test jacocoTestReport --parallel
     - find /tmp/node/*/bin -name node -exec ln -s {} /tmp/node/node \;
     - export PATH="/tmp/node/:$PATH"
@@ -78,7 +117,7 @@ test-and-sonarqube:
     reports:
       junit:
         - ./backend/build/test-results/test/TEST-*.xml
-        - ./frontend/karma-junit-tests-report/TEST*.xml
+        # - ./frontend/karma-junit-tests-report/TEST*.xml
   only:
     refs:
       - merge_requests
@@ -100,6 +139,13 @@ build:
   stage: build
   script:
     - ./gradlew assemble
+  cache:
+    key: "${CI_COMMIT_REF_NAME}"
+    policy: pull
+    paths:
+      - ".gradle"
+      - "frontend/.gradle/"
+      - "frontend/node_modules/"
   artifacts:
     paths:
       - "$JAR_PATH"
@@ -134,7 +180,7 @@ restart-config-server-openstack:
 # DEPLOY
 
 
-.deploy-to-vm-proxmox: &deploy_to_vm_proxmox
+.deploy-to-vm: &deploy_to_vm
   # Hidden job which serves as template for executed jobs below.
   # See https://docs.gitlab.com/ee/ci/yaml/#anchors
   retry: 2
@@ -155,38 +201,16 @@ restart-config-server-openstack:
       - .gitlab-ci.yml
       - backend/src/**/*
       - frontend/**/*
-
-
-.deploy-to-vm-openstack: &deploy_to_vm_openstack
-  # Hidden job which serves as template for executed jobs below.
-  # See https://docs.gitlab.com/ee/ci/yaml/#anchors
-  retry: 2
-  tags:
-   - openstack
-  script:
-    ## SSH initialization
-    - eval $(ssh-agent -s)
-    - ssh-add <(echo "${SSH_PRIVATE_KEY}")
-    - ssh -o StrictHostKeyChecking=no ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} 'echo "Successfully connected on $(hostname)"'
-    # Copy jar
-    - scp ./backend/build/libs/${APP_NAME}.jar ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK}:/tmp/${APP_NAME}-${ENV}.jar
-    - ssh ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} "sudo mv /tmp/${APP_NAME}-${ENV}.jar /opt/bootapp/${APP_NAME}-${ENV}.jar ; sudo chown -R bootapp:bootapp /opt/bootapp/"
-    # Restarting service with the updated jar and the according Spring profiles enabled
-    - ssh ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} "sudo systemctl restart bootapp@${APP_NAME}-${ENV}"
-    - eval $(ssh-agent -k)
-    - echo "Deploy done. Application should be available at http://${SERVER_IP_OPENSTACK}:${APP_PORT}/${CONTEXT_PATH}"
-  only:
-    changes:
-      - .gitlab-ci.yml
-      - backend/src/**/*
-      - frontend/**/*
-  when: manual
   allow_failure: false
 
 deploy-to-beta:
+  tags:
+   - openstack
   stage: deploy-beta
-  extends: .deploy-to-vm-openstack
+  extends: .deploy-to-vm
   variables:
+    SERVER_USER: ${SERVER_USER_OPENSTACK}
+    SERVER_IP: ${SERVER_IP_OPENSTACK_DEV}
     APP_PORT: ${BETA_FAIDARE_PORT}
     ENV: beta
     CONTEXT_PATH: faidare-beta
@@ -199,9 +223,13 @@ deploy-to-beta:
   when: always
 
 deploy-to-staging:
+  tags:
+   - openstack
   stage: deploy-staging
-  extends: .deploy-to-vm-openstack
+  extends: .deploy-to-vm
   variables:
+    SERVER_USER: ${SERVER_USER_OPENSTACK}
+    SERVER_IP: ${SERVER_IP_OPENSTACK_DEV}
     APP_PORT: ${STAGING_FAIDARE_PORT}
     ENV: staging
     CONTEXT_PATH: faidare-staging
@@ -214,9 +242,13 @@ deploy-to-staging:
   when: manual
 
 deploy-to-int:
+  tags:
+   - proxmox
   stage: deploy-production
-  extends: .deploy-to-vm-proxmox
+  extends: .deploy-to-vm
   variables:
+    SERVER_USER: ${SERVER_USER_PROXMOX}
+    SERVER_IP: ${SERVER_IP_PROXMOX}
     APP_PORT: ${INT_FAIDARE_PORT}
     ENV: int
     CONTEXT_PATH: faidare-int
@@ -226,9 +258,13 @@ deploy-to-int:
   when: manual
 
 deploy-to-prod-public:
+  tags:
+   - proxmox
   stage: deploy-production
-  extends: .deploy-to-vm-proxmox
+  extends: .deploy-to-vm
   variables:
+    SERVER_USER: ${SERVER_USER_PROXMOX}
+    SERVER_IP: ${SERVER_IP_PROXMOX}
     APP_PORT: ${PROD_PUBLIC_FAIDARE_PORT}
     ENV: prod-public
     CONTEXT_PATH: faidare
@@ -238,9 +274,13 @@ deploy-to-prod-public:
   when: manual
 
 deploy-to-prod-private:
+  tags:
+   - proxmox
   stage: deploy-production
-  extends: .deploy-to-vm-proxmox
+  extends: .deploy-to-vm
   variables:
+    SERVER_USER: ${SERVER_USER_PROXMOX}
+    SERVER_IP: ${SERVER_IP_PROXMOX}
     APP_PORT: ${PROD_PRIVATE_FAIDARE_PORT}
     ENV: prod-private
     CONTEXT_PATH: faidare-private

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "frontend/package-lock.json|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2021-04-16T10:41:53Z",
+  "generated_at": "2021-08-02T15:23:45Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -51,21 +51,21 @@
         "hashed_secret": "2907dcd1b70a82032e52be9b6b804abbb4a7525e",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 83,
+        "line_number": 127,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "dd447c7c799dd4ebaacca8f0ad3da45a097d7211",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 199,
+        "line_number": 228,
         "type": "Base64 High Entropy String"
       },
       {
         "hashed_secret": "8074db38f8a8acec1a147bc5daf2799ff6693fff",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 214,
+        "line_number": 247,
         "type": "Base64 High Entropy String"
       }
     ],

Dockerfile

+FROM alpine
+LABEL Author="Raphaël FLORES <raphael.flores@inrae.fr>"
+
+COPY scripts/harvest.sh /opt/scripts/
+
+# COPY dao settings
+COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json
+
+# COPY dao mappings
+COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/
+
+RUN apk add --update --no-cache bash curl jq parallel wget grep gzip sed date coreutils
+
+RUN chmod +x /opt/scripts/harvest.sh
+RUN mkdir ~/.parallel && touch ~/.parallel/will-cite
+
+ENTRYPOINT ["/opt/scripts/harvest.sh"]

README.md

@@ -83,6 +83,38 @@ cd frontend
 ng serve
 ```
 
+## Harvest
+
+Before all, take care to get data locally before running any indexing script.
+
+### TL;DR
+
+Data indexing to your local Elasticsearch is done using the following command (take care to change the path to local data). Note that your local Elasticsearch instance should be already runing using `docker-compose up`:
+
+```sh
+docker run -t --volume /path/to/local/data:/opt/data/ --network=container:elasticsearch-faidare registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest -jsonDir /opt/data/ --help
+```
+Remove the `--help` parameter to run the loading with default params.
+
+### Portability
+
+#### Docker
+
+[TL;DR](#TLDR) section above expects to have an available docker image on the forgemia docker registry. The Gitlab CI rebuil it when needed, but you can update or push such an image using the following commands:
+
+```sh
+# build the image
+docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest .
+
+# Login before pushing the image
+docker login registry.forgemia.inra.fr/urgi-is/docker-rare -u <your ForgeMIA username>
+
+# push the built image
+docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest
+```
+
+That should ease the indexing of data without having to craft a dedicated environment.
+
 ## GitLab CI
 
 When creating merge requests on the ForgeMIA GitLab, the GitLab CI will 

backend/build.gradle.kts

@@ -100,8 +100,8 @@ dependencies {
     implementation("org.springframework.cloud:spring-cloud-starter-config")
 
     // Elasticsearch
-    implementation("org.elasticsearch:elasticsearch:6.5.4")
-    implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.5.4")
+    implementation("org.elasticsearch:elasticsearch:6.6.2")
+    implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.6.2")
 
     // Swagger
     implementation("io.swagger:swagger-annotations:1.5.21")

docker-compose.yml

 version: '3.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.2
     container_name: elasticsearch-faidare
     environment:
       - discovery.type=single-node
@@ -9,7 +9,7 @@ services:
       - 9200:9200
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:6.5.4
+    image: docker.elastic.co/kibana/kibana:6.6.2
     container_name: kibana-faidare
     environment:
       - "ELASTICSEARCH_URL=http://elasticsearch:9200"

frontend/src/karma.conf.js

@@ -44,6 +44,8 @@ module.exports = function (config) {
                 flags: ['--no-sandbox'] // required for Gitlab CI
             }
         },
+        captureTimeout: 1200000, // increase timeouts: https://github.com/jasmine/jasmine/issues/1413#issuecomment-334247097
+        browserNoActivityTimeout: 120000,
         singleRun: false,
         restartOnFileChange: true
     });