Commit 9f28116c authored by Raphaël Flores's avatar Raphaël Flores Committed by Cyril Pommier
Browse files

Create Docker image for FAIDARE data loader.

parent 6e12e55e
.git/
frontend/node_modules/
backend/out
frontend/.gradle
.gradle/
--- ---
stages: stages:
- pre-build
- test - test
- build - build
- deploy-beta - deploy-beta
...@@ -20,7 +21,9 @@ variables: ...@@ -20,7 +21,9 @@ variables:
APP_NAME: faidare APP_NAME: faidare
JAR_PATH: "backend/build/libs/${APP_NAME}.jar" JAR_PATH: "backend/build/libs/${APP_NAME}.jar"
GIT_DEPTH: 0 GIT_DEPTH: 0
IMAGE_TAG: $CI_COMMIT_REF_SLUG
ELASTIC_VERSION: "6.6.2"
DOCKER_OPTS: "--mtu=1450"
# Gradle cache for all jobs # Gradle cache for all jobs
cache: cache:
...@@ -30,6 +33,31 @@ cache: ...@@ -30,6 +33,31 @@ cache:
- "frontend/.gradle/" - "frontend/.gradle/"
- "frontend/node_modules/" - "frontend/node_modules/"
# PRE-BUILD
build-loader-docker-image:
image: registry.forgemia.inra.fr/urgi-is/docker-rare/docker-git:latest
stage: pre-build
services:
- docker:20.10.6-dind
script:
# build the image
- docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:${IMAGE_TAG} .
- docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest .
# Login before pushing the image
- docker login registry.forgemia.inra.fr -u $CONTAINER_REGISTRY_USERNAME -p $CONTAINER_REGISTRY_TOKEN
# push the built image
- docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:${IMAGE_TAG}
# only push latest tag on master branch
- if [ "master" == "${CI_COMMIT_REF_SLUG}" ] ; then docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest ; fi;
only:
changes:
- scripts/*
- backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json
- backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json
- .gitlab-ci.yml
allow_failure: true
# TESTS # TESTS
...@@ -39,6 +67,13 @@ lint: ...@@ -39,6 +67,13 @@ lint:
tags: tags:
- openstack - openstack
script: "./gradlew lint" script: "./gradlew lint"
cache:
key: "$CI_COMMIT_REF_NAME"
policy: pull
paths:
- ".gradle"
- "frontend/.gradle/"
- "frontend/node_modules/"
test-and-sonarqube: test-and-sonarqube:
...@@ -52,7 +87,7 @@ test-and-sonarqube: ...@@ -52,7 +87,7 @@ test-and-sonarqube:
# because we need to pass some variables, but they are passed to _all_ containers # because we need to pass some variables, but they are passed to _all_ containers
# so they fail the start of other docker images like urgi/docker-browsers # so they fail the start of other docker images like urgi/docker-browsers
# the only solution is to override the entrypoint of the service and pass the arguments manually # the only solution is to override the entrypoint of the service and pass the arguments manually
- name: docker.elastic.co/elasticsearch/elasticsearch:6.5.4 - name: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
alias: elasticsearch alias: elasticsearch
# discovery.type=single-node # discovery.type=single-node
# single-node is necessary to start in development mode # single-node is necessary to start in development mode
...@@ -65,11 +100,15 @@ test-and-sonarqube: ...@@ -65,11 +100,15 @@ test-and-sonarqube:
SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" # Defines the location of the analysis task cache SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" # Defines the location of the analysis task cache
GIT_DEPTH: "0" # Tells git to fetch all the branches of the project, required by the analysis task GIT_DEPTH: "0" # Tells git to fetch all the branches of the project, required by the analysis task
cache: cache:
key: "${CI_JOB_NAME}" key: "${CI_COMMIT_REF_NAME}"
policy: pull-push
paths: paths:
- ".gradle"
- "frontend/.gradle/"
- "frontend/node_modules/"
- .sonar/cache - .sonar/cache
script: script:
- ./gradlew :frontend:test --parallel - ./gradlew :frontend:assemble --parallel
- ./gradlew :backend:test jacocoTestReport --parallel - ./gradlew :backend:test jacocoTestReport --parallel
- find /tmp/node/*/bin -name node -exec ln -s {} /tmp/node/node \; - find /tmp/node/*/bin -name node -exec ln -s {} /tmp/node/node \;
- export PATH="/tmp/node/:$PATH" - export PATH="/tmp/node/:$PATH"
...@@ -78,7 +117,7 @@ test-and-sonarqube: ...@@ -78,7 +117,7 @@ test-and-sonarqube:
reports: reports:
junit: junit:
- ./backend/build/test-results/test/TEST-*.xml - ./backend/build/test-results/test/TEST-*.xml
- ./frontend/karma-junit-tests-report/TEST*.xml # - ./frontend/karma-junit-tests-report/TEST*.xml
only: only:
refs: refs:
- merge_requests - merge_requests
...@@ -100,6 +139,13 @@ build: ...@@ -100,6 +139,13 @@ build:
stage: build stage: build
script: script:
- ./gradlew assemble - ./gradlew assemble
cache:
key: "${CI_COMMIT_REF_NAME}"
policy: pull
paths:
- ".gradle"
- "frontend/.gradle/"
- "frontend/node_modules/"
artifacts: artifacts:
paths: paths:
- "$JAR_PATH" - "$JAR_PATH"
...@@ -134,7 +180,7 @@ restart-config-server-openstack: ...@@ -134,7 +180,7 @@ restart-config-server-openstack:
# DEPLOY # DEPLOY
.deploy-to-vm-proxmox: &deploy_to_vm_proxmox .deploy-to-vm: &deploy_to_vm
# Hidden job which serves as template for executed jobs below. # Hidden job which serves as template for executed jobs below.
# See https://docs.gitlab.com/ee/ci/yaml/#anchors # See https://docs.gitlab.com/ee/ci/yaml/#anchors
retry: 2 retry: 2
...@@ -155,38 +201,16 @@ restart-config-server-openstack: ...@@ -155,38 +201,16 @@ restart-config-server-openstack:
- .gitlab-ci.yml - .gitlab-ci.yml
- backend/src/**/* - backend/src/**/*
- frontend/**/* - frontend/**/*
.deploy-to-vm-openstack: &deploy_to_vm_openstack
# Hidden job which serves as template for executed jobs below.
# See https://docs.gitlab.com/ee/ci/yaml/#anchors
retry: 2
tags:
- openstack
script:
## SSH initialization
- eval $(ssh-agent -s)
- ssh-add <(echo "${SSH_PRIVATE_KEY}")
- ssh -o StrictHostKeyChecking=no ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} 'echo "Successfully connected on $(hostname)"'
# Copy jar
- scp ./backend/build/libs/${APP_NAME}.jar ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK}:/tmp/${APP_NAME}-${ENV}.jar
- ssh ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} "sudo mv /tmp/${APP_NAME}-${ENV}.jar /opt/bootapp/${APP_NAME}-${ENV}.jar ; sudo chown -R bootapp:bootapp /opt/bootapp/"
# Restarting service with the updated jar and the according Spring profiles enabled
- ssh ${SERVER_USER_OPENSTACK}@${SERVER_IP_OPENSTACK} "sudo systemctl restart bootapp@${APP_NAME}-${ENV}"
- eval $(ssh-agent -k)
- echo "Deploy done. Application should be available at http://${SERVER_IP_OPENSTACK}:${APP_PORT}/${CONTEXT_PATH}"
only:
changes:
- .gitlab-ci.yml
- backend/src/**/*
- frontend/**/*
when: manual
allow_failure: false allow_failure: false
deploy-to-beta: deploy-to-beta:
tags:
- openstack
stage: deploy-beta stage: deploy-beta
extends: .deploy-to-vm-openstack extends: .deploy-to-vm
variables: variables:
SERVER_USER: ${SERVER_USER_OPENSTACK}
SERVER_IP: ${SERVER_IP_OPENSTACK_DEV}
APP_PORT: ${BETA_FAIDARE_PORT} APP_PORT: ${BETA_FAIDARE_PORT}
ENV: beta ENV: beta
CONTEXT_PATH: faidare-beta CONTEXT_PATH: faidare-beta
...@@ -199,9 +223,13 @@ deploy-to-beta: ...@@ -199,9 +223,13 @@ deploy-to-beta:
when: always when: always
deploy-to-staging: deploy-to-staging:
tags:
- openstack
stage: deploy-staging stage: deploy-staging
extends: .deploy-to-vm-openstack extends: .deploy-to-vm
variables: variables:
SERVER_USER: ${SERVER_USER_OPENSTACK}
SERVER_IP: ${SERVER_IP_OPENSTACK_DEV}
APP_PORT: ${STAGING_FAIDARE_PORT} APP_PORT: ${STAGING_FAIDARE_PORT}
ENV: staging ENV: staging
CONTEXT_PATH: faidare-staging CONTEXT_PATH: faidare-staging
...@@ -214,9 +242,13 @@ deploy-to-staging: ...@@ -214,9 +242,13 @@ deploy-to-staging:
when: manual when: manual
deploy-to-int: deploy-to-int:
tags:
- proxmox
stage: deploy-production stage: deploy-production
extends: .deploy-to-vm-proxmox extends: .deploy-to-vm
variables: variables:
SERVER_USER: ${SERVER_USER_PROXMOX}
SERVER_IP: ${SERVER_IP_PROXMOX}
APP_PORT: ${INT_FAIDARE_PORT} APP_PORT: ${INT_FAIDARE_PORT}
ENV: int ENV: int
CONTEXT_PATH: faidare-int CONTEXT_PATH: faidare-int
...@@ -226,9 +258,13 @@ deploy-to-int: ...@@ -226,9 +258,13 @@ deploy-to-int:
when: manual when: manual
deploy-to-prod-public: deploy-to-prod-public:
tags:
- proxmox
stage: deploy-production stage: deploy-production
extends: .deploy-to-vm-proxmox extends: .deploy-to-vm
variables: variables:
SERVER_USER: ${SERVER_USER_PROXMOX}
SERVER_IP: ${SERVER_IP_PROXMOX}
APP_PORT: ${PROD_PUBLIC_FAIDARE_PORT} APP_PORT: ${PROD_PUBLIC_FAIDARE_PORT}
ENV: prod-public ENV: prod-public
CONTEXT_PATH: faidare CONTEXT_PATH: faidare
...@@ -238,9 +274,13 @@ deploy-to-prod-public: ...@@ -238,9 +274,13 @@ deploy-to-prod-public:
when: manual when: manual
deploy-to-prod-private: deploy-to-prod-private:
tags:
- proxmox
stage: deploy-production stage: deploy-production
extends: .deploy-to-vm-proxmox extends: .deploy-to-vm
variables: variables:
SERVER_USER: ${SERVER_USER_PROXMOX}
SERVER_IP: ${SERVER_IP_PROXMOX}
APP_PORT: ${PROD_PRIVATE_FAIDARE_PORT} APP_PORT: ${PROD_PRIVATE_FAIDARE_PORT}
ENV: prod-private ENV: prod-private
CONTEXT_PATH: faidare-private CONTEXT_PATH: faidare-private
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
"files": "frontend/package-lock.json|^.secrets.baseline$", "files": "frontend/package-lock.json|^.secrets.baseline$",
"lines": null "lines": null
}, },
"generated_at": "2021-04-16T10:41:53Z", "generated_at": "2021-08-02T15:23:45Z",
"plugins_used": [ "plugins_used": [
{ {
"name": "AWSKeyDetector" "name": "AWSKeyDetector"
...@@ -51,21 +51,21 @@ ...@@ -51,21 +51,21 @@
"hashed_secret": "2907dcd1b70a82032e52be9b6b804abbb4a7525e", "hashed_secret": "2907dcd1b70a82032e52be9b6b804abbb4a7525e",
"is_secret": false, "is_secret": false,
"is_verified": false, "is_verified": false,
"line_number": 83, "line_number": 127,
"type": "Base64 High Entropy String" "type": "Base64 High Entropy String"
}, },
{ {
"hashed_secret": "dd447c7c799dd4ebaacca8f0ad3da45a097d7211", "hashed_secret": "dd447c7c799dd4ebaacca8f0ad3da45a097d7211",
"is_secret": false, "is_secret": false,
"is_verified": false, "is_verified": false,
"line_number": 199, "line_number": 228,
"type": "Base64 High Entropy String" "type": "Base64 High Entropy String"
}, },
{ {
"hashed_secret": "8074db38f8a8acec1a147bc5daf2799ff6693fff", "hashed_secret": "8074db38f8a8acec1a147bc5daf2799ff6693fff",
"is_secret": false, "is_secret": false,
"is_verified": false, "is_verified": false,
"line_number": 214, "line_number": 247,
"type": "Base64 High Entropy String" "type": "Base64 High Entropy String"
} }
], ],
......
FROM alpine
LABEL Author="Raphaël FLORES <raphael.flores@inrae.fr>"
COPY scripts/harvest.sh /opt/scripts/
# COPY dao settings
COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/settings.json
# COPY dao mappings
COPY backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/*_mapping.json /opt/backend/src/test/resources/fr/inra/urgi/faidare/repository/es/setup/index/
RUN apk add --update --no-cache bash curl jq parallel wget grep gzip sed date coreutils
RUN chmod +x /opt/scripts/harvest.sh
RUN mkdir ~/.parallel && touch ~/.parallel/will-cite
ENTRYPOINT ["/opt/scripts/harvest.sh"]
...@@ -83,6 +83,38 @@ cd frontend ...@@ -83,6 +83,38 @@ cd frontend
ng serve ng serve
``` ```
## Harvest
Before all, take care to get data locally before running any indexing script.
### TL;DR
Data indexing to your local Elasticsearch is done using the following command (take care to change the path to local data). Note that your local Elasticsearch instance should be already runing using `docker-compose up`:
```sh
docker run -t --volume /path/to/local/data:/opt/data/ --network=container:elasticsearch-faidare registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest -jsonDir /opt/data/ --help
```
Remove the `--help` parameter to run the loading with default params.
### Portability
#### Docker
[TL;DR](#TLDR) section above expects to have an available docker image on the forgemia docker registry. The Gitlab CI rebuil it when needed, but you can update or push such an image using the following commands:
```sh
# build the image
docker build -t registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest .
# Login before pushing the image
docker login registry.forgemia.inra.fr/urgi-is/docker-rare -u <your ForgeMIA username>
# push the built image
docker push registry.forgemia.inra.fr/urgi-is/docker-rare/faidare-loader:latest
```
That should ease the indexing of data without having to craft a dedicated environment.
## GitLab CI ## GitLab CI
When creating merge requests on the ForgeMIA GitLab, the GitLab CI will When creating merge requests on the ForgeMIA GitLab, the GitLab CI will
......
...@@ -100,8 +100,8 @@ dependencies { ...@@ -100,8 +100,8 @@ dependencies {
implementation("org.springframework.cloud:spring-cloud-starter-config") implementation("org.springframework.cloud:spring-cloud-starter-config")
// Elasticsearch // Elasticsearch
implementation("org.elasticsearch:elasticsearch:6.5.4") implementation("org.elasticsearch:elasticsearch:6.6.2")
implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.5.4") implementation("org.elasticsearch.client:elasticsearch-rest-high-level-client:6.6.2")
// Swagger // Swagger
implementation("io.swagger:swagger-annotations:1.5.21") implementation("io.swagger:swagger-annotations:1.5.21")
......
version: '3.3' version: '3.3'
services: services:
elasticsearch: elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.5.4 image: docker.elastic.co/elasticsearch/elasticsearch:6.6.2
container_name: elasticsearch-faidare container_name: elasticsearch-faidare
environment: environment:
- discovery.type=single-node - discovery.type=single-node
...@@ -9,7 +9,7 @@ services: ...@@ -9,7 +9,7 @@ services:
- 9200:9200 - 9200:9200
kibana: kibana:
image: docker.elastic.co/kibana/kibana:6.5.4 image: docker.elastic.co/kibana/kibana:6.6.2
container_name: kibana-faidare container_name: kibana-faidare
environment: environment:
- "ELASTICSEARCH_URL=http://elasticsearch:9200" - "ELASTICSEARCH_URL=http://elasticsearch:9200"
......
...@@ -44,6 +44,8 @@ module.exports = function (config) { ...@@ -44,6 +44,8 @@ module.exports = function (config) {
flags: ['--no-sandbox'] // required for Gitlab CI flags: ['--no-sandbox'] // required for Gitlab CI
} }
}, },
captureTimeout: 1200000, // increase timeouts: https://github.com/jasmine/jasmine/issues/1413#issuecomment-334247097
browserNoActivityTimeout: 120000,
singleRun: false, singleRun: false,
restartOnFileChange: true restartOnFileChange: true
}); });
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment