Commit b4857b0a authored by Raphaël Flores's avatar Raphaël Flores Committed by Célia Michotey
Browse files

Fix code review. GNP-5494.

parent 06f79016
......@@ -42,14 +42,6 @@ test:
- /bin/sh
- -c
- "export PATH=/usr/share/elasticsearch/bin:$PATH ;
elasticsearch-plugin list | grep ingest-datadiscovery-description-tokenizer
|| elasticsearch-plugin install --batch \
https://urgi.versailles.inra.fr/nexus/content/repositories/releases/fr/inra/plugin/ingest/ingest-datadiscovery-description-tokenizer/${ELASTIC_VERSION}.0/ingest-datadiscovery-description-tokenizer-${ELASTIC_VERSION}.0.zip \
|| {
echo '[ERROR] Cannot install Elasticsearch plugin from Nexus repository. Need to install it, more info: \
https://forgemia.inra.fr/urgi-is/es-plugins/blob/master/ingest-datadiscovery-description-tokenizer/README.md' ;
exit 1 ;
} ;
exec chroot --userspec=1000 / elasticsearch -Ediscovery.type=single-node -Ecluster.name=elasticsearch"
script: ./gradlew --parallel test
cache:
......
......@@ -100,7 +100,7 @@ by these tests. The documentation is generated in the folder `backend/build/asci
Harvesting (i.e. importing documents stored in JSON files into Elasticsearch) consists in
creating the necessary index and aliases and Elasticsearch templates.
To create the index and its aliases execute the script with appropriate
To create the index and its aliases execute the script below for local dev environment:
./scripts/createIndexAndAliases.sh
......@@ -114,7 +114,7 @@ You can run the scripts:
./scripts/harvestRare.sh
./scripts/harvestWheatis.sh
to trigger a harvest of the resources stored in the Git LFS directories `data/rare` and `data/wheatis`.
to trigger a harvest of the resources stored in the Git LFS directories `data/rare` and `data/wheatis` respectively.
## Indices and aliases
......
......@@ -48,7 +48,7 @@ public final class SuggestionDocument implements fr.inra.urgi.datadiscovery.doma
}
public int hashCode() {
return Objects.hash(//id,
return Objects.hash(
suggestions);
}
......
......@@ -12,7 +12,6 @@
},
"name": {
"type": "text",
"copy_to": "suggestions",
"analyzer": "french",
"fields": {
"keyword": {
......@@ -27,7 +26,6 @@
},
"pillarName": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -37,7 +35,6 @@
},
"databaseSource": {
"type": "text",
"copy_to": "suggestions",
"analyzer": "french",
"fields": {
"keyword": {
......@@ -62,7 +59,6 @@
},
"domain": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -72,7 +68,6 @@
},
"taxon": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -82,7 +77,6 @@
},
"family": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -92,7 +86,6 @@
},
"genus": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -102,7 +95,6 @@
},
"species": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -112,7 +104,6 @@
},
"materialType": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -123,7 +114,6 @@
},
"biotopeType": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -134,7 +124,6 @@
},
"countryOfOrigin": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"type": "keyword",
......@@ -148,9 +137,8 @@
},
"countryOfCollect": {
"type": "text",
"copy_to": "suggestions",
"fields": {
"keyword": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
"null_value": "NULL"
......@@ -159,10 +147,6 @@
},
"locationOfCollect": {
"type": "geo_point"
},
"suggestions": {
"type" : "completion",
"analyzer": "custom_suggestion_analyzer"
}
}
}
......@@ -17,38 +17,4 @@ public abstract class DocumentDaoTest {
@Autowired
protected ElasticsearchRestTemplate elasticsearchTemplate;
protected void installDatadiscoveryDescriptionTokenizerPipeline() {
String source =
"{" +
"\"description\":" +
"\"[TEST] Convert a description field to make it usable by an auto-completion query\"," +
"\"processors\":" +
"[" +
"{" +
" \"datadiscovery_description_tokenizer\":" +
"{" +
"\"field\":\"description\"," +
"\"target_field\":\"suggestions\"" +
"}" +
"}" +
"]" +
"}";
PutPipelineRequest request = new PutPipelineRequest(
"datadiscovery-description-tokenizer-pipeline",
new BytesArray(source.getBytes(StandardCharsets.UTF_8)),
XContentType.JSON
);
try {
AcknowledgedResponse response = elasticsearchTemplate.getClient().ingest().putPipeline(request, RequestOptions.DEFAULT);
if (!response.isAcknowledged()) {
throw new RuntimeException("Couldn't create 'datadiscovery-description-tokenizer-pipeline pipeline':\n\t " + request.getSource());
}
} catch (ElasticsearchStatusException | IOException e) {
throw new RuntimeException("Couldn't create pipeline 'datadiscovery-description-tokenizer-pipeline'. \n\t\t" +
"Make sure you have correctly installed the 'ingest-datadiscovery-description-tokenizer' plugin" +
" in your Elasticsearch cluster. \n\t\tMore info: " +
"https://forgemia.inra.fr/urgi-is/es-plugins/blob/master/ingest-datadiscovery-description-tokenizer/README.md"
, e);
}
}
}
......@@ -56,7 +56,6 @@ class RareDocumentDaoTest extends DocumentDaoTest {
@BeforeAll
void prepareIndex() {
installDatadiscoveryDescriptionTokenizerPipeline();
ElasticsearchPersistentEntity documentEntity = elasticsearchTemplate.getPersistentEntityFor(
RareDocument.class);
ElasticsearchPersistentEntity suggestionDocumentEntity = elasticsearchTemplate.getPersistentEntityFor(
......@@ -216,9 +215,6 @@ class RareDocumentDaoTest extends DocumentDaoTest {
assertThat(documentDao.suggest("hel")).containsOnly("Hello");
assertThat(documentDao.suggest("wor")).containsOnly("world");
assertThat(documentDao.suggest("hel")).containsOnly("Hello");
assertThat(documentDao.suggest("wor")).containsOnly("world");
}
@Test
......
......@@ -51,7 +51,6 @@ class WheatisDocumentDaoTest extends DocumentDaoTest {
@BeforeAll
void prepareIndex() {
installDatadiscoveryDescriptionTokenizerPipeline();
ElasticsearchPersistentEntity documentEntity = elasticsearchTemplate.getPersistentEntityFor(
WheatisDocument.class);
ElasticsearchPersistentEntity suggestionDocumentEntity = elasticsearchTemplate.getPersistentEntityFor(
......
......@@ -6,11 +6,6 @@ services:
- /bin/sh
- -c
- "export PATH=/usr/share/elasticsearch/bin:$PATH ;
elasticsearch-plugin list | grep ingest-datadiscovery-description-tokenizer ||
elasticsearch-plugin install --batch \
https://urgi.versailles.inra.fr/nexus/content/repositories/releases/fr/inra/plugin/ingest/ingest-datadiscovery-description-tokenizer/${ELASTIC_VERSION}.0/ingest-datadiscovery-description-tokenizer-${ELASTIC_VERSION}.0.zip \
|| { echo '[ERROR] Cannot install Elasticsearch plugin from Nexus repository. Need to install it, more info: \
https://forgemia.inra.fr/urgi-is/es-plugins/blob/master/ingest-datadiscovery-description-tokenizer/README.md' ; exit 1 ; } ;
/usr/local/bin/docker-entrypoint.sh"
container_name: elasticsearch
environment:
......
......@@ -28,7 +28,7 @@ DESCRIPTION:
-> last step is the generation of bulk files
-> in parallel, but on the fly, the data is splitted in chunks and each data line sees inserted before it a bulk
action-metadata line
-> then, the output is redirected to a compressed (to keep or not?) file
-> then, the output is redirected to compressed files
Script used to create index and aliases for Data Discovery portals (RARe, WheatIS and DataDiscovery)
USAGE:
......
......@@ -9,7 +9,7 @@ DESCRIPTION:
Script used to index data in Data Discovery portals (RARe, WheatIS and DataDiscovery)
USAGE:
$0 -host <elasticsearch_host> -port <elasticsearch_port> -app <application name> -env <environment name> -copy [-h|--help]
$0 -host <elasticsearch_host> -port <elasticsearch_port> -app <application name> -env <environment name> [-h|--help]
PARAMS:
-host the hostname or IP of Elasticsearch node (default: $ES_HOST)
......
......@@ -17,7 +17,7 @@ def to_bulk(identifier):
else
("WARNING: Following document has been dropped from bulk because it misses the given key identifier ("
+ identifier + "): " + (.| tostring)) | stderr # print warning into stderr
| empty # drop the current malformed object from bulk
| empty # drop the current malformed object from bulk
end
;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment