From 48a7e6c7e86ceb294fa96537b0f1770d723283f7 Mon Sep 17 00:00:00 2001 From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr> Date: Thu, 10 Mar 2022 17:32:24 +0100 Subject: [PATCH 1/9] =?UTF-8?q?Ajout=20de=20clefs=20=C3=A9trang=C3=A8res?= =?UTF-8?q?=20sur=20references?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../JsonTableRepositoryTemplate.java | 13 +- .../persistence/ReferenceValueRepository.java | 20 ++ .../fr/inra/oresing/rest/OreSiService.java | 5 +- .../inra/oresing/rest/ReferenceImporter.java | 2 +- .../migration/application/V1__init_schema.sql | 19 +- .../inra/oresing/rest/OreSiResourcesTest.java | 328 +++++++++--------- 6 files changed, 212 insertions(+), 175 deletions(-) diff --git a/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java b/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java index fcd7c5902..048b73733 100644 --- a/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java +++ b/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java @@ -11,10 +11,7 @@ import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.jdbc.core.namedparam.SqlParameterSource; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.stream.Stream; abstract class JsonTableRepositoryTemplate<T extends OreSiEntity> implements InitializingBean { @@ -46,8 +43,9 @@ abstract class JsonTableRepositoryTemplate<T extends OreSiEntity> implements Ini return Iterators.partition(stream.iterator(), 50); } - public void storeAll(Stream<T> stream) { + public List<UUID> storeAll(Stream<T> stream) { String query = getUpsertQuery(); + List<UUID> uuids = new LinkedList<>(); partition(stream).forEachRemaining(entities -> { entities.forEach(e -> { if (e.getId() == null) { @@ -55,9 +53,10 @@ abstract class JsonTableRepositoryTemplate<T extends OreSiEntity> implements Ini } }); String json = getJsonRowMapper().toJson(entities); - List<UUID> result = namedParameterJdbcTemplate.queryForList( - query, new MapSqlParameterSource("json", json), UUID.class); + uuids.addAll(namedParameterJdbcTemplate.queryForList( + query, new MapSqlParameterSource("json", json), UUID.class)); }); + return uuids; } protected abstract String getUpsertQuery(); diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java index 982c937bf..d4f750574 100644 --- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java @@ -10,8 +10,10 @@ import fr.inra.oresing.model.ReferenceDatum; import fr.inra.oresing.model.ReferenceValue; import fr.inra.oresing.rest.ApplicationResult; import org.apache.commons.lang3.StringUtils; +import org.postgresql.util.PSQLException; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; @@ -144,4 +146,22 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi return findAllByReferenceType(referenceType).stream() .collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, ReferenceValue::getId)); } + + public void updateConstraintForeignReferences(List<UUID> uuids) { + String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" + + "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" + + "from " + getTable().getSqlIdentifier() + "\n" + + "where id in (:ids)" + + "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;"; + final String ids = uuids.stream() + .map(uuid -> String.format("'%s'::uuid", uuid)) + .collect(Collectors.joining(",")); + try { + List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper()); + } catch (DataIntegrityViolationException e) { + if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){ + throw e; + } + } + } } \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java index 4b74d5f77..b9e96d0a3 100644 --- a/src/main/java/fr/inra/oresing/rest/OreSiService.java +++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java @@ -376,7 +376,8 @@ public class OreSiService { ReferenceImporter referenceImporter = new ReferenceImporter(referenceImporterContext) { @Override void storeAll(Stream<ReferenceValue> stream) { - referenceValueRepository.storeAll(stream); + final List<UUID> uuids = referenceValueRepository.storeAll(stream); + referenceValueRepository.updateConstraintForeignReferences(uuids); } }; referenceImporter.doImport(file, fileId); @@ -1332,4 +1333,4 @@ public class OreSiService { int lineNumber; List<Map.Entry<String, String>> columns; } -} +} \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java index 932caf158..ad9b24f3d 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java @@ -494,4 +494,4 @@ abstract class ReferenceImporter { return streamBeforePreloading; } } -} +} \ No newline at end of file diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql index 47f954f3f..a97afb63f 100644 --- a/src/main/resources/migration/application/V1__init_schema.sql +++ b/src/main/resources/migration/application/V1__init_schema.sql @@ -28,6 +28,13 @@ create table ReferenceValue CONSTRAINT "hierarchicalKey_uniqueness" UNIQUE (application, referenceType, hierarchicalKey) ); +create table Reference_Reference +( + referenceId entityid REFERENCES ReferenceValue(id) ON DELETE CASCADE, + referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT, + CONSTRAINT "Reference_Reference_PK" PRIMARY KEY (referenceId, referencedBy) +); + CREATE INDEX ref_refslinkedto_index ON ReferenceValue USING gin (refsLinkedTo); CREATE INDEX ref_refvalues_index ON ReferenceValue USING gin (refValues); @@ -85,6 +92,14 @@ create table Data dataValues jsonb, binaryFile EntityRef REFERENCES BinaryFile (id) ); + +create table Data_Reference +( + dataId entityid REFERENCES Data(id) ON DELETE CASCADE, + referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT, + CONSTRAINT "Data_Reference_PK" PRIMARY KEY (dataId, referencedBy) +); + CREATE INDEX data_refslinkedto_index ON Data USING gin (refsLinkedTo); CREATE INDEX data_refvalues_index ON Data USING gin (dataValues); @@ -105,15 +120,17 @@ CREATE TABLE OreSiAuthorization GRANT ALL PRIVILEGES ON BinaryFile TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON ReferenceValue TO "superadmin" WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON Reference_Reference TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON Data TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON OreSiAuthorization TO "superadmin" WITH GRANT OPTION; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON BinaryFile TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON ReferenceValue TO public; +GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Reference_Reference TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Data TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON OreSiAuthorization TO public; --ALTER TABLE BinaryFile ENABLE ROW LEVEL SECURITY; --ALTER TABLE ReferenceValue ENABLE ROW LEVEL SECURITY; ALTER TABLE Data - ENABLE ROW LEVEL SECURITY; + ENABLE ROW LEVEL SECURITY; \ No newline at end of file diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java index 05466322b..1dbd5e200 100644 --- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java +++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java @@ -92,6 +92,170 @@ public class OreSiResourcesTest { .andReturn().getResponse().getCookie(AuthHelper.JWT_COOKIE_NAME); } + @Test + public void addApplicationMonsoreWithRepository() throws Exception { + URL resource = getClass().getResource(fixtures.getMonsoreApplicationConfigurationResourceName()); + String oirFilesUUID; + try (InputStream in = Objects.requireNonNull(resource).openStream()) { + MockMultipartFile configuration = new MockMultipartFile("file", "monsore.yaml", "text/plain", in); + //définition de l'application + authenticationService.addUserRightCreateApplication(userId); + + String response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore") + .file(configuration) + .cookie(authCookie)) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.id", IsNull.notNullValue())) + .andReturn().getResponse().getContentAsString(); + + JsonPath.parse(response).read("$.id"); + } + + String response = null; + // Ajout de referentiel + for (Map.Entry<String, String> e : fixtures.getMonsoreReferentielFiles().entrySet()) { + try (InputStream refStream = getClass().getResourceAsStream(e.getValue())) { + MockMultipartFile refFile = new MockMultipartFile("file", e.getValue(), "text/plain", refStream); + + response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/references/{refType}", e.getKey()) + .file(refFile) + .cookie(authCookie)) + .andExpect(status().isCreated()) + .andExpect(jsonPath("$.id", IsNull.notNullValue())) + .andReturn().getResponse().getContentAsString(); + + JsonPath.parse(response).read("$.id"); + } + } + // ajout de data + String projet = "manche"; + String plateforme = "plateforme"; + String site = "oir"; + resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site)); + + // on dépose 3 fois le même fichier sans le publier + try (InputStream refStream = Objects.requireNonNull(resource).openStream()) { + MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream); + + //fileOrUUID.binaryFileDataset/applications/{name}/file/{id} + for (int i = 0; i < 3; i++) { + response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") + .file(refFile) + .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, false)) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andReturn().getResponse().getContentAsString(); + } + log.debug(response); + //on regarde les versions déposées + response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem") + .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site)) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andExpect(jsonPath("$").isArray()) + .andExpect(jsonPath("$", Matchers.hasSize(3))) + .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3))) + .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0))) + .andReturn().getResponse().getContentAsString(); +// log.debug(response); + //récupération de l'identifiant de la dernière version déposée + oirFilesUUID = JsonPath.parse(response).read("$[2].id"); + + // on vérifie l'absence de data + response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem") + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andExpect(jsonPath("$.totalRows").value(-1)) + .andReturn().getResponse().getContentAsString(); + log.debug(response); + + + // on publie le dernier fichier déposé + + response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") + .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, true)) + .cookie(authCookie)) + // .andExpect(status().is2xxSuccessful()) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + + // on récupère la liste des versions déposées + + response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem") + .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site)) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andExpect(jsonPath("$").isArray()) + .andExpect(jsonPath("$", Matchers.hasSize(3))) + .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(2))) + .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(1))) + .andExpect(jsonPath("$[*][?(@.params.published == true )].id").value(oirFilesUUID)) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + + // on récupère le data en base + + response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem") + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andExpect(jsonPath("$.totalRows").value(34)) + .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(34))) + .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'plateforme.oir.oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(34))) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + } + //on publie 4 fichiers + + publishOrDepublish("manche", "plateforme", "scarff", 68, true, 1, true); + publishOrDepublish("atlantique", "plateforme", "scarff", 34, true, 1, true); + publishOrDepublish("atlantique", "plateforme", "nivelle", 34, true, 1, true); + publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 1, true); + //on publie une autre version + String fileUUID = publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 2, true); + // on supprime l'application publiée + response = mockMvc.perform(MockMvcRequestBuilders.delete("/api/v1/applications/monsore/file/" + fileUUID) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + testFilesAndDataOnServer(plateforme, "manche", "nivelle", 0, 1, fileUUID, false); + + + // on depublie le fichier oir déposé + + response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") + .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, false)) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + + // on récupère la liste des versions déposées + + response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem") + .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site)) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andExpect(jsonPath("$").isArray()) + .andExpect(jsonPath("$", Matchers.hasSize(3))) + .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3))) + .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0))) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + + // on récupère le data en base + + response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem") + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andExpect(jsonPath("$.totalRows").value(136)) + .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(136))) + .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(0))) + .andReturn().getResponse().getContentAsString(); + log.debug(StringUtils.abbreviate(response, 50)); + // on supprime le fic + } + @Test public void addApplicationMonsore() throws Exception { String appId; @@ -381,170 +545,6 @@ public class OreSiResourcesTest { // changement du fichier de config avec un mauvais (qui ne permet pas d'importer les fichiers } - @Test - public void addApplicationMonsoreWithRepository() throws Exception { - URL resource = getClass().getResource(fixtures.getMonsoreApplicationConfigurationResourceName()); - String oirFilesUUID; - try (InputStream in = Objects.requireNonNull(resource).openStream()) { - MockMultipartFile configuration = new MockMultipartFile("file", "monsore.yaml", "text/plain", in); - //définition de l'application - authenticationService.addUserRightCreateApplication(userId); - - String response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore") - .file(configuration) - .cookie(authCookie)) - .andExpect(status().isCreated()) - .andExpect(jsonPath("$.id", IsNull.notNullValue())) - .andReturn().getResponse().getContentAsString(); - - JsonPath.parse(response).read("$.id"); - } - - String response = null; - // Ajout de referentiel - for (Map.Entry<String, String> e : fixtures.getMonsoreReferentielFiles().entrySet()) { - try (InputStream refStream = getClass().getResourceAsStream(e.getValue())) { - MockMultipartFile refFile = new MockMultipartFile("file", e.getValue(), "text/plain", refStream); - - response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/references/{refType}", e.getKey()) - .file(refFile) - .cookie(authCookie)) - .andExpect(status().isCreated()) - .andExpect(jsonPath("$.id", IsNull.notNullValue())) - .andReturn().getResponse().getContentAsString(); - - JsonPath.parse(response).read("$.id"); - } - } - // ajout de data - String projet = "manche"; - String plateforme = "plateforme"; - String site = "oir"; - resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site)); - - // on dépose 3 fois le même fichier sans le publier - try (InputStream refStream = Objects.requireNonNull(resource).openStream()) { - MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream); - - //fileOrUUID.binaryFileDataset/applications/{name}/file/{id} - for (int i = 0; i < 3; i++) { - response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") - .file(refFile) - .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, false)) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andReturn().getResponse().getContentAsString(); - } - log.debug(response); - //on regarde les versions déposées - response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem") - .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site)) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andExpect(jsonPath("$").isArray()) - .andExpect(jsonPath("$", Matchers.hasSize(3))) - .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3))) - .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0))) - .andReturn().getResponse().getContentAsString(); -// log.debug(response); - //récupération de l'identifiant de la dernière version déposée - oirFilesUUID = JsonPath.parse(response).read("$[2].id"); - - // on vérifie l'absence de data - response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem") - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andExpect(jsonPath("$.totalRows").value(-1)) - .andReturn().getResponse().getContentAsString(); - log.debug(response); - - - // on publie le dernier fichier déposé - - response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") - .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, true)) - .cookie(authCookie)) - // .andExpect(status().is2xxSuccessful()) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - - // on récupère la liste des versions déposées - - response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem") - .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site)) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andExpect(jsonPath("$").isArray()) - .andExpect(jsonPath("$", Matchers.hasSize(3))) - .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(2))) - .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(1))) - .andExpect(jsonPath("$[*][?(@.params.published == true )].id").value(oirFilesUUID)) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - - // on récupère le data en base - - response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem") - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andExpect(jsonPath("$.totalRows").value(34)) - .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(34))) - .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'plateforme.oir.oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(34))) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - } - //on publie 4 fichiers - - publishOrDepublish("manche", "plateforme", "scarff", 68, true, 1, true); - publishOrDepublish("atlantique", "plateforme", "scarff", 34, true, 1, true); - publishOrDepublish("atlantique", "plateforme", "nivelle", 34, true, 1, true); - publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 1, true); - //on publie une autre version - String fileUUID = publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 2, true); - // on supprime l'application publiée - response = mockMvc.perform(MockMvcRequestBuilders.delete("/api/v1/applications/monsore/file/" + fileUUID) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - testFilesAndDataOnServer(plateforme, "manche", "nivelle", 0, 1, fileUUID, false); - - - // on depublie le fichier oir déposé - - response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") - .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, false)) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - - // on récupère la liste des versions déposées - - response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem") - .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site)) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andExpect(jsonPath("$").isArray()) - .andExpect(jsonPath("$", Matchers.hasSize(3))) - .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3))) - .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0))) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - - // on récupère le data en base - - response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem") - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andExpect(jsonPath("$.totalRows").value(136)) - .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(136))) - .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(0))) - .andReturn().getResponse().getContentAsString(); - log.debug(StringUtils.abbreviate(response, 50)); - // on supprime le fic - } - private String publishOrDepublish(String projet, String plateforme, String site, int expected, boolean toPublish, int numberOfVersions, boolean published) throws Exception { URL resource; String response; -- GitLab From b74e043f1c85a82694e9621874692432c2938b55 Mon Sep 17 00:00:00 2001 From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr> Date: Fri, 11 Mar 2022 10:24:52 +0100 Subject: [PATCH 2/9] =?UTF-8?q?M=C3=AAme=20chose=20pour=20les=20data=20et?= =?UTF-8?q?=20suppression=20des=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oresing/persistence/DataRepository.java | 24 ++++++ .../fr/inra/oresing/rest/OreSiService.java | 4 +- .../migration/application/V1__init_schema.sql | 7 +- .../migration/main/V1__init_schema.sql | 84 ------------------- .../inra/oresing/rest/OreSiResourcesTest.java | 48 +++++------ 5 files changed, 55 insertions(+), 112 deletions(-) diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java index 386f1d4a7..a89299444 100644 --- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java @@ -5,14 +5,17 @@ import com.google.common.collect.ImmutableMap; import fr.inra.oresing.model.Application; import fr.inra.oresing.model.Data; import fr.inra.oresing.rest.DownloadDatasetQuery; +import org.postgresql.util.PSQLException; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; +import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.stereotype.Component; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.stream.Collectors; @Component @Scope(scopeName = ConfigurableBeanFactory.SCOPE_PROTOTYPE) @@ -82,4 +85,25 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla int count = getNamedParameterJdbcTemplate().update(sql, sqlParams); return count; } + + public void updateConstraintForeigData(List<UUID> uuids) { + String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)\n" + + "with tuple as (\n" + + "select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy\n" + + "from " + getTable().getSqlIdentifier() + "\n" + + ")\n" + + "select dataId, referencedBy from tuple\n" + + "where dataId in (:ids) and referencedBy is not null\n" + + "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING;"; + final String ids = uuids.stream() + .map(uuid -> String.format("'%s'::uuid", uuid)) + .collect(Collectors.joining(",")); + try { + List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper()); + } catch (DataIntegrityViolationException e) { + if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){ + throw e; + } + } + } } \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java index b9e96d0a3..943aaeea4 100644 --- a/src/main/java/fr/inra/oresing/rest/OreSiService.java +++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java @@ -511,7 +511,9 @@ public class OreSiService { .map(buildReplaceMissingValuesByDefaultValuesFn(app, dataType, binaryFileDataset == null ? null : binaryFileDataset.getRequiredauthorizations())) .flatMap(buildLineValuesToEntityStreamFn(app, dataType, storedFile.getId(), errors, binaryFileDataset)); - repo.getRepository(app).data().storeAll(dataStream); + final DataRepository dataRepository = repo.getRepository(app).data(); + final List<UUID> uuids = dataRepository.storeAll(dataStream); + dataRepository.updateConstraintForeigData(uuids); } } diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql index a97afb63f..0a03eec4f 100644 --- a/src/main/resources/migration/application/V1__init_schema.sql +++ b/src/main/resources/migration/application/V1__init_schema.sql @@ -22,7 +22,7 @@ create table ReferenceValue hierarchicalKey ltree NOT NULL, hierarchicalReference ltree NOT NULL, naturalKey ltree NOT NULL, - refsLinkedTo jsonb check (refs_check_for_reference('${applicationSchema}', application, refsLinkedTo)), + refsLinkedTo jsonb , refValues jsonb, binaryFile EntityRef REFERENCES BinaryFile (id), @@ -87,8 +87,7 @@ create table Data rowId TEXT NOT NULL, datagroup TEXT GENERATED ALWAYS AS (("authorization").datagroup[1]) STORED NOT NULL, "authorization" ${applicationSchema}.authorization NOT NULL check (("authorization").datagroup[1] is not null), - refsLinkedTo jsonb check (refs_check_for_datatype('${applicationSchema}', application, refsLinkedTo, - datatype)), + refsLinkedTo jsonb , dataValues jsonb, binaryFile EntityRef REFERENCES BinaryFile (id) ); @@ -122,12 +121,14 @@ GRANT ALL PRIVILEGES ON BinaryFile TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON ReferenceValue TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON Reference_Reference TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON Data TO "superadmin" WITH GRANT OPTION; +GRANT ALL PRIVILEGES ON Data_Reference TO "superadmin" WITH GRANT OPTION; GRANT ALL PRIVILEGES ON OreSiAuthorization TO "superadmin" WITH GRANT OPTION; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON BinaryFile TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON ReferenceValue TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Reference_Reference TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Data TO public; +GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Data_Reference TO public; GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON OreSiAuthorization TO public; --ALTER TABLE BinaryFile ENABLE ROW LEVEL SECURITY; diff --git a/src/main/resources/migration/main/V1__init_schema.sql b/src/main/resources/migration/main/V1__init_schema.sql index a7126613f..0fc93b18c 100644 --- a/src/main/resources/migration/main/V1__init_schema.sql +++ b/src/main/resources/migration/main/V1__init_schema.sql @@ -39,79 +39,6 @@ BEGIN END; $$ language 'plpgsql';*/ --- check les foreign key pour le colonne references de la table data - -CREATE OR REPLACE FUNCTION refs_check_for_datatype(aschema text, application UUID, refValues jsonb, dtype TEXT) - RETURNS BOOLEAN AS $$ -DECLARE - result TEXT; -BEGIN - EXECUTE 'with agg as ( - SELECT application."configuration" - ->''datatypes'' - ->$4 - ->''data'' - ->jsonb_object_keys( - application."configuration" - ->''datatypes'' - ->$4 - ->''data'' - ) - ->''components'' - ->jsonb_object_keys( - application."configuration"->''datatypes'' - ->$4 - ->''data'' - ->jsonb_object_keys( - application."configuration" - ->''datatypes'' - ->$4 - ->''data'' - ) - ->''components'') - ->''checker'' - ->''params'' - ->>''refType'' reftype, - $3 - ->jsonb_object_keys( - application."configuration" - ->''datatypes'' - ->$4 - ->''data'' - ) - ->jsonb_object_keys( - application."configuration" - ->''datatypes'' - ->$4 - ->''data'' - ->jsonb_object_keys( - application."configuration" - ->''datatypes'' - ->$4 - ->''data'' - ) - ->''components'' - ) reference - FROM application - where application.id = $2), - byref as ( - select jsonb_build_object(reftype::TEXT, array_agg(distinct reference)) byref - from agg - where reftype is not null and reference is not null - group by reftype), - refvalues as ( - select jsonb_object_agg(byref) refvalues - from byref - group by $2) - SELECT count(id) = jsonb_count_items(refvalues.refvalues) - from refvalues, ' || aSchema || '.referencevalue - where application=$2::uuid and jsonb_build_object(referenceType, ARRAY[id]) <@ refvalues.refvalues - group by refvalues.refvalues;' - INTO result USING aschema, application, refValues, dtype; - return result; -END; -$$ LANGUAGE plpgsql; - --check if all elements of oreSiUser array are users CREATE OR REPLACE FUNCTION checks_users(users uuid[]) RETURNS BOOLEAN AS $$ @@ -123,18 +50,7 @@ BEGIN END; $$ LANGUAGE plpgsql; --- check les foreign key pour le colonne references de la table data -CREATE OR REPLACE FUNCTION refs_check_for_reference(aSchema text, application UUID, refValues jsonb) -RETURNS BOOLEAN AS $$ -DECLARE - result TEXT; -BEGIN - EXECUTE 'select count(id) = jsonb_count_items($2) from ' || aSchema || '.referencevalue where application=$1::uuid and jsonb_build_object(referenceType, ARRAY[id]) <@ $2 ' || - '' INTO result USING application, refValues; - RETURN result; -END; -$$ language 'plpgsql'; CREATE OR REPLACE FUNCTION name_check(application UUID, targetColumn TEXT, val TEXT) RETURNS BOOLEAN AS $$ diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java index 1dbd5e200..ca59fbc36 100644 --- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java +++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java @@ -256,6 +256,30 @@ public class OreSiResourcesTest { // on supprime le fic } + private String publishOrDepublish(String projet, String plateforme, String site, int expected, boolean toPublish, int numberOfVersions, boolean published) throws Exception { + URL resource; + String response; + resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site)); + try (InputStream refStream = Objects.requireNonNull(resource).openStream()) { + + //dépôt et publication d'un fichier projet site__p1 + MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream); + refFile.transferTo(Path.of("/tmp/pem.csv")); + response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") + .file(refFile) + .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, toPublish)) + .cookie(authCookie)) + .andExpect(status().is2xxSuccessful()) + .andReturn().getResponse().getContentAsString(); + String fileUUID = JsonPath.parse(response).read("$.fileId"); + + //liste des fichiers projet/site + testFilesAndDataOnServer(plateforme, projet, site, expected, numberOfVersions, fileUUID, published); + log.debug(StringUtils.abbreviate(response, 50)); + return fileUUID; + } + } + @Test public void addApplicationMonsore() throws Exception { String appId; @@ -545,30 +569,6 @@ public class OreSiResourcesTest { // changement du fichier de config avec un mauvais (qui ne permet pas d'importer les fichiers } - private String publishOrDepublish(String projet, String plateforme, String site, int expected, boolean toPublish, int numberOfVersions, boolean published) throws Exception { - URL resource; - String response; - resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site)); - try (InputStream refStream = Objects.requireNonNull(resource).openStream()) { - - //dépôt et publication d'un fichier projet site__p1 - MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream); - refFile.transferTo(Path.of("/tmp/pem.csv")); - response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem") - .file(refFile) - .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, toPublish)) - .cookie(authCookie)) - .andExpect(status().is2xxSuccessful()) - .andReturn().getResponse().getContentAsString(); - String fileUUID = JsonPath.parse(response).read("$.fileId"); - - //liste des fichiers projet/site - testFilesAndDataOnServer(plateforme, projet, site, expected, numberOfVersions, fileUUID, published); - log.debug(StringUtils.abbreviate(response, 50)); - return fileUUID; - } - } - @Test public void testRecursivity() throws Exception { -- GitLab From 7455aa4328f5fd33bea9b984e80a5fac0c0e6361 Mon Sep 17 00:00:00 2001 From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr> Date: Fri, 11 Mar 2022 10:54:29 +0100 Subject: [PATCH 3/9] =?UTF-8?q?D=C3=A9coupage=20by=20batch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit -le nombre de paramètres que l'on peut passer est limité (max et min 32767) --- .../oresing/persistence/DataRepository.java | 24 ++-- .../persistence/ReferenceValueRepository.java | 110 +++++++----------- 2 files changed, 54 insertions(+), 80 deletions(-) diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java index a89299444..f39676eef 100644 --- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java @@ -2,6 +2,7 @@ package fr.inra.oresing.persistence; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterators; import fr.inra.oresing.model.Application; import fr.inra.oresing.model.Data; import fr.inra.oresing.rest.DownloadDatasetQuery; @@ -95,15 +96,18 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla "select dataId, referencedBy from tuple\n" + "where dataId in (:ids) and referencedBy is not null\n" + "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING;"; - final String ids = uuids.stream() - .map(uuid -> String.format("'%s'::uuid", uuid)) - .collect(Collectors.joining(",")); - try { - List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper()); - } catch (DataIntegrityViolationException e) { - if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){ - throw e; - } - } + Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE-1) + .forEachRemaining(uuidsByBatch -> { + final String ids = uuidsByBatch.stream() + .map(uuid -> String.format("'%s'::uuid", uuid)) + .collect(Collectors.joining(",")); + try { + getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper()); + } catch (DataIntegrityViolationException e) { + if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) { + throw e; + } + } + }); } } \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java index d4f750574..56fb6c68d 100644 --- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java @@ -2,12 +2,8 @@ package fr.inra.oresing.persistence; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import fr.inra.oresing.model.Application; -import fr.inra.oresing.model.ReferenceColumn; -import fr.inra.oresing.model.ReferenceColumnSingleValue; -import fr.inra.oresing.model.ReferenceColumnValue; -import fr.inra.oresing.model.ReferenceDatum; -import fr.inra.oresing.model.ReferenceValue; +import com.google.common.collect.Iterators; +import fr.inra.oresing.model.*; import fr.inra.oresing.rest.ApplicationResult; import org.apache.commons.lang3.StringUtils; import org.postgresql.util.PSQLException; @@ -42,14 +38,7 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi @Override protected String getUpsertQuery() { - return "INSERT INTO " + getTable().getSqlIdentifier() + "\n" + - "(id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile) \n" + - "SELECT id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile \n" + - "FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", \n" + - ":json::json) \n" - + " ON CONFLICT ON CONSTRAINT \"hierarchicalKey_uniqueness\" \n" + - "DO UPDATE SET updateDate=current_timestamp, hierarchicalKey=EXCLUDED.hierarchicalKey, hierarchicalReference=EXCLUDED.hierarchicalReference, naturalKey=EXCLUDED.naturalKey, refsLinkedTo=EXCLUDED.refsLinkedTo, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile" - + " RETURNING id"; + return "INSERT INTO " + getTable().getSqlIdentifier() + "\n" + "(id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile) \n" + "SELECT id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile \n" + "FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", \n" + ":json::json) \n" + " ON CONFLICT ON CONSTRAINT \"hierarchicalKey_uniqueness\" \n" + "DO UPDATE SET updateDate=current_timestamp, hierarchicalKey=EXCLUDED.hierarchicalKey, hierarchicalReference=EXCLUDED.hierarchicalReference, naturalKey=EXCLUDED.naturalKey, refsLinkedTo=EXCLUDED.refsLinkedTo, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile" + " RETURNING id"; } @Override @@ -68,10 +57,8 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi */ public List<ReferenceValue> findAllByReferenceType(String refType, MultiValueMap<String, String> params) { MultiValueMap<String, String> toto = new LinkedMultiValueMap<>(); - String query = "SELECT DISTINCT '" + ReferenceValue.class.getName() + "' as \"@class\", to_jsonb(t) as json FROM " - + getTable().getSqlIdentifier() + " t, jsonb_each_text(t.refvalues) kv WHERE application=:applicationId::uuid AND referenceType=:refType"; - MapSqlParameterSource paramSource = new MapSqlParameterSource("applicationId", getApplication().getId()) - .addValue("refType", refType); + String query = "SELECT DISTINCT '" + ReferenceValue.class.getName() + "' as \"@class\", to_jsonb(t) as json FROM " + getTable().getSqlIdentifier() + " t, jsonb_each_text(t.refvalues) kv WHERE application=:applicationId::uuid AND referenceType=:refType"; + MapSqlParameterSource paramSource = new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType); AtomicInteger i = new AtomicInteger(); // kv.value='LPF' OR t.refvalues @> '{"esp_nom":"ALO"}'::jsonb @@ -79,13 +66,12 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi String k = e.getKey(); if (StringUtils.equalsAnyIgnoreCase("_row_id_", k)) { String collect = e.getValue().stream().map(v -> { - String arg = ":arg" + i.getAndIncrement(); - paramSource.addValue(arg, v); - return String.format("'%s'::uuid", v); - }) - .collect(Collectors.joining(", ")); + String arg = ":arg" + i.getAndIncrement(); + paramSource.addValue(arg, v); + return String.format("'%s'::uuid", v); + }).collect(Collectors.joining(", ")); return Stream.ofNullable(String.format("array[id]::uuid[] <@ array[%s]::uuid[]", collect)); - }else if (StringUtils.equalsAnyIgnoreCase("any", k)) { + } else if (StringUtils.equalsAnyIgnoreCase("any", k)) { return e.getValue().stream().map(v -> { String arg = ":arg" + i.getAndIncrement(); paramSource.addValue(arg, v); @@ -94,9 +80,7 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi } else { return e.getValue().stream().map(v -> "t.refvalues @> '{\"" + k + "\":\"" + v + "\"}'::jsonb"); } - }) - .filter(k->k!=null). - collect(Collectors.joining(" OR ")); + }).filter(k -> k != null).collect(Collectors.joining(" OR ")); if (StringUtils.isNotBlank(cond)) { cond = " AND (" + cond + ")"; @@ -108,60 +92,46 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi public List<List<String>> findReferenceValue(String refType, String column) { AtomicInteger ai = new AtomicInteger(0); - String select = Stream.of(column.split(",")) - .map(c -> String.format("refValues->>'%1$s' as \"%1$s"+ai.getAndIncrement()+"\"", c)) - .collect(Collectors.joining(", ")); - String sqlPattern = " SELECT %s " - + " FROM " + getTable().getSqlIdentifier() + " t" - + " WHERE application=:applicationId::uuid AND referenceType=:refType"; + String select = Stream.of(column.split(",")).map(c -> String.format("refValues->>'%1$s' as \"%1$s" + ai.getAndIncrement() + "\"", c)).collect(Collectors.joining(", ")); + String sqlPattern = " SELECT %s " + " FROM " + getTable().getSqlIdentifier() + " t" + " WHERE application=:applicationId::uuid AND referenceType=:refType"; String query = String.format(sqlPattern, select); - List<List<String>> result = getNamedParameterJdbcTemplate().queryForList(query, new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType)) - .stream() - .map(m -> m.values().stream().map(v -> (String) v).collect(Collectors.toList())) - .collect(Collectors.toList()); + List<List<String>> result = getNamedParameterJdbcTemplate().queryForList(query, new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType)).stream().map(m -> m.values().stream().map(v -> (String) v).collect(Collectors.toList())).collect(Collectors.toList()); return result; } public ImmutableMap<Ltree, ApplicationResult.Reference.ReferenceUUIDAndDisplay> getReferenceIdAndDisplayPerKeys(String referenceType, String locale) { - Function<ReferenceValue, ApplicationResult.Reference.ReferenceUUIDAndDisplay> referenceValueToReferenceUuidAndDisplayFunction = - result -> { - ReferenceDatum referenceDatum = result.getRefValues(); - ReferenceColumn referenceColumnForDisplay = ReferenceColumn.forDisplay(locale); - String display; - if (referenceDatum.contains(referenceColumnForDisplay)) { - ReferenceColumnValue referenceColumnValueForDisplay = referenceDatum.get(referenceColumnForDisplay); - Preconditions.checkState(referenceColumnValueForDisplay instanceof ReferenceColumnSingleValue); - display = ((ReferenceColumnSingleValue) referenceColumnValueForDisplay).getValue(); - } else { - display = null; - } - Map<String, String> values = referenceDatum.toJsonForFrontend(); - return new ApplicationResult.Reference.ReferenceUUIDAndDisplay(display, result.getId(), values); - }; - return findAllByReferenceType(referenceType).stream() - .collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, referenceValueToReferenceUuidAndDisplayFunction)); + Function<ReferenceValue, ApplicationResult.Reference.ReferenceUUIDAndDisplay> referenceValueToReferenceUuidAndDisplayFunction = result -> { + ReferenceDatum referenceDatum = result.getRefValues(); + ReferenceColumn referenceColumnForDisplay = ReferenceColumn.forDisplay(locale); + String display; + if (referenceDatum.contains(referenceColumnForDisplay)) { + ReferenceColumnValue referenceColumnValueForDisplay = referenceDatum.get(referenceColumnForDisplay); + Preconditions.checkState(referenceColumnValueForDisplay instanceof ReferenceColumnSingleValue); + display = ((ReferenceColumnSingleValue) referenceColumnValueForDisplay).getValue(); + } else { + display = null; + } + Map<String, String> values = referenceDatum.toJsonForFrontend(); + return new ApplicationResult.Reference.ReferenceUUIDAndDisplay(display, result.getId(), values); + }; + return findAllByReferenceType(referenceType).stream().collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, referenceValueToReferenceUuidAndDisplayFunction)); } public ImmutableMap<Ltree, UUID> getReferenceIdPerKeys(String referenceType) { - return findAllByReferenceType(referenceType).stream() - .collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, ReferenceValue::getId)); + return findAllByReferenceType(referenceType).stream().collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, ReferenceValue::getId)); } public void updateConstraintForeignReferences(List<UUID> uuids) { - String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" + - "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" + - "from " + getTable().getSqlIdentifier() + "\n" + - "where id in (:ids)" + - "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;"; - final String ids = uuids.stream() - .map(uuid -> String.format("'%s'::uuid", uuid)) - .collect(Collectors.joining(",")); - try { - List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper()); - } catch (DataIntegrityViolationException e) { - if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){ - throw e; + String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" + "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" + "from " + getTable().getSqlIdentifier() + "\n" + "where id in (:ids)" + "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;"; + Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1).forEachRemaining(uuidsByBatch -> { + final String ids = uuidsByBatch.stream().map(uuid -> String.format("'%s'::uuid", uuid)).collect(Collectors.joining(",")); + try { + getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper()); + } catch (DataIntegrityViolationException e) { + if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) { + throw e; + } } - } + }); } } \ No newline at end of file -- GitLab From 8abc4e2a0afff64bc09a81f912fe689f863fb36d Mon Sep 17 00:00:00 2001 From: Brendan Le Ny <bleny@codelutin.com> Date: Fri, 11 Mar 2022 18:08:20 +0100 Subject: [PATCH 4/9] =?UTF-8?q?=C3=89vite=20un=20chargement=20du=20r=C3=A9?= =?UTF-8?q?f=C3=A9rentiel=20avant=20sa=20mise=20=C3=A0=20jour=20(semble=20?= =?UTF-8?q?inutile)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/fr/inra/oresing/rest/OreSiService.java | 2 -- .../java/fr/inra/oresing/rest/ReferenceImporter.java | 7 ------- .../fr/inra/oresing/rest/ReferenceImporterContext.java | 9 --------- 3 files changed, 18 deletions(-) diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java index 4b8745d43..2e60490c4 100644 --- a/src/main/java/fr/inra/oresing/rest/OreSiService.java +++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java @@ -387,7 +387,6 @@ public class OreSiService { ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue(); Configuration conf = app.getConfiguration(); ImmutableSet<LineChecker> lineCheckers = checkerFactory.getReferenceValidationLineCheckers(app, refType); - final ImmutableMap<Ltree, UUID> storedReferences = referenceValueRepository.getReferenceIdPerKeys(refType); ImmutableMap<ReferenceColumn, Multiplicity> multiplicityPerColumns = lineCheckers.stream() .filter(lineChecker -> lineChecker instanceof ReferenceLineChecker) @@ -451,7 +450,6 @@ public class OreSiService { conf, refType, lineCheckers, - storedReferences, columns ); return referenceImporterContext; diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java index 4b3433c46..4cfa936c4 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java @@ -292,13 +292,6 @@ abstract class ReferenceImporter { final Ltree hierarchicalReference = recursionStrategy.getHierarchicalReference(naturalKey); referenceDatum.putAll(InternationalizationDisplay.getDisplays(referenceImporterContext.getDisplayPattern(), referenceImporterContext.getDisplayColumns(), referenceDatum)); - /** - * on remplace l'id par celle en base si elle existe - * a noter que pour les references récursives on récupère l'id depuis referenceLineChecker.getReferenceValues() ce qui revient au même - */ - - referenceImporterContext.getIdForSameHierarchicalKeyInDatabase(hierarchicalKey) - .ifPresent(e::setId); e.setBinaryFile(fileId); e.setReferenceType(referenceImporterContext.getRefType()); e.setHierarchicalKey(hierarchicalKey); diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java index ae0d71e64..57e858ca6 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java @@ -63,11 +63,6 @@ public class ReferenceImporterContext { */ private final ImmutableSet<LineChecker> lineCheckers; - /** - * Les clés techniques de chaque clé naturelle hiérarchique de toutes les lignes existantes en base (avant l'import) - */ - private final ImmutableMap<Ltree, UUID> storedReferences; - private final ImmutableMap<String, Column> columnsPerHeader; private Optional<InternationalizationReferenceMap> getInternationalizationReferenceMap() { @@ -196,10 +191,6 @@ public class ReferenceImporterContext { return applicationId; } - public Optional<UUID> getIdForSameHierarchicalKeyInDatabase(Ltree hierarchicalKey) { - return Optional.ofNullable(storedReferences.get(hierarchicalKey)); - } - public void pushValue(ReferenceDatum referenceDatum, String header, String cellContent, SetMultimap<String, UUID> refsLinkedTo) { Column column = columnsPerHeader.get(header); column.pushValue(cellContent, referenceDatum, refsLinkedTo); -- GitLab From de7df312a29d330ff7b20942e16e8766465edd5c Mon Sep 17 00:00:00 2001 From: Brendan Le Ny <bleny@codelutin.com> Date: Mon, 14 Mar 2022 17:50:01 +0100 Subject: [PATCH 5/9] =?UTF-8?q?=C3=89vite=20une=20exception=20due=20=C3=A0?= =?UTF-8?q?=20une=20mauvaise=20utilisation=20de=20la=20persistance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../oresing/persistence/DataRepository.java | 35 +++++++------------ .../persistence/ReferenceValueRepository.java | 30 ++++++++-------- 2 files changed, 28 insertions(+), 37 deletions(-) diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java index f39676eef..bebbcd378 100644 --- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java @@ -6,17 +6,15 @@ import com.google.common.collect.Iterators; import fr.inra.oresing.model.Application; import fr.inra.oresing.model.Data; import fr.inra.oresing.rest.DownloadDatasetQuery; -import org.postgresql.util.PSQLException; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.stereotype.Component; +import java.sql.PreparedStatement; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.stream.Collectors; @Component @Scope(scopeName = ConfigurableBeanFactory.SCOPE_PROTOTYPE) @@ -88,26 +86,17 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla } public void updateConstraintForeigData(List<UUID> uuids) { - String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)\n" + - "with tuple as (\n" + - "select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy\n" + - "from " + getTable().getSqlIdentifier() + "\n" + - ")\n" + - "select dataId, referencedBy from tuple\n" + - "where dataId in (:ids) and referencedBy is not null\n" + - "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING;"; + String sql = String.join(" " + , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)" + , "with tuple as (" + , " select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy" + , " from " + getTable().getSqlIdentifier() + "" + , ")" + , "select dataId, referencedBy from tuple" + , "where dataId in (:ids) and referencedBy is not null" + , "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING" + ); Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE-1) - .forEachRemaining(uuidsByBatch -> { - final String ids = uuidsByBatch.stream() - .map(uuid -> String.format("'%s'::uuid", uuid)) - .collect(Collectors.joining(",")); - try { - getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper()); - } catch (DataIntegrityViolationException e) { - if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) { - throw e; - } - } - }); + .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute)); } } \ No newline at end of file diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java index 56fb6c68d..d704df063 100644 --- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java @@ -3,18 +3,22 @@ package fr.inra.oresing.persistence; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; -import fr.inra.oresing.model.*; +import fr.inra.oresing.model.Application; +import fr.inra.oresing.model.ReferenceColumn; +import fr.inra.oresing.model.ReferenceColumnSingleValue; +import fr.inra.oresing.model.ReferenceColumnValue; +import fr.inra.oresing.model.ReferenceDatum; +import fr.inra.oresing.model.ReferenceValue; import fr.inra.oresing.rest.ApplicationResult; import org.apache.commons.lang3.StringUtils; -import org.postgresql.util.PSQLException; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; -import org.springframework.dao.DataIntegrityViolationException; import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import java.sql.PreparedStatement; import java.util.List; import java.util.Map; import java.util.UUID; @@ -122,16 +126,14 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi } public void updateConstraintForeignReferences(List<UUID> uuids) { - String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" + "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" + "from " + getTable().getSqlIdentifier() + "\n" + "where id in (:ids)" + "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;"; - Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1).forEachRemaining(uuidsByBatch -> { - final String ids = uuidsByBatch.stream().map(uuid -> String.format("'%s'::uuid", uuid)).collect(Collectors.joining(",")); - try { - getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper()); - } catch (DataIntegrityViolationException e) { - if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) { - throw e; - } - } - }); + String sql = String.join(" " + , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)" + , "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy" + , "from " + getTable().getSqlIdentifier() + , "where id in (:ids)" + , "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING" + ); + Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1) + .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute)); } } \ No newline at end of file -- GitLab From 95113af899655363972c853cfb27522bb82e39ec Mon Sep 17 00:00:00 2001 From: Brendan Le Ny <bleny@codelutin.com> Date: Wed, 16 Mar 2022 16:54:26 +0100 Subject: [PATCH 6/9] =?UTF-8?q?Met=20=C3=A0=20jour=20les=20tables=20associ?= =?UTF-8?q?ations=20en=20cas=20de=20suppression=20d'un=20lien=20dans=20le?= =?UTF-8?q?=20refsLinkedTo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/fr/inra/oresing/persistence/DataRepository.java | 4 +++- .../fr/inra/oresing/persistence/ReferenceValueRepository.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java index bebbcd378..b155f8581 100644 --- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java @@ -86,7 +86,8 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla } public void updateConstraintForeigData(List<UUID> uuids) { - String sql = String.join(" " + String deleteSql = "DELETE FROM " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference WHERE dataId in (:ids)"; + String insertSql = String.join(" " , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)" , "with tuple as (" , " select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy" @@ -96,6 +97,7 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla , "where dataId in (:ids) and referencedBy is not null" , "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING" ); + String sql = String.join(";", deleteSql, insertSql); Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE-1) .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute)); } diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java index d704df063..3d8f5a865 100644 --- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java +++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java @@ -126,13 +126,15 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi } public void updateConstraintForeignReferences(List<UUID> uuids) { - String sql = String.join(" " + String deleteSql = "DELETE FROM " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference WHERE referenceId in (:ids)"; + String insertSql = String.join(" " , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)" , "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy" , "from " + getTable().getSqlIdentifier() , "where id in (:ids)" , "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING" ); + String sql = String.join(";", insertSql, deleteSql); Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1) .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute)); } -- GitLab From b47ee3b55b204705480f122f13a6afdc40440ba1 Mon Sep 17 00:00:00 2001 From: Brendan Le Ny <bleny@codelutin.com> Date: Wed, 16 Mar 2022 17:55:13 +0100 Subject: [PATCH 7/9] =?UTF-8?q?Permet=20la=20mise=20=C3=A0=20jour=20des=20?= =?UTF-8?q?r=C3=A9f=C3=A9rentiel=20sans=20reprendre=20la=20cl=C3=A9=20tech?= =?UTF-8?q?nique?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../inra/oresing/rest/ReferenceImporter.java | 18 ------------------ .../migration/application/V1__init_schema.sql | 4 ++-- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java index f7b10e4c9..a48ecc22f 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java @@ -51,7 +51,6 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -287,8 +286,6 @@ abstract class ReferenceImporter { final ReferenceValue e = new ReferenceValue(); final Ltree naturalKey = keysAndReferenceDatumAfterChecking.getNaturalKey(); - recursionStrategy.getKnownId(naturalKey) - .ifPresent(e::setId); final Ltree hierarchicalReference = recursionStrategy.getHierarchicalReference(naturalKey); referenceDatum.putAll(InternationalizationDisplay.getDisplays(referenceImporterContext.getDisplayPattern(), referenceImporterContext.getDisplayColumns(), referenceDatum)); @@ -363,8 +360,6 @@ abstract class ReferenceImporter { Ltree getHierarchicalReference(Ltree naturalKey); - Optional<UUID> getKnownId(Ltree naturalKey); - Stream<RowWithReferenceDatum> firstPass(Stream<RowWithReferenceDatum> streamBeforePreloading); } @@ -375,14 +370,6 @@ abstract class ReferenceImporter { private final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>(); - @Override - public Optional<UUID> getKnownId(Ltree naturalKey) { - if (afterPreloadReferenceUuids.containsKey(naturalKey)) { - return Optional.of(afterPreloadReferenceUuids.get(naturalKey)); - } - return Optional.empty(); - } - @Override public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) { Ltree recursiveNaturalKey = getRecursiveNaturalKey(naturalKey); @@ -465,11 +452,6 @@ abstract class ReferenceImporter { private class WithoutRecursion implements RecursionStrategy { - @Override - public Optional<UUID> getKnownId(Ltree naturalKey) { - return Optional.empty(); - } - @Override public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) { return referenceImporterContext.newHierarchicalKey(naturalKey, referenceDatum); diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql index 0a03eec4f..2e0debf6f 100644 --- a/src/main/resources/migration/application/V1__init_schema.sql +++ b/src/main/resources/migration/application/V1__init_schema.sql @@ -31,7 +31,7 @@ create table ReferenceValue create table Reference_Reference ( referenceId entityid REFERENCES ReferenceValue(id) ON DELETE CASCADE, - referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT, + referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED, CONSTRAINT "Reference_Reference_PK" PRIMARY KEY (referenceId, referencedBy) ); @@ -95,7 +95,7 @@ create table Data create table Data_Reference ( dataId entityid REFERENCES Data(id) ON DELETE CASCADE, - referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT, + referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED, CONSTRAINT "Data_Reference_PK" PRIMARY KEY (dataId, referencedBy) ); -- GitLab From 0b89a96dfad803edaa670703167fd4e7b21d5545 Mon Sep 17 00:00:00 2001 From: Brendan Le Ny <bleny@codelutin.com> Date: Wed, 16 Mar 2022 17:58:54 +0100 Subject: [PATCH 8/9] Convertit un champs en variable locale --- .../java/fr/inra/oresing/rest/ReferenceImporter.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java index a48ecc22f..5c2fd88d5 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java @@ -368,8 +368,6 @@ abstract class ReferenceImporter { private final Map<Ltree, Ltree> parentReferenceMap = new LinkedHashMap<>(); - private final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>(); - @Override public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) { Ltree recursiveNaturalKey = getRecursiveNaturalKey(naturalKey); @@ -404,7 +402,7 @@ abstract class ReferenceImporter { final ReferenceColumn columnToLookForParentKey = referenceImporterContext.getColumnToLookForParentKey(); ReferenceLineChecker referenceLineChecker = referenceImporterContext.getReferenceLineChecker(); final ImmutableMap<Ltree, UUID> beforePreloadReferenceUuids = referenceLineChecker.getReferenceValues(); - afterPreloadReferenceUuids.putAll(beforePreloadReferenceUuids); + final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>(beforePreloadReferenceUuids); ListMultimap<Ltree, Integer> missingParentReferences = LinkedListMultimap.create(); List<RowWithReferenceDatum> collect = streamBeforePreloading .peek(rowWithReferenceDatum -> { @@ -426,7 +424,8 @@ abstract class ReferenceImporter { missingParentReferences.removeAll(naturalKey); }) .collect(Collectors.toList()); - checkMissingParentReferencesIsEmpty(missingParentReferences); + Set<Ltree> knownReferences = afterPreloadReferenceUuids.keySet(); + checkMissingParentReferencesIsEmpty(missingParentReferences, knownReferences); referenceLineChecker.setReferenceValues(ImmutableMap.copyOf(afterPreloadReferenceUuids)); return collect.stream(); } @@ -436,13 +435,13 @@ abstract class ReferenceImporter { * * @param missingParentReferences pour chaque parent manquant, les lignes du CSV où il est mentionné */ - private void checkMissingParentReferencesIsEmpty(ListMultimap<Ltree, Integer> missingParentReferences) { + private void checkMissingParentReferencesIsEmpty(ListMultimap<Ltree, Integer> missingParentReferences, Set<Ltree> knownReferences) { List<CsvRowValidationCheckResult> rowErrors = missingParentReferences.entries().stream() .map(entry -> { Ltree missingParentReference = entry.getKey(); Integer lineNumber = entry.getValue(); ValidationCheckResult validationCheckResult = - new MissingParentLineValidationCheckResult(lineNumber, referenceImporterContext.getRefType(), missingParentReference, afterPreloadReferenceUuids.keySet()); + new MissingParentLineValidationCheckResult(lineNumber, referenceImporterContext.getRefType(), missingParentReference, knownReferences); return new CsvRowValidationCheckResult(validationCheckResult, lineNumber); }) .collect(Collectors.toUnmodifiableList()); -- GitLab From 0e288b627436c038483486ee2bd13690fdae5632 Mon Sep 17 00:00:00 2001 From: Brendan Le Ny <bleny@codelutin.com> Date: Wed, 16 Mar 2022 18:27:53 +0100 Subject: [PATCH 9/9] =?UTF-8?q?Gr=C3=A2ce=20au=20contr=C3=B4l=C3=A9=20defe?= =?UTF-8?q?rable=20il=20n'est=20plus=20n=C3=A9cessaire=20de=20faire=20l'in?= =?UTF-8?q?sertion=20des=20r=C3=A9f=C3=A9rentiels=20dans=20l'ordre?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/fr/inra/oresing/rest/ReferenceImporter.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java index 5c2fd88d5..8ea34b7ef 100644 --- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java +++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java @@ -45,7 +45,6 @@ import java.io.IOException; import java.io.InputStream; import java.time.format.DateTimeFormatter; import java.util.Collection; -import java.util.Comparator; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -116,8 +115,7 @@ abstract class ReferenceImporter { boolean canSave = encounteredHierarchicalKeysForConflictDetection.get(hierarchicalKey).size() == 1; return canSave; }) - .map(keysAndReferenceDatumAfterChecking -> toEntity(keysAndReferenceDatumAfterChecking, fileId)) - .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql())); + .map(keysAndReferenceDatumAfterChecking -> toEntity(keysAndReferenceDatumAfterChecking, fileId)); storeAll(referenceValuesStream); } -- GitLab