From 48a7e6c7e86ceb294fa96537b0f1770d723283f7 Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Thu, 10 Mar 2022 17:32:24 +0100
Subject: [PATCH 1/9] =?UTF-8?q?Ajout=20de=20clefs=20=C3=A9trang=C3=A8res?=
 =?UTF-8?q?=20sur=20references?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../JsonTableRepositoryTemplate.java          |  13 +-
 .../persistence/ReferenceValueRepository.java |  20 ++
 .../fr/inra/oresing/rest/OreSiService.java    |   5 +-
 .../inra/oresing/rest/ReferenceImporter.java  |   2 +-
 .../migration/application/V1__init_schema.sql |  19 +-
 .../inra/oresing/rest/OreSiResourcesTest.java | 328 +++++++++---------
 6 files changed, 212 insertions(+), 175 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java b/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java
index fcd7c5902..048b73733 100644
--- a/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java
+++ b/src/main/java/fr/inra/oresing/persistence/JsonTableRepositoryTemplate.java
@@ -11,10 +11,7 @@ import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
 import org.springframework.jdbc.core.namedparam.SqlParameterSource;
 
-import java.util.List;
-import java.util.NoSuchElementException;
-import java.util.Optional;
-import java.util.UUID;
+import java.util.*;
 import java.util.stream.Stream;
 
 abstract class JsonTableRepositoryTemplate<T extends OreSiEntity> implements InitializingBean {
@@ -46,8 +43,9 @@ abstract class JsonTableRepositoryTemplate<T extends OreSiEntity> implements Ini
         return Iterators.partition(stream.iterator(), 50);
     }
 
-    public void storeAll(Stream<T> stream) {
+    public List<UUID> storeAll(Stream<T> stream) {
         String query = getUpsertQuery();
+        List<UUID> uuids = new LinkedList<>();
         partition(stream).forEachRemaining(entities -> {
             entities.forEach(e -> {
                 if (e.getId() == null) {
@@ -55,9 +53,10 @@ abstract class JsonTableRepositoryTemplate<T extends OreSiEntity> implements Ini
                 }
             });
             String json = getJsonRowMapper().toJson(entities);
-            List<UUID> result = namedParameterJdbcTemplate.queryForList(
-                    query, new MapSqlParameterSource("json", json), UUID.class);
+            uuids.addAll(namedParameterJdbcTemplate.queryForList(
+                    query, new MapSqlParameterSource("json", json), UUID.class));
         });
+        return uuids;
     }
 
     protected abstract String getUpsertQuery();
diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
index 982c937bf..d4f750574 100644
--- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
@@ -10,8 +10,10 @@ import fr.inra.oresing.model.ReferenceDatum;
 import fr.inra.oresing.model.ReferenceValue;
 import fr.inra.oresing.rest.ApplicationResult;
 import org.apache.commons.lang3.StringUtils;
+import org.postgresql.util.PSQLException;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Scope;
+import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Component;
 import org.springframework.util.LinkedMultiValueMap;
@@ -144,4 +146,22 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
         return findAllByReferenceType(referenceType).stream()
                 .collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, ReferenceValue::getId));
     }
+
+    public void updateConstraintForeignReferences(List<UUID> uuids) {
+        String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" +
+                "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" +
+                "from  " + getTable().getSqlIdentifier() + "\n" +
+                "where id in (:ids)" +
+                "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;";
+        final String ids = uuids.stream()
+                .map(uuid -> String.format("'%s'::uuid", uuid))
+                .collect(Collectors.joining(","));
+        try {
+            List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper());
+        } catch (DataIntegrityViolationException e) {
+            if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){
+                throw e;
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 4b74d5f77..b9e96d0a3 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -376,7 +376,8 @@ public class OreSiService {
         ReferenceImporter referenceImporter = new ReferenceImporter(referenceImporterContext) {
             @Override
             void storeAll(Stream<ReferenceValue> stream) {
-                referenceValueRepository.storeAll(stream);
+                final List<UUID> uuids = referenceValueRepository.storeAll(stream);
+                referenceValueRepository.updateConstraintForeignReferences(uuids);
             }
         };
         referenceImporter.doImport(file, fileId);
@@ -1332,4 +1333,4 @@ public class OreSiService {
         int lineNumber;
         List<Map.Entry<String, String>> columns;
     }
-}
+}
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
index 932caf158..ad9b24f3d 100644
--- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
+++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
@@ -494,4 +494,4 @@ abstract class ReferenceImporter {
             return streamBeforePreloading;
         }
     }
-}
+}
\ No newline at end of file
diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql
index 47f954f3f..a97afb63f 100644
--- a/src/main/resources/migration/application/V1__init_schema.sql
+++ b/src/main/resources/migration/application/V1__init_schema.sql
@@ -28,6 +28,13 @@ create table ReferenceValue
 
     CONSTRAINT "hierarchicalKey_uniqueness" UNIQUE (application, referenceType, hierarchicalKey)
 );
+create table Reference_Reference
+(
+    referenceId entityid REFERENCES ReferenceValue(id) ON DELETE CASCADE,
+    referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT,
+    CONSTRAINT "Reference_Reference_PK" PRIMARY KEY (referenceId, referencedBy)
+);
+
 CREATE INDEX ref_refslinkedto_index ON ReferenceValue USING gin (refsLinkedTo);
 CREATE INDEX ref_refvalues_index ON ReferenceValue USING gin (refValues);
 
@@ -85,6 +92,14 @@ create table Data
     dataValues      jsonb,
     binaryFile      EntityRef REFERENCES BinaryFile (id)
 );
+
+create table Data_Reference
+(
+    dataId entityid REFERENCES Data(id) ON DELETE CASCADE,
+    referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT,
+    CONSTRAINT "Data_Reference_PK" PRIMARY KEY (dataId, referencedBy)
+);
+
 CREATE INDEX data_refslinkedto_index ON Data USING gin (refsLinkedTo);
 CREATE INDEX data_refvalues_index ON Data USING gin (dataValues);
 
@@ -105,15 +120,17 @@ CREATE TABLE OreSiAuthorization
 
 GRANT ALL PRIVILEGES ON BinaryFile TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON ReferenceValue TO "superadmin" WITH GRANT OPTION;
+GRANT ALL PRIVILEGES ON Reference_Reference TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON Data TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON OreSiAuthorization TO "superadmin" WITH GRANT OPTION;
 
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON BinaryFile TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON ReferenceValue TO public;
+GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Reference_Reference TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Data TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON OreSiAuthorization TO public;
 
 --ALTER TABLE BinaryFile ENABLE ROW LEVEL SECURITY;
 --ALTER TABLE ReferenceValue ENABLE ROW LEVEL SECURITY;
 ALTER TABLE Data
-    ENABLE ROW LEVEL SECURITY;
+    ENABLE ROW LEVEL SECURITY;
\ No newline at end of file
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index 05466322b..1dbd5e200 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -92,6 +92,170 @@ public class OreSiResourcesTest {
                 .andReturn().getResponse().getCookie(AuthHelper.JWT_COOKIE_NAME);
     }
 
+    @Test
+    public void addApplicationMonsoreWithRepository() throws Exception {
+        URL resource = getClass().getResource(fixtures.getMonsoreApplicationConfigurationResourceName());
+        String oirFilesUUID;
+        try (InputStream in = Objects.requireNonNull(resource).openStream()) {
+            MockMultipartFile configuration = new MockMultipartFile("file", "monsore.yaml", "text/plain", in);
+            //définition de l'application
+            authenticationService.addUserRightCreateApplication(userId);
+
+            String response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore")
+                            .file(configuration)
+                            .cookie(authCookie))
+                    .andExpect(status().isCreated())
+                    .andExpect(jsonPath("$.id", IsNull.notNullValue()))
+                    .andReturn().getResponse().getContentAsString();
+
+            JsonPath.parse(response).read("$.id");
+        }
+
+        String response = null;
+        // Ajout de referentiel
+        for (Map.Entry<String, String> e : fixtures.getMonsoreReferentielFiles().entrySet()) {
+            try (InputStream refStream = getClass().getResourceAsStream(e.getValue())) {
+                MockMultipartFile refFile = new MockMultipartFile("file", e.getValue(), "text/plain", refStream);
+
+                response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/references/{refType}", e.getKey())
+                                .file(refFile)
+                                .cookie(authCookie))
+                        .andExpect(status().isCreated())
+                        .andExpect(jsonPath("$.id", IsNull.notNullValue()))
+                        .andReturn().getResponse().getContentAsString();
+
+                JsonPath.parse(response).read("$.id");
+            }
+        }
+        // ajout de data
+        String projet = "manche";
+        String plateforme = "plateforme";
+        String site = "oir";
+        resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site));
+
+        // on dépose 3 fois le même fichier sans le publier
+        try (InputStream refStream = Objects.requireNonNull(resource).openStream()) {
+            MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream);
+
+            //fileOrUUID.binaryFileDataset/applications/{name}/file/{id}
+            for (int i = 0; i < 3; i++) {
+                response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
+                                .file(refFile)
+                                .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, false))
+                                .cookie(authCookie))
+                        .andExpect(status().is2xxSuccessful())
+                        .andReturn().getResponse().getContentAsString();
+            }
+            log.debug(response);
+            //on regarde les versions déposées
+            response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem")
+                            .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site))
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andExpect(jsonPath("$").isArray())
+                    .andExpect(jsonPath("$", Matchers.hasSize(3)))
+                    .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3)))
+                    .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0)))
+                    .andReturn().getResponse().getContentAsString();
+//            log.debug(response);
+            //récupération de l'identifiant de la dernière version déposée
+            oirFilesUUID = JsonPath.parse(response).read("$[2].id");
+
+            // on vérifie l'absence de data
+            response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem")
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andExpect(jsonPath("$.totalRows").value(-1))
+                    .andReturn().getResponse().getContentAsString();
+            log.debug(response);
+
+
+            // on publie le dernier fichier déposé
+
+            response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
+                            .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, true))
+                            .cookie(authCookie))
+                    // .andExpect(status().is2xxSuccessful())
+                    .andReturn().getResponse().getContentAsString();
+            log.debug(StringUtils.abbreviate(response, 50));
+
+            // on récupère la liste des versions déposées
+
+            response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem")
+                            .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site))
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andExpect(jsonPath("$").isArray())
+                    .andExpect(jsonPath("$", Matchers.hasSize(3)))
+                    .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(2)))
+                    .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(1)))
+                    .andExpect(jsonPath("$[*][?(@.params.published == true )].id").value(oirFilesUUID))
+                    .andReturn().getResponse().getContentAsString();
+            log.debug(StringUtils.abbreviate(response, 50));
+
+            // on récupère le data en base
+
+            response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem")
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andExpect(jsonPath("$.totalRows").value(34))
+                    .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(34)))
+                    .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'plateforme.oir.oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(34)))
+                    .andReturn().getResponse().getContentAsString();
+            log.debug(StringUtils.abbreviate(response, 50));
+        }
+        //on publie 4 fichiers
+
+        publishOrDepublish("manche", "plateforme", "scarff", 68, true, 1, true);
+        publishOrDepublish("atlantique", "plateforme", "scarff", 34, true, 1, true);
+        publishOrDepublish("atlantique", "plateforme", "nivelle", 34, true, 1, true);
+        publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 1, true);
+        //on publie une autre version
+        String fileUUID = publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 2, true);
+        // on supprime l'application publiée
+        response = mockMvc.perform(MockMvcRequestBuilders.delete("/api/v1/applications/monsore/file/" + fileUUID)
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andReturn().getResponse().getContentAsString();
+        log.debug(StringUtils.abbreviate(response, 50));
+        testFilesAndDataOnServer(plateforme, "manche", "nivelle", 0, 1, fileUUID, false);
+
+
+        // on depublie le fichier oir déposé
+
+        response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
+                        .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, false))
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andReturn().getResponse().getContentAsString();
+        log.debug(StringUtils.abbreviate(response, 50));
+
+        // on récupère la liste des versions déposées
+
+        response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem")
+                        .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site))
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$").isArray())
+                .andExpect(jsonPath("$", Matchers.hasSize(3)))
+                .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3)))
+                .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0)))
+                .andReturn().getResponse().getContentAsString();
+        log.debug(StringUtils.abbreviate(response, 50));
+
+        // on récupère le data en base
+
+        response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem")
+                        .cookie(authCookie))
+                .andExpect(status().is2xxSuccessful())
+                .andExpect(jsonPath("$.totalRows").value(136))
+                .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(136)))
+                .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(0)))
+                .andReturn().getResponse().getContentAsString();
+        log.debug(StringUtils.abbreviate(response, 50));
+        // on supprime le fic
+    }
+
     @Test
     public void addApplicationMonsore() throws Exception {
         String appId;
@@ -381,170 +545,6 @@ public class OreSiResourcesTest {
         // changement du fichier de config avec un mauvais (qui ne permet pas d'importer les fichiers
     }
 
-    @Test
-    public void addApplicationMonsoreWithRepository() throws Exception {
-        URL resource = getClass().getResource(fixtures.getMonsoreApplicationConfigurationResourceName());
-        String oirFilesUUID;
-        try (InputStream in = Objects.requireNonNull(resource).openStream()) {
-            MockMultipartFile configuration = new MockMultipartFile("file", "monsore.yaml", "text/plain", in);
-            //définition de l'application
-            authenticationService.addUserRightCreateApplication(userId);
-
-            String response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore")
-                            .file(configuration)
-                            .cookie(authCookie))
-                    .andExpect(status().isCreated())
-                    .andExpect(jsonPath("$.id", IsNull.notNullValue()))
-                    .andReturn().getResponse().getContentAsString();
-
-            JsonPath.parse(response).read("$.id");
-        }
-
-        String response = null;
-        // Ajout de referentiel
-        for (Map.Entry<String, String> e : fixtures.getMonsoreReferentielFiles().entrySet()) {
-            try (InputStream refStream = getClass().getResourceAsStream(e.getValue())) {
-                MockMultipartFile refFile = new MockMultipartFile("file", e.getValue(), "text/plain", refStream);
-
-                response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/references/{refType}", e.getKey())
-                                .file(refFile)
-                                .cookie(authCookie))
-                        .andExpect(status().isCreated())
-                        .andExpect(jsonPath("$.id", IsNull.notNullValue()))
-                        .andReturn().getResponse().getContentAsString();
-
-                JsonPath.parse(response).read("$.id");
-            }
-        }
-        // ajout de data
-        String projet = "manche";
-        String plateforme = "plateforme";
-        String site = "oir";
-        resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site));
-
-        // on dépose 3 fois le même fichier sans le publier
-        try (InputStream refStream = Objects.requireNonNull(resource).openStream()) {
-            MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream);
-
-            //fileOrUUID.binaryFileDataset/applications/{name}/file/{id}
-            for (int i = 0; i < 3; i++) {
-                response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
-                                .file(refFile)
-                                .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, false))
-                                .cookie(authCookie))
-                        .andExpect(status().is2xxSuccessful())
-                        .andReturn().getResponse().getContentAsString();
-            }
-            log.debug(response);
-            //on regarde les versions déposées
-            response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem")
-                            .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site))
-                            .cookie(authCookie))
-                    .andExpect(status().is2xxSuccessful())
-                    .andExpect(jsonPath("$").isArray())
-                    .andExpect(jsonPath("$", Matchers.hasSize(3)))
-                    .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3)))
-                    .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0)))
-                    .andReturn().getResponse().getContentAsString();
-//            log.debug(response);
-            //récupération de l'identifiant de la dernière version déposée
-            oirFilesUUID = JsonPath.parse(response).read("$[2].id");
-
-            // on vérifie l'absence de data
-            response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem")
-                            .cookie(authCookie))
-                    .andExpect(status().is2xxSuccessful())
-                    .andExpect(jsonPath("$.totalRows").value(-1))
-                    .andReturn().getResponse().getContentAsString();
-            log.debug(response);
-
-
-            // on publie le dernier fichier déposé
-
-            response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
-                            .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, true))
-                            .cookie(authCookie))
-                    // .andExpect(status().is2xxSuccessful())
-                    .andReturn().getResponse().getContentAsString();
-            log.debug(StringUtils.abbreviate(response, 50));
-
-            // on récupère la liste des versions déposées
-
-            response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem")
-                            .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site))
-                            .cookie(authCookie))
-                    .andExpect(status().is2xxSuccessful())
-                    .andExpect(jsonPath("$").isArray())
-                    .andExpect(jsonPath("$", Matchers.hasSize(3)))
-                    .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(2)))
-                    .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(1)))
-                    .andExpect(jsonPath("$[*][?(@.params.published == true )].id").value(oirFilesUUID))
-                    .andReturn().getResponse().getContentAsString();
-            log.debug(StringUtils.abbreviate(response, 50));
-
-            // on récupère le data en base
-
-            response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem")
-                            .cookie(authCookie))
-                    .andExpect(status().is2xxSuccessful())
-                    .andExpect(jsonPath("$.totalRows").value(34))
-                    .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(34)))
-                    .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'plateforme.oir.oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(34)))
-                    .andReturn().getResponse().getContentAsString();
-            log.debug(StringUtils.abbreviate(response, 50));
-        }
-        //on publie 4 fichiers
-
-        publishOrDepublish("manche", "plateforme", "scarff", 68, true, 1, true);
-        publishOrDepublish("atlantique", "plateforme", "scarff", 34, true, 1, true);
-        publishOrDepublish("atlantique", "plateforme", "nivelle", 34, true, 1, true);
-        publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 1, true);
-        //on publie une autre version
-        String fileUUID = publishOrDepublish("manche", "plateforme", "nivelle", 34, true, 2, true);
-        // on supprime l'application publiée
-        response = mockMvc.perform(MockMvcRequestBuilders.delete("/api/v1/applications/monsore/file/" + fileUUID)
-                        .cookie(authCookie))
-                .andExpect(status().is2xxSuccessful())
-                .andReturn().getResponse().getContentAsString();
-        log.debug(StringUtils.abbreviate(response, 50));
-        testFilesAndDataOnServer(plateforme, "manche", "nivelle", 0, 1, fileUUID, false);
-
-
-        // on depublie le fichier oir déposé
-
-        response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
-                        .param("params", fixtures.getPemRepositoryParamsWithId(projet, plateforme, site, oirFilesUUID, false))
-                        .cookie(authCookie))
-                .andExpect(status().is2xxSuccessful())
-                .andReturn().getResponse().getContentAsString();
-        log.debug(StringUtils.abbreviate(response, 50));
-
-        // on récupère la liste des versions déposées
-
-        response = mockMvc.perform(get("/api/v1/applications/monsore/filesOnRepository/pem")
-                        .param("repositoryId", fixtures.getPemRepositoryId(plateforme, projet, site))
-                        .cookie(authCookie))
-                .andExpect(status().is2xxSuccessful())
-                .andExpect(jsonPath("$").isArray())
-                .andExpect(jsonPath("$", Matchers.hasSize(3)))
-                .andExpect(jsonPath("$[*][?(@.params.published == false )]", Matchers.hasSize(3)))
-                .andExpect(jsonPath("$[*][?(@.params.published == true )]", Matchers.hasSize(0)))
-                .andReturn().getResponse().getContentAsString();
-        log.debug(StringUtils.abbreviate(response, 50));
-
-        // on récupère le data en base
-
-        response = mockMvc.perform(get("/api/v1/applications/monsore/data/pem")
-                        .cookie(authCookie))
-                .andExpect(status().is2xxSuccessful())
-                .andExpect(jsonPath("$.totalRows").value(136))
-                .andExpect(jsonPath("$.rows[*]", Matchers.hasSize(136)))
-                .andExpect(jsonPath("$.rows[*].values[? (@.site.chemin == 'oir__p1')][? (@.projet.value == 'projet_manche')]", Matchers.hasSize(0)))
-                .andReturn().getResponse().getContentAsString();
-        log.debug(StringUtils.abbreviate(response, 50));
-        // on supprime le fic
-    }
-
     private String publishOrDepublish(String projet, String plateforme, String site, int expected, boolean toPublish, int numberOfVersions, boolean published) throws Exception {
         URL resource;
         String response;
-- 
GitLab


From b74e043f1c85a82694e9621874692432c2938b55 Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Fri, 11 Mar 2022 10:24:52 +0100
Subject: [PATCH 2/9] =?UTF-8?q?M=C3=AAme=20chose=20pour=20les=20data=20et?=
 =?UTF-8?q?=20suppression=20des=20checks?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../oresing/persistence/DataRepository.java   | 24 ++++++
 .../fr/inra/oresing/rest/OreSiService.java    |  4 +-
 .../migration/application/V1__init_schema.sql |  7 +-
 .../migration/main/V1__init_schema.sql        | 84 -------------------
 .../inra/oresing/rest/OreSiResourcesTest.java | 48 +++++------
 5 files changed, 55 insertions(+), 112 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
index 386f1d4a7..a89299444 100644
--- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
@@ -5,14 +5,17 @@ import com.google.common.collect.ImmutableMap;
 import fr.inra.oresing.model.Application;
 import fr.inra.oresing.model.Data;
 import fr.inra.oresing.rest.DownloadDatasetQuery;
+import org.postgresql.util.PSQLException;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Scope;
+import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 @Component
 @Scope(scopeName = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
@@ -82,4 +85,25 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla
         int count = getNamedParameterJdbcTemplate().update(sql, sqlParams);
         return count;
     }
+
+    public void updateConstraintForeigData(List<UUID> uuids) {
+        String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)\n" +
+                "with tuple as (\n" +
+                "select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy\n" +
+                "from " + getTable().getSqlIdentifier() + "\n" +
+                ")\n" +
+                "select dataId, referencedBy from tuple\n" +
+                "where dataId in (:ids) and referencedBy is not null\n" +
+                "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING;";
+        final String ids = uuids.stream()
+                .map(uuid -> String.format("'%s'::uuid", uuid))
+                .collect(Collectors.joining(","));
+        try {
+            List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper());
+        } catch (DataIntegrityViolationException e) {
+            if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){
+                throw e;
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index b9e96d0a3..943aaeea4 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -511,7 +511,9 @@ public class OreSiService {
                     .map(buildReplaceMissingValuesByDefaultValuesFn(app, dataType, binaryFileDataset == null ? null : binaryFileDataset.getRequiredauthorizations()))
                     .flatMap(buildLineValuesToEntityStreamFn(app, dataType, storedFile.getId(), errors, binaryFileDataset));
 
-            repo.getRepository(app).data().storeAll(dataStream);
+            final DataRepository dataRepository = repo.getRepository(app).data();
+            final List<UUID> uuids = dataRepository.storeAll(dataStream);
+            dataRepository.updateConstraintForeigData(uuids);
         }
     }
 
diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql
index a97afb63f..0a03eec4f 100644
--- a/src/main/resources/migration/application/V1__init_schema.sql
+++ b/src/main/resources/migration/application/V1__init_schema.sql
@@ -22,7 +22,7 @@ create table ReferenceValue
     hierarchicalKey       ltree NOT NULL,
     hierarchicalReference ltree NOT NULL,
     naturalKey            ltree NOT NULL,
-    refsLinkedTo          jsonb check (refs_check_for_reference('${applicationSchema}', application, refsLinkedTo)),
+    refsLinkedTo          jsonb ,
     refValues             jsonb,
     binaryFile            EntityRef REFERENCES BinaryFile (id),
 
@@ -87,8 +87,7 @@ create table Data
     rowId           TEXT                                                             NOT NULL,
     datagroup       TEXT GENERATED ALWAYS AS (("authorization").datagroup[1]) STORED NOT NULL,
     "authorization" ${applicationSchema}.authorization                               NOT NULL check (("authorization").datagroup[1] is not null),
-    refsLinkedTo    jsonb check (refs_check_for_datatype('${applicationSchema}', application, refsLinkedTo,
-                                                         datatype)),
+    refsLinkedTo    jsonb ,
     dataValues      jsonb,
     binaryFile      EntityRef REFERENCES BinaryFile (id)
 );
@@ -122,12 +121,14 @@ GRANT ALL PRIVILEGES ON BinaryFile TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON ReferenceValue TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON Reference_Reference TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON Data TO "superadmin" WITH GRANT OPTION;
+GRANT ALL PRIVILEGES ON Data_Reference TO "superadmin" WITH GRANT OPTION;
 GRANT ALL PRIVILEGES ON OreSiAuthorization TO "superadmin" WITH GRANT OPTION;
 
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON BinaryFile TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON ReferenceValue TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Reference_Reference TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Data TO public;
+GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON Data_Reference TO public;
 GRANT SELECT, INSERT, UPDATE, DELETE, REFERENCES ON OreSiAuthorization TO public;
 
 --ALTER TABLE BinaryFile ENABLE ROW LEVEL SECURITY;
diff --git a/src/main/resources/migration/main/V1__init_schema.sql b/src/main/resources/migration/main/V1__init_schema.sql
index a7126613f..0fc93b18c 100644
--- a/src/main/resources/migration/main/V1__init_schema.sql
+++ b/src/main/resources/migration/main/V1__init_schema.sql
@@ -39,79 +39,6 @@ BEGIN
 END;
 $$ language 'plpgsql';*/
 
--- check les foreign key pour le colonne references de la table data
-
-CREATE OR REPLACE FUNCTION refs_check_for_datatype(aschema text, application UUID, refValues jsonb, dtype TEXT)
-    RETURNS BOOLEAN AS $$
-DECLARE
-    result TEXT;
-BEGIN
-    EXECUTE 'with agg as (
-   SELECT application."configuration"
-   ->''datatypes''
-   ->$4
-	->''data''
-	->jsonb_object_keys(
-		application."configuration"
-			->''datatypes''
-			->$4
-			->''data''
-	)
-	->''components''
-	->jsonb_object_keys(
-		application."configuration"->''datatypes''
-			->$4
-			->''data''
-			->jsonb_object_keys(
-						application."configuration"
-							->''datatypes''
-							->$4
-							->''data''
-					)
-			->''components'')
-	->''checker''
-	->''params''
-	->>''refType'' reftype,
-	$3
-	->jsonb_object_keys(
-		application."configuration"
-			->''datatypes''
-			->$4
-			->''data''
-		)
-	->jsonb_object_keys(
-		application."configuration"
-			->''datatypes''
-			->$4
-			->''data''
-			->jsonb_object_keys(
-				application."configuration"
-					->''datatypes''
-					->$4
-					->''data''
-					)
-			->''components''
-			) reference
-	FROM   application
-   where application.id = $2),
-    byref as (
-   select jsonb_build_object(reftype::TEXT, array_agg(distinct reference)) byref
-        from agg
-        where reftype is not null and reference is not null
-        group by reftype),
-    refvalues as (
-	select  jsonb_object_agg(byref) refvalues
-	from byref
-	group by $2)
-	SELECT count(id) = jsonb_count_items(refvalues.refvalues)
-	from refvalues, ' || aSchema || '.referencevalue
-	where application=$2::uuid and jsonb_build_object(referenceType, ARRAY[id]) <@ refvalues.refvalues
-	group by refvalues.refvalues;'
-   INTO result USING aschema, application, refValues, dtype;
-    return result;
-END;
-$$  LANGUAGE plpgsql;
-
 --check if all elements of oreSiUser array are users
 CREATE OR REPLACE FUNCTION checks_users(users uuid[])
     RETURNS BOOLEAN AS $$
@@ -123,18 +50,7 @@ BEGIN
 END;
 $$  LANGUAGE plpgsql;
 
--- check les foreign key pour le colonne references de la table data
 
-CREATE OR REPLACE FUNCTION refs_check_for_reference(aSchema text, application UUID, refValues jsonb)
-RETURNS BOOLEAN AS $$
-DECLARE
-    result TEXT;
-BEGIN
-    EXECUTE 'select count(id) = jsonb_count_items($2) from ' || aSchema || '.referencevalue where application=$1::uuid and jsonb_build_object(referenceType, ARRAY[id]) <@ $2 ' ||
-            '' INTO result USING application, refValues;
-    RETURN result;
-END;
-$$ language 'plpgsql';
 
 CREATE OR REPLACE FUNCTION name_check(application UUID, targetColumn TEXT, val TEXT)
 RETURNS BOOLEAN AS $$
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index 1dbd5e200..ca59fbc36 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -256,6 +256,30 @@ public class OreSiResourcesTest {
         // on supprime le fic
     }
 
+    private String publishOrDepublish(String projet, String plateforme, String site, int expected, boolean toPublish, int numberOfVersions, boolean published) throws Exception {
+        URL resource;
+        String response;
+        resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site));
+        try (InputStream refStream = Objects.requireNonNull(resource).openStream()) {
+
+            //dépôt et publication d'un fichier projet site__p1
+            MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream);
+            refFile.transferTo(Path.of("/tmp/pem.csv"));
+            response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
+                            .file(refFile)
+                            .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, toPublish))
+                            .cookie(authCookie))
+                    .andExpect(status().is2xxSuccessful())
+                    .andReturn().getResponse().getContentAsString();
+            String fileUUID = JsonPath.parse(response).read("$.fileId");
+
+            //liste des fichiers projet/site
+            testFilesAndDataOnServer(plateforme, projet, site, expected, numberOfVersions, fileUUID, published);
+            log.debug(StringUtils.abbreviate(response, 50));
+            return fileUUID;
+        }
+    }
+
     @Test
     public void addApplicationMonsore() throws Exception {
         String appId;
@@ -545,30 +569,6 @@ public class OreSiResourcesTest {
         // changement du fichier de config avec un mauvais (qui ne permet pas d'importer les fichiers
     }
 
-    private String publishOrDepublish(String projet, String plateforme, String site, int expected, boolean toPublish, int numberOfVersions, boolean published) throws Exception {
-        URL resource;
-        String response;
-        resource = getClass().getResource(fixtures.getPemRepositoryDataResourceName(projet, site));
-        try (InputStream refStream = Objects.requireNonNull(resource).openStream()) {
-
-            //dépôt et publication d'un fichier projet site__p1
-            MockMultipartFile refFile = new MockMultipartFile("file", String.format("%s-%s-p1-pem.csv", projet, site), "text/plain", refStream);
-            refFile.transferTo(Path.of("/tmp/pem.csv"));
-            response = mockMvc.perform(MockMvcRequestBuilders.multipart("/api/v1/applications/monsore/data/pem")
-                            .file(refFile)
-                            .param("params", fixtures.getPemRepositoryParams(projet, plateforme, site, toPublish))
-                            .cookie(authCookie))
-                    .andExpect(status().is2xxSuccessful())
-                    .andReturn().getResponse().getContentAsString();
-            String fileUUID = JsonPath.parse(response).read("$.fileId");
-
-            //liste des fichiers projet/site
-            testFilesAndDataOnServer(plateforme, projet, site, expected, numberOfVersions, fileUUID, published);
-            log.debug(StringUtils.abbreviate(response, 50));
-            return fileUUID;
-        }
-    }
-
     @Test
     public void testRecursivity() throws Exception {
 
-- 
GitLab


From 7455aa4328f5fd33bea9b984e80a5fac0c0e6361 Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Fri, 11 Mar 2022 10:54:29 +0100
Subject: [PATCH 3/9] =?UTF-8?q?D=C3=A9coupage=20by=20batch?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

-le nombre de paramètres que l'on peut passer est limité (max et min 32767)
---
 .../oresing/persistence/DataRepository.java   |  24 ++--
 .../persistence/ReferenceValueRepository.java | 110 +++++++-----------
 2 files changed, 54 insertions(+), 80 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
index a89299444..f39676eef 100644
--- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
@@ -2,6 +2,7 @@ package fr.inra.oresing.persistence;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterators;
 import fr.inra.oresing.model.Application;
 import fr.inra.oresing.model.Data;
 import fr.inra.oresing.rest.DownloadDatasetQuery;
@@ -95,15 +96,18 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla
                 "select dataId, referencedBy from tuple\n" +
                 "where dataId in (:ids) and referencedBy is not null\n" +
                 "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING;";
-        final String ids = uuids.stream()
-                .map(uuid -> String.format("'%s'::uuid", uuid))
-                .collect(Collectors.joining(","));
-        try {
-            List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper());
-        } catch (DataIntegrityViolationException e) {
-            if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){
-                throw e;
-            }
-        }
+        Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE-1)
+                .forEachRemaining(uuidsByBatch -> {
+                    final String ids = uuidsByBatch.stream()
+                            .map(uuid -> String.format("'%s'::uuid", uuid))
+                            .collect(Collectors.joining(","));
+                    try {
+                        getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper());
+                    } catch (DataIntegrityViolationException e) {
+                        if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) {
+                            throw e;
+                        }
+                    }
+                });
     }
 }
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
index d4f750574..56fb6c68d 100644
--- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
@@ -2,12 +2,8 @@ package fr.inra.oresing.persistence;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
-import fr.inra.oresing.model.Application;
-import fr.inra.oresing.model.ReferenceColumn;
-import fr.inra.oresing.model.ReferenceColumnSingleValue;
-import fr.inra.oresing.model.ReferenceColumnValue;
-import fr.inra.oresing.model.ReferenceDatum;
-import fr.inra.oresing.model.ReferenceValue;
+import com.google.common.collect.Iterators;
+import fr.inra.oresing.model.*;
 import fr.inra.oresing.rest.ApplicationResult;
 import org.apache.commons.lang3.StringUtils;
 import org.postgresql.util.PSQLException;
@@ -42,14 +38,7 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
 
     @Override
     protected String getUpsertQuery() {
-        return "INSERT INTO " + getTable().getSqlIdentifier() + "\n" +
-                "(id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile) \n" +
-                "SELECT id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile \n" +
-                "FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", \n" +
-                ":json::json) \n"
-                + " ON CONFLICT ON CONSTRAINT \"hierarchicalKey_uniqueness\" \n" +
-                "DO UPDATE SET updateDate=current_timestamp, hierarchicalKey=EXCLUDED.hierarchicalKey, hierarchicalReference=EXCLUDED.hierarchicalReference, naturalKey=EXCLUDED.naturalKey, refsLinkedTo=EXCLUDED.refsLinkedTo, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile"
-                + " RETURNING id";
+        return "INSERT INTO " + getTable().getSqlIdentifier() + "\n" + "(id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile) \n" + "SELECT id, application, referenceType, hierarchicalKey, hierarchicalReference, naturalKey, refsLinkedTo, refValues, binaryFile \n" + "FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", \n" + ":json::json) \n" + " ON CONFLICT ON CONSTRAINT \"hierarchicalKey_uniqueness\" \n" + "DO UPDATE SET updateDate=current_timestamp, hierarchicalKey=EXCLUDED.hierarchicalKey, hierarchicalReference=EXCLUDED.hierarchicalReference, naturalKey=EXCLUDED.naturalKey, refsLinkedTo=EXCLUDED.refsLinkedTo, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile" + " RETURNING id";
     }
 
     @Override
@@ -68,10 +57,8 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
      */
     public List<ReferenceValue> findAllByReferenceType(String refType, MultiValueMap<String, String> params) {
         MultiValueMap<String, String> toto = new LinkedMultiValueMap<>();
-        String query = "SELECT DISTINCT '" + ReferenceValue.class.getName() + "' as \"@class\",  to_jsonb(t) as json FROM "
-                + getTable().getSqlIdentifier() + " t, jsonb_each_text(t.refvalues) kv WHERE application=:applicationId::uuid AND referenceType=:refType";
-        MapSqlParameterSource paramSource = new MapSqlParameterSource("applicationId", getApplication().getId())
-                .addValue("refType", refType);
+        String query = "SELECT DISTINCT '" + ReferenceValue.class.getName() + "' as \"@class\",  to_jsonb(t) as json FROM " + getTable().getSqlIdentifier() + " t, jsonb_each_text(t.refvalues) kv WHERE application=:applicationId::uuid AND referenceType=:refType";
+        MapSqlParameterSource paramSource = new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType);
 
         AtomicInteger i = new AtomicInteger();
         // kv.value='LPF' OR t.refvalues @> '{"esp_nom":"ALO"}'::jsonb
@@ -79,13 +66,12 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
             String k = e.getKey();
             if (StringUtils.equalsAnyIgnoreCase("_row_id_", k)) {
                 String collect = e.getValue().stream().map(v -> {
-                            String arg = ":arg" + i.getAndIncrement();
-                            paramSource.addValue(arg, v);
-                            return String.format("'%s'::uuid", v);
-                        })
-                        .collect(Collectors.joining(", "));
+                    String arg = ":arg" + i.getAndIncrement();
+                    paramSource.addValue(arg, v);
+                    return String.format("'%s'::uuid", v);
+                }).collect(Collectors.joining(", "));
                 return Stream.ofNullable(String.format("array[id]::uuid[] <@ array[%s]::uuid[]", collect));
-            }else if (StringUtils.equalsAnyIgnoreCase("any", k)) {
+            } else if (StringUtils.equalsAnyIgnoreCase("any", k)) {
                 return e.getValue().stream().map(v -> {
                     String arg = ":arg" + i.getAndIncrement();
                     paramSource.addValue(arg, v);
@@ -94,9 +80,7 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
             } else {
                 return e.getValue().stream().map(v -> "t.refvalues @> '{\"" + k + "\":\"" + v + "\"}'::jsonb");
             }
-        })
-                .filter(k->k!=null).
-                collect(Collectors.joining(" OR "));
+        }).filter(k -> k != null).collect(Collectors.joining(" OR "));
 
         if (StringUtils.isNotBlank(cond)) {
             cond = " AND (" + cond + ")";
@@ -108,60 +92,46 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
 
     public List<List<String>> findReferenceValue(String refType, String column) {
         AtomicInteger ai = new AtomicInteger(0);
-        String select = Stream.of(column.split(","))
-                .map(c -> String.format("refValues->>'%1$s' as \"%1$s"+ai.getAndIncrement()+"\"", c))
-                .collect(Collectors.joining(", "));
-        String sqlPattern = " SELECT %s "
-                + " FROM " + getTable().getSqlIdentifier() + " t"
-                + " WHERE application=:applicationId::uuid AND referenceType=:refType";
+        String select = Stream.of(column.split(",")).map(c -> String.format("refValues->>'%1$s' as \"%1$s" + ai.getAndIncrement() + "\"", c)).collect(Collectors.joining(", "));
+        String sqlPattern = " SELECT %s " + " FROM " + getTable().getSqlIdentifier() + " t" + " WHERE application=:applicationId::uuid AND referenceType=:refType";
         String query = String.format(sqlPattern, select);
-        List<List<String>> result = getNamedParameterJdbcTemplate().queryForList(query, new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType))
-                .stream()
-                .map(m -> m.values().stream().map(v -> (String) v).collect(Collectors.toList()))
-                .collect(Collectors.toList());
+        List<List<String>> result = getNamedParameterJdbcTemplate().queryForList(query, new MapSqlParameterSource("applicationId", getApplication().getId()).addValue("refType", refType)).stream().map(m -> m.values().stream().map(v -> (String) v).collect(Collectors.toList())).collect(Collectors.toList());
         return result;
     }
 
     public ImmutableMap<Ltree, ApplicationResult.Reference.ReferenceUUIDAndDisplay> getReferenceIdAndDisplayPerKeys(String referenceType, String locale) {
-        Function<ReferenceValue, ApplicationResult.Reference.ReferenceUUIDAndDisplay> referenceValueToReferenceUuidAndDisplayFunction =
-                result -> {
-                    ReferenceDatum referenceDatum = result.getRefValues();
-                    ReferenceColumn referenceColumnForDisplay = ReferenceColumn.forDisplay(locale);
-                    String display;
-                    if (referenceDatum.contains(referenceColumnForDisplay)) {
-                        ReferenceColumnValue referenceColumnValueForDisplay = referenceDatum.get(referenceColumnForDisplay);
-                        Preconditions.checkState(referenceColumnValueForDisplay instanceof ReferenceColumnSingleValue);
-                        display = ((ReferenceColumnSingleValue) referenceColumnValueForDisplay).getValue();
-                    } else {
-                        display = null;
-                    }
-                    Map<String, String> values = referenceDatum.toJsonForFrontend();
-                    return new ApplicationResult.Reference.ReferenceUUIDAndDisplay(display, result.getId(), values);
-                };
-        return findAllByReferenceType(referenceType).stream()
-                .collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, referenceValueToReferenceUuidAndDisplayFunction));
+        Function<ReferenceValue, ApplicationResult.Reference.ReferenceUUIDAndDisplay> referenceValueToReferenceUuidAndDisplayFunction = result -> {
+            ReferenceDatum referenceDatum = result.getRefValues();
+            ReferenceColumn referenceColumnForDisplay = ReferenceColumn.forDisplay(locale);
+            String display;
+            if (referenceDatum.contains(referenceColumnForDisplay)) {
+                ReferenceColumnValue referenceColumnValueForDisplay = referenceDatum.get(referenceColumnForDisplay);
+                Preconditions.checkState(referenceColumnValueForDisplay instanceof ReferenceColumnSingleValue);
+                display = ((ReferenceColumnSingleValue) referenceColumnValueForDisplay).getValue();
+            } else {
+                display = null;
+            }
+            Map<String, String> values = referenceDatum.toJsonForFrontend();
+            return new ApplicationResult.Reference.ReferenceUUIDAndDisplay(display, result.getId(), values);
+        };
+        return findAllByReferenceType(referenceType).stream().collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, referenceValueToReferenceUuidAndDisplayFunction));
     }
 
     public ImmutableMap<Ltree, UUID> getReferenceIdPerKeys(String referenceType) {
-        return findAllByReferenceType(referenceType).stream()
-                .collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, ReferenceValue::getId));
+        return findAllByReferenceType(referenceType).stream().collect(ImmutableMap.toImmutableMap(ReferenceValue::getHierarchicalKey, ReferenceValue::getId));
     }
 
     public void updateConstraintForeignReferences(List<UUID> uuids) {
-        String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" +
-                "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" +
-                "from  " + getTable().getSqlIdentifier() + "\n" +
-                "where id in (:ids)" +
-                "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;";
-        final String ids = uuids.stream()
-                .map(uuid -> String.format("'%s'::uuid", uuid))
-                .collect(Collectors.joining(","));
-        try {
-            List result = getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuids), getJsonRowMapper());
-        } catch (DataIntegrityViolationException e) {
-            if(e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException)e.getCause()).getSQLState())){
-                throw e;
+        String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" + "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" + "from  " + getTable().getSqlIdentifier() + "\n" + "where id in (:ids)" + "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;";
+        Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1).forEachRemaining(uuidsByBatch -> {
+            final String ids = uuidsByBatch.stream().map(uuid -> String.format("'%s'::uuid", uuid)).collect(Collectors.joining(","));
+            try {
+                getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper());
+            } catch (DataIntegrityViolationException e) {
+                if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) {
+                    throw e;
+                }
             }
-        }
+        });
     }
 }
\ No newline at end of file
-- 
GitLab


From 8abc4e2a0afff64bc09a81f912fe689f863fb36d Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Fri, 11 Mar 2022 18:08:20 +0100
Subject: [PATCH 4/9] =?UTF-8?q?=C3=89vite=20un=20chargement=20du=20r=C3=A9?=
 =?UTF-8?q?f=C3=A9rentiel=20avant=20sa=20mise=20=C3=A0=20jour=20(semble=20?=
 =?UTF-8?q?inutile)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/java/fr/inra/oresing/rest/OreSiService.java     | 2 --
 .../java/fr/inra/oresing/rest/ReferenceImporter.java     | 7 -------
 .../fr/inra/oresing/rest/ReferenceImporterContext.java   | 9 ---------
 3 files changed, 18 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index 4b8745d43..2e60490c4 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -387,7 +387,6 @@ public class OreSiService {
         ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue();
         Configuration conf = app.getConfiguration();
         ImmutableSet<LineChecker> lineCheckers = checkerFactory.getReferenceValidationLineCheckers(app, refType);
-        final ImmutableMap<Ltree, UUID> storedReferences = referenceValueRepository.getReferenceIdPerKeys(refType);
 
         ImmutableMap<ReferenceColumn, Multiplicity> multiplicityPerColumns = lineCheckers.stream()
                 .filter(lineChecker -> lineChecker instanceof ReferenceLineChecker)
@@ -451,7 +450,6 @@ public class OreSiService {
                         conf,
                         refType,
                         lineCheckers,
-                        storedReferences,
                         columns
                 );
         return referenceImporterContext;
diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
index 4b3433c46..4cfa936c4 100644
--- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
+++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
@@ -292,13 +292,6 @@ abstract class ReferenceImporter {
         final Ltree hierarchicalReference = recursionStrategy.getHierarchicalReference(naturalKey);
         referenceDatum.putAll(InternationalizationDisplay.getDisplays(referenceImporterContext.getDisplayPattern(), referenceImporterContext.getDisplayColumns(), referenceDatum));
 
-        /**
-         * on remplace l'id par celle en base si elle existe
-         * a noter que pour les references récursives on récupère l'id depuis  referenceLineChecker.getReferenceValues() ce qui revient au même
-         */
-
-        referenceImporterContext.getIdForSameHierarchicalKeyInDatabase(hierarchicalKey)
-                .ifPresent(e::setId);
         e.setBinaryFile(fileId);
         e.setReferenceType(referenceImporterContext.getRefType());
         e.setHierarchicalKey(hierarchicalKey);
diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java
index ae0d71e64..57e858ca6 100644
--- a/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java
+++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporterContext.java
@@ -63,11 +63,6 @@ public class ReferenceImporterContext {
      */
     private final ImmutableSet<LineChecker> lineCheckers;
 
-    /**
-     * Les clés techniques de chaque clé naturelle hiérarchique de toutes les lignes existantes en base (avant l'import)
-     */
-    private final ImmutableMap<Ltree, UUID> storedReferences;
-
     private final ImmutableMap<String, Column> columnsPerHeader;
 
     private Optional<InternationalizationReferenceMap> getInternationalizationReferenceMap() {
@@ -196,10 +191,6 @@ public class ReferenceImporterContext {
         return applicationId;
     }
 
-    public Optional<UUID> getIdForSameHierarchicalKeyInDatabase(Ltree hierarchicalKey) {
-        return Optional.ofNullable(storedReferences.get(hierarchicalKey));
-    }
-
     public void pushValue(ReferenceDatum referenceDatum, String header, String cellContent, SetMultimap<String, UUID> refsLinkedTo) {
         Column column = columnsPerHeader.get(header);
         column.pushValue(cellContent, referenceDatum, refsLinkedTo);
-- 
GitLab


From de7df312a29d330ff7b20942e16e8766465edd5c Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Mon, 14 Mar 2022 17:50:01 +0100
Subject: [PATCH 5/9] =?UTF-8?q?=C3=89vite=20une=20exception=20due=20=C3=A0?=
 =?UTF-8?q?=20une=20mauvaise=20utilisation=20de=20la=20persistance?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../oresing/persistence/DataRepository.java   | 35 +++++++------------
 .../persistence/ReferenceValueRepository.java | 30 ++++++++--------
 2 files changed, 28 insertions(+), 37 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
index f39676eef..bebbcd378 100644
--- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
@@ -6,17 +6,15 @@ import com.google.common.collect.Iterators;
 import fr.inra.oresing.model.Application;
 import fr.inra.oresing.model.Data;
 import fr.inra.oresing.rest.DownloadDatasetQuery;
-import org.postgresql.util.PSQLException;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Scope;
-import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Component;
 
+import java.sql.PreparedStatement;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
-import java.util.stream.Collectors;
 
 @Component
 @Scope(scopeName = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
@@ -88,26 +86,17 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla
     }
 
     public void updateConstraintForeigData(List<UUID> uuids) {
-        String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)\n" +
-                "with tuple as (\n" +
-                "select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy\n" +
-                "from " + getTable().getSqlIdentifier() + "\n" +
-                ")\n" +
-                "select dataId, referencedBy from tuple\n" +
-                "where dataId in (:ids) and referencedBy is not null\n" +
-                "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING;";
+        String sql = String.join(" "
+                , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)"
+                , "with tuple as ("
+                , "  select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy"
+                , "  from " + getTable().getSqlIdentifier() + ""
+                , ")"
+                , "select dataId, referencedBy from tuple"
+                , "where dataId in (:ids) and referencedBy is not null"
+                , "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING"
+        );
         Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE-1)
-                .forEachRemaining(uuidsByBatch -> {
-                    final String ids = uuidsByBatch.stream()
-                            .map(uuid -> String.format("'%s'::uuid", uuid))
-                            .collect(Collectors.joining(","));
-                    try {
-                        getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper());
-                    } catch (DataIntegrityViolationException e) {
-                        if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) {
-                            throw e;
-                        }
-                    }
-                });
+                .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute));
     }
 }
\ No newline at end of file
diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
index 56fb6c68d..d704df063 100644
--- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
@@ -3,18 +3,22 @@ package fr.inra.oresing.persistence;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterators;
-import fr.inra.oresing.model.*;
+import fr.inra.oresing.model.Application;
+import fr.inra.oresing.model.ReferenceColumn;
+import fr.inra.oresing.model.ReferenceColumnSingleValue;
+import fr.inra.oresing.model.ReferenceColumnValue;
+import fr.inra.oresing.model.ReferenceDatum;
+import fr.inra.oresing.model.ReferenceValue;
 import fr.inra.oresing.rest.ApplicationResult;
 import org.apache.commons.lang3.StringUtils;
-import org.postgresql.util.PSQLException;
 import org.springframework.beans.factory.config.ConfigurableBeanFactory;
 import org.springframework.context.annotation.Scope;
-import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.jdbc.core.namedparam.MapSqlParameterSource;
 import org.springframework.stereotype.Component;
 import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
 
+import java.sql.PreparedStatement;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -122,16 +126,14 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
     }
 
     public void updateConstraintForeignReferences(List<UUID> uuids) {
-        String sql = "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)\n" + "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy\n" + "from  " + getTable().getSqlIdentifier() + "\n" + "where id in (:ids)" + "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING;";
-        Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1).forEachRemaining(uuidsByBatch -> {
-            final String ids = uuidsByBatch.stream().map(uuid -> String.format("'%s'::uuid", uuid)).collect(Collectors.joining(","));
-            try {
-                getNamedParameterJdbcTemplate().query(sql, ImmutableMap.of("ids", uuidsByBatch), getJsonRowMapper());
-            } catch (DataIntegrityViolationException e) {
-                if (e.getCause() instanceof PSQLException && !"02000".equals(((PSQLException) e.getCause()).getSQLState())) {
-                    throw e;
-                }
-            }
-        });
+        String sql = String.join(" "
+                , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)"
+                , "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy"
+                , "from " + getTable().getSqlIdentifier()
+                , "where id in (:ids)"
+                , "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING"
+        );
+        Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1)
+                .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute));
     }
 }
\ No newline at end of file
-- 
GitLab


From 95113af899655363972c853cfb27522bb82e39ec Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Wed, 16 Mar 2022 16:54:26 +0100
Subject: [PATCH 6/9] =?UTF-8?q?Met=20=C3=A0=20jour=20les=20tables=20associ?=
 =?UTF-8?q?ations=20en=20cas=20de=20suppression=20d'un=20lien=20dans=20le?=
 =?UTF-8?q?=20refsLinkedTo?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/java/fr/inra/oresing/persistence/DataRepository.java | 4 +++-
 .../fr/inra/oresing/persistence/ReferenceValueRepository.java | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/persistence/DataRepository.java b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
index bebbcd378..b155f8581 100644
--- a/src/main/java/fr/inra/oresing/persistence/DataRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/DataRepository.java
@@ -86,7 +86,8 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla
     }
 
     public void updateConstraintForeigData(List<UUID> uuids) {
-        String sql = String.join(" "
+        String deleteSql = "DELETE FROM " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference WHERE dataId in (:ids)";
+        String insertSql = String.join(" "
                 , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Data_Reference(dataId, referencedBy)"
                 , "with tuple as ("
                 , "  select id dataId,((jsonb_each_text( (jsonb_each(refsLinkedTo)).value)).value)::uuid referencedBy"
@@ -96,6 +97,7 @@ public class DataRepository extends JsonTableInApplicationSchemaRepositoryTempla
                 , "where dataId in (:ids) and referencedBy is not null"
                 , "ON CONFLICT ON CONSTRAINT \"Data_Reference_PK\" DO NOTHING"
         );
+        String sql = String.join(";", deleteSql, insertSql);
         Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE-1)
                 .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute));
     }
diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
index d704df063..3d8f5a865 100644
--- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
@@ -126,13 +126,15 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
     }
 
     public void updateConstraintForeignReferences(List<UUID> uuids) {
-        String sql = String.join(" "
+        String deleteSql = "DELETE FROM " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference WHERE referenceId in (:ids)";
+        String insertSql = String.join(" "
                 , "INSERT INTO " + getTable().getSchema().getSqlIdentifier() + ".Reference_Reference(referenceId, referencedBy)"
                 , "select id referenceId, (jsonb_array_elements_text((jsonb_each(refsLinkedTo)).value))::uuid referencedBy"
                 , "from " + getTable().getSqlIdentifier()
                 , "where id in (:ids)"
                 , "ON CONFLICT ON CONSTRAINT \"Reference_Reference_PK\" DO NOTHING"
         );
+        String sql = String.join(";", insertSql, deleteSql);
         Iterators.partition(uuids.stream().iterator(), Short.MAX_VALUE - 1)
                 .forEachRemaining(uuidsByBatch -> getNamedParameterJdbcTemplate().execute(sql, ImmutableMap.of("ids", uuidsByBatch), PreparedStatement::execute));
     }
-- 
GitLab


From b47ee3b55b204705480f122f13a6afdc40440ba1 Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Wed, 16 Mar 2022 17:55:13 +0100
Subject: [PATCH 7/9] =?UTF-8?q?Permet=20la=20mise=20=C3=A0=20jour=20des=20?=
 =?UTF-8?q?r=C3=A9f=C3=A9rentiel=20sans=20reprendre=20la=20cl=C3=A9=20tech?=
 =?UTF-8?q?nique?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../inra/oresing/rest/ReferenceImporter.java   | 18 ------------------
 .../migration/application/V1__init_schema.sql  |  4 ++--
 2 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
index f7b10e4c9..a48ecc22f 100644
--- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
+++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
@@ -51,7 +51,6 @@ import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.SortedSet;
 import java.util.TreeSet;
@@ -287,8 +286,6 @@ abstract class ReferenceImporter {
 
         final ReferenceValue e = new ReferenceValue();
         final Ltree naturalKey = keysAndReferenceDatumAfterChecking.getNaturalKey();
-        recursionStrategy.getKnownId(naturalKey)
-                .ifPresent(e::setId);
         final Ltree hierarchicalReference = recursionStrategy.getHierarchicalReference(naturalKey);
         referenceDatum.putAll(InternationalizationDisplay.getDisplays(referenceImporterContext.getDisplayPattern(), referenceImporterContext.getDisplayColumns(), referenceDatum));
 
@@ -363,8 +360,6 @@ abstract class ReferenceImporter {
 
         Ltree getHierarchicalReference(Ltree naturalKey);
 
-        Optional<UUID> getKnownId(Ltree naturalKey);
-
         Stream<RowWithReferenceDatum> firstPass(Stream<RowWithReferenceDatum> streamBeforePreloading);
 
     }
@@ -375,14 +370,6 @@ abstract class ReferenceImporter {
 
         private final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>();
 
-        @Override
-        public Optional<UUID> getKnownId(Ltree naturalKey) {
-            if (afterPreloadReferenceUuids.containsKey(naturalKey)) {
-                return Optional.of(afterPreloadReferenceUuids.get(naturalKey));
-            }
-            return Optional.empty();
-        }
-
         @Override
         public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) {
             Ltree recursiveNaturalKey = getRecursiveNaturalKey(naturalKey);
@@ -465,11 +452,6 @@ abstract class ReferenceImporter {
 
     private class WithoutRecursion implements RecursionStrategy {
 
-        @Override
-        public Optional<UUID> getKnownId(Ltree naturalKey) {
-            return Optional.empty();
-        }
-
         @Override
         public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) {
             return referenceImporterContext.newHierarchicalKey(naturalKey, referenceDatum);
diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql
index 0a03eec4f..2e0debf6f 100644
--- a/src/main/resources/migration/application/V1__init_schema.sql
+++ b/src/main/resources/migration/application/V1__init_schema.sql
@@ -31,7 +31,7 @@ create table ReferenceValue
 create table Reference_Reference
 (
     referenceId entityid REFERENCES ReferenceValue(id) ON DELETE CASCADE,
-    referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT,
+    referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED,
     CONSTRAINT "Reference_Reference_PK" PRIMARY KEY (referenceId, referencedBy)
 );
 
@@ -95,7 +95,7 @@ create table Data
 create table Data_Reference
 (
     dataId entityid REFERENCES Data(id) ON DELETE CASCADE,
-    referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE RESTRICT,
+    referencedBy entityid REFERENCES ReferenceValue(id) ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED,
     CONSTRAINT "Data_Reference_PK" PRIMARY KEY (dataId, referencedBy)
 );
 
-- 
GitLab


From 0b89a96dfad803edaa670703167fd4e7b21d5545 Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Wed, 16 Mar 2022 17:58:54 +0100
Subject: [PATCH 8/9] Convertit un champs en variable locale

---
 .../java/fr/inra/oresing/rest/ReferenceImporter.java  | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
index a48ecc22f..5c2fd88d5 100644
--- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
+++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
@@ -368,8 +368,6 @@ abstract class ReferenceImporter {
 
         private final Map<Ltree, Ltree> parentReferenceMap = new LinkedHashMap<>();
 
-        private final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>();
-
         @Override
         public Ltree getHierarchicalKey(Ltree naturalKey, ReferenceDatum referenceDatum) {
             Ltree recursiveNaturalKey = getRecursiveNaturalKey(naturalKey);
@@ -404,7 +402,7 @@ abstract class ReferenceImporter {
             final ReferenceColumn columnToLookForParentKey = referenceImporterContext.getColumnToLookForParentKey();
             ReferenceLineChecker referenceLineChecker = referenceImporterContext.getReferenceLineChecker();
             final ImmutableMap<Ltree, UUID> beforePreloadReferenceUuids = referenceLineChecker.getReferenceValues();
-            afterPreloadReferenceUuids.putAll(beforePreloadReferenceUuids);
+            final Map<Ltree, UUID> afterPreloadReferenceUuids = new LinkedHashMap<>(beforePreloadReferenceUuids);
             ListMultimap<Ltree, Integer> missingParentReferences = LinkedListMultimap.create();
             List<RowWithReferenceDatum> collect = streamBeforePreloading
                     .peek(rowWithReferenceDatum -> {
@@ -426,7 +424,8 @@ abstract class ReferenceImporter {
                         missingParentReferences.removeAll(naturalKey);
                     })
                     .collect(Collectors.toList());
-            checkMissingParentReferencesIsEmpty(missingParentReferences);
+            Set<Ltree> knownReferences = afterPreloadReferenceUuids.keySet();
+            checkMissingParentReferencesIsEmpty(missingParentReferences, knownReferences);
             referenceLineChecker.setReferenceValues(ImmutableMap.copyOf(afterPreloadReferenceUuids));
             return collect.stream();
         }
@@ -436,13 +435,13 @@ abstract class ReferenceImporter {
          *
          * @param missingParentReferences pour chaque parent manquant, les lignes du CSV où il est mentionné
          */
-        private void checkMissingParentReferencesIsEmpty(ListMultimap<Ltree, Integer> missingParentReferences) {
+        private void checkMissingParentReferencesIsEmpty(ListMultimap<Ltree, Integer> missingParentReferences, Set<Ltree> knownReferences) {
             List<CsvRowValidationCheckResult> rowErrors = missingParentReferences.entries().stream()
                     .map(entry -> {
                         Ltree missingParentReference = entry.getKey();
                         Integer lineNumber = entry.getValue();
                         ValidationCheckResult validationCheckResult =
-                                new MissingParentLineValidationCheckResult(lineNumber, referenceImporterContext.getRefType(), missingParentReference, afterPreloadReferenceUuids.keySet());
+                                new MissingParentLineValidationCheckResult(lineNumber, referenceImporterContext.getRefType(), missingParentReference, knownReferences);
                         return new CsvRowValidationCheckResult(validationCheckResult, lineNumber);
                     })
                     .collect(Collectors.toUnmodifiableList());
-- 
GitLab


From 0e288b627436c038483486ee2bd13690fdae5632 Mon Sep 17 00:00:00 2001
From: Brendan Le Ny <bleny@codelutin.com>
Date: Wed, 16 Mar 2022 18:27:53 +0100
Subject: [PATCH 9/9] =?UTF-8?q?Gr=C3=A2ce=20au=20contr=C3=B4l=C3=A9=20defe?=
 =?UTF-8?q?rable=20il=20n'est=20plus=20n=C3=A9cessaire=20de=20faire=20l'in?=
 =?UTF-8?q?sertion=20des=20r=C3=A9f=C3=A9rentiels=20dans=20l'ordre?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/main/java/fr/inra/oresing/rest/ReferenceImporter.java | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
index 5c2fd88d5..8ea34b7ef 100644
--- a/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
+++ b/src/main/java/fr/inra/oresing/rest/ReferenceImporter.java
@@ -45,7 +45,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.time.format.DateTimeFormatter;
 import java.util.Collection;
-import java.util.Comparator;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
@@ -116,8 +115,7 @@ abstract class ReferenceImporter {
                         boolean canSave = encounteredHierarchicalKeysForConflictDetection.get(hierarchicalKey).size() == 1;
                         return canSave;
                     })
-                    .map(keysAndReferenceDatumAfterChecking -> toEntity(keysAndReferenceDatumAfterChecking, fileId))
-                    .sorted(Comparator.comparing(a -> a.getHierarchicalKey().getSql()));
+                    .map(keysAndReferenceDatumAfterChecking -> toEntity(keysAndReferenceDatumAfterChecking, fileId));
             storeAll(referenceValuesStream);
         }
 
-- 
GitLab