From a09db4a356ee73186fe093543e526ac80c8d891d Mon Sep 17 00:00:00 2001
From: TCHERNIATINSKY <philippe.tcherniatinsky@inrae.fr>
Date: Wed, 9 Jun 2021 17:00:05 +0200
Subject: [PATCH] =?UTF-8?q?Prise=20en=20compte=20des=20variableComponents?=
=?UTF-8?q?=20dans=20les=20fichiers=20de=20r=C3=A9f=C3=A9rence?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Pour le moment le nom de la colonne associée à un variable_component est variable_component
Ajout d'un refslinkto dans référence (je n'ai pas su écrire le schema version 2)
Il manque le check sur la ligne pour vérifier que la référence existe (quid des référence dans le même fichier)???
Il faut encore gérer les erreurs
- le checker renvoie une erreur (ligne 292)
- aucune colonne ne correpond au variableComponent (ligne 275)
On peut rajouter le séparateur variable<->component
La proposition permet de renseigner l'internationalisation, mais il faudrait pouvoir indiquer que la variable est internationalisable pour l'utiliser
---
.../inra/oresing/checker/CheckerFactory.java | 29 ++-
.../fr/inra/oresing/model/ReferenceValue.java | 2 +
.../persistence/ReferenceValueRepository.java | 2 +-
.../fr/inra/oresing/rest/OreSiService.java | 134 +++++++------
.../migration/application/V1__init_schema.sql | 1 +
.../java/fr/inra/oresing/rest/Fixtures.java | 8 +-
.../inra/oresing/rest/OreSiResourcesTest.java | 2 +-
src/test/resources/data/monsore/monsore.yaml | 178 ++++++++++++------
..._donnees_par_themes_de_sites_et_projet.csv | 2 +-
...riables_et_unites_par_types_de_donnees.csv | 2 +-
10 files changed, 232 insertions(+), 128 deletions(-)
diff --git a/src/main/java/fr/inra/oresing/checker/CheckerFactory.java b/src/main/java/fr/inra/oresing/checker/CheckerFactory.java
index b0e55013d..b1bdd1a5d 100644
--- a/src/main/java/fr/inra/oresing/checker/CheckerFactory.java
+++ b/src/main/java/fr/inra/oresing/checker/CheckerFactory.java
@@ -12,6 +12,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
+import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
import java.util.function.Function;
@@ -24,19 +25,35 @@ public class CheckerFactory {
private OreSiRepository repository;
public ImmutableMap<VariableComponentKey, ReferenceLineChecker> getReferenceLineCheckers(Application app, String dataType) {
- return getLineCheckers(app, dataType).stream()
+ return getLineCheckersInDatatype(app, dataType).stream()
.filter(lineChecker -> lineChecker instanceof ReferenceLineChecker)
.map(lineChecker -> (ReferenceLineChecker) lineChecker)
.collect(ImmutableMap.toImmutableMap(ReferenceLineChecker::getVariableComponentKey, Function.identity()));
}
-
- public ImmutableSet<LineChecker> getLineCheckers(Application app, String dataType) {
+ public ImmutableSet<LineChecker> getLineCheckersInDatatype(Application app, String dataType) {
Preconditions.checkArgument(app.getConfiguration().getDataTypes().containsKey(dataType), "Pas de type de données " + dataType + " dans " + app);
Configuration.DataTypeDescription dataTypeDescription = app.getConfiguration().getDataTypes().get(dataType);
+ LinkedHashMap<String, Configuration.ColumnDescription> data = dataTypeDescription.getData();
+ LinkedHashMap<String, Configuration.LineValidationRuleDescription> validations = dataTypeDescription.getValidations();
+ return getLineCheckers(app, dataType, data, validations);
+ }
+
+ public ImmutableSet<LineChecker> getLineCheckersInReference(Application app, String reference) {
+ Preconditions.checkArgument(app.getConfiguration().getReferences().containsKey(reference), "Pas de type de données " + reference + " dans " + app);
+ Configuration.ReferenceDescription referenceDescription = app.getConfiguration().getReferences().get(reference);
+ LinkedHashMap<String, Configuration.ColumnDescription> data = referenceDescription.getColumns();
+ LinkedHashMap<String, Configuration.LineValidationRuleDescription> validations = new LinkedHashMap<>();//referenceDescription.getValidations();
+ return getLineCheckers(app, reference, data, validations);
+ }
+
+ protected ImmutableSet<LineChecker> getLineCheckers(Application app, String name, LinkedHashMap<String, Configuration.ColumnDescription> data, LinkedHashMap<String, Configuration.LineValidationRuleDescription> validations) {
ImmutableSet.Builder<LineChecker> checkersBuilder = ImmutableSet.builder();
- for (Map.Entry<String, Configuration.ColumnDescription> variableEntry : dataTypeDescription.getData().entrySet()) {
+ for (Map.Entry<String, Configuration.ColumnDescription> variableEntry : data.entrySet()) {
String variable = variableEntry.getKey();
Configuration.ColumnDescription variableDescription = variableEntry.getValue();
+ if(variableDescription==null){
+ continue;
+ }
for (Map.Entry<String, Configuration.VariableComponentDescription> componentEntry : variableDescription.getComponents().entrySet()) {
String component = componentEntry.getKey();
VariableComponentKey variableComponentKey = new VariableComponentKey(variable, component);
@@ -70,7 +87,7 @@ public class CheckerFactory {
}
}
}
- for (Map.Entry<String, Configuration.LineValidationRuleDescription> validationEntry : dataTypeDescription.getValidations().entrySet()) {
+ for (Map.Entry<String, Configuration.LineValidationRuleDescription> validationEntry : validations.entrySet()) {
Configuration.LineValidationRuleDescription lineValidationRuleDescription = validationEntry.getValue();
Configuration.CheckerDescription checkerDescription = lineValidationRuleDescription.getChecker();
LineChecker lineChecker;
@@ -84,7 +101,7 @@ public class CheckerFactory {
}
ImmutableSet<LineChecker> lineCheckers = checkersBuilder.build();
if (log.isTraceEnabled()) {
- log.trace("pour " + app.getName() + ", " + dataType + ", on validera avec " + lineCheckers);
+ log.trace("pour " + app.getName() + ", " + name + ", on validera avec " + lineCheckers);
}
return lineCheckers;
}
diff --git a/src/main/java/fr/inra/oresing/model/ReferenceValue.java b/src/main/java/fr/inra/oresing/model/ReferenceValue.java
index 72a272cf3..f5970ba46 100644
--- a/src/main/java/fr/inra/oresing/model/ReferenceValue.java
+++ b/src/main/java/fr/inra/oresing/model/ReferenceValue.java
@@ -4,6 +4,7 @@ import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
+import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -13,6 +14,7 @@ import java.util.UUID;
public class ReferenceValue extends OreSiEntity {
private UUID application;
private String referenceType;
+ private List<UUID> refsLinkedTo;
public void setCompositeKey(String compositeKey) {
this.compositeKey = compositeKey;
diff --git a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
index 9474fb7e1..c02a5d4cc 100644
--- a/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
+++ b/src/main/java/fr/inra/oresing/persistence/ReferenceValueRepository.java
@@ -31,7 +31,7 @@ public class ReferenceValueRepository extends JsonTableInApplicationSchemaReposi
@Override
protected String getUpsertQuery() {
- return "INSERT INTO " + getTable().getSqlIdentifier() + "(id, application, referenceType, compositeKey, refValues, binaryFile) SELECT id, application, referenceType, compositeKey, refValues, binaryFile FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", :json::json) "
+ return "INSERT INTO " + getTable().getSqlIdentifier() + "(id, application, referenceType, compositeKey, refslinkedto, refValues, binaryFile) SELECT id, application, referenceType, compositeKey, refslinkedto, refValues, binaryFile FROM json_populate_recordset(NULL::" + getTable().getSqlIdentifier() + ", :json::json) "
+ " ON CONFLICT (id) DO UPDATE SET updateDate=current_timestamp, application=EXCLUDED.application, referenceType=EXCLUDED.referenceType, compositeKey=EXCLUDED.compositeKey, refValues=EXCLUDED.refValues, binaryFile=EXCLUDED.binaryFile"
+ " RETURNING id";
}
diff --git a/src/main/java/fr/inra/oresing/rest/OreSiService.java b/src/main/java/fr/inra/oresing/rest/OreSiService.java
index e6b73e2d1..288390edb 100644
--- a/src/main/java/fr/inra/oresing/rest/OreSiService.java
+++ b/src/main/java/fr/inra/oresing/rest/OreSiService.java
@@ -1,42 +1,12 @@
package fr.inra.oresing.rest;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Splitter;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSetMultimap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Maps;
-import com.google.common.collect.MoreCollectors;
-import com.google.common.collect.Ordering;
+import com.google.common.base.*;
+import com.google.common.collect.*;
import com.google.common.primitives.Ints;
import fr.inra.oresing.OreSiTechnicalException;
-import fr.inra.oresing.checker.CheckerException;
-import fr.inra.oresing.checker.CheckerFactory;
-import fr.inra.oresing.checker.DateLineChecker;
-import fr.inra.oresing.checker.LineChecker;
-import fr.inra.oresing.checker.ReferenceLineChecker;
-import fr.inra.oresing.checker.ReferenceValidationCheckResult;
-import fr.inra.oresing.model.Application;
-import fr.inra.oresing.model.BinaryFile;
-import fr.inra.oresing.model.Configuration;
-import fr.inra.oresing.model.Data;
-import fr.inra.oresing.model.LocalDateTimeRange;
-import fr.inra.oresing.model.ReferenceValue;
-import fr.inra.oresing.model.VariableComponentKey;
-import fr.inra.oresing.persistence.AuthenticationService;
-import fr.inra.oresing.persistence.DataRepository;
-import fr.inra.oresing.persistence.OreSiRepository;
-import fr.inra.oresing.persistence.ReferenceValueRepository;
-import fr.inra.oresing.persistence.SqlPolicy;
-import fr.inra.oresing.persistence.SqlSchema;
-import fr.inra.oresing.persistence.SqlSchemaForApplication;
-import fr.inra.oresing.persistence.SqlService;
+import fr.inra.oresing.checker.*;
+import fr.inra.oresing.model.*;
+import fr.inra.oresing.persistence.*;
import fr.inra.oresing.persistence.roles.OreSiRightOnApplicationRole;
import fr.inra.oresing.persistence.roles.OreSiUserRole;
import lombok.Value;
@@ -60,17 +30,8 @@ import javax.sql.DataSource;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.regex.Matcher;
@@ -85,7 +46,7 @@ public class OreSiService {
/**
* Déliminateur entre les différents niveaux d'un ltree postgresql.
- *
+ * <p>
* https://www.postgresql.org/docs/current/ltree.html
*/
private static final String LTREE_SEPARATOR = ".";
@@ -260,7 +221,7 @@ public class OreSiService {
}
private void validateStoredData(Application app, String dataType) {
- ImmutableSet<LineChecker> lineCheckers = checkerFactory.getLineCheckers(app, dataType);
+ ImmutableSet<LineChecker> lineCheckers = checkerFactory.getLineCheckersInDatatype(app, dataType);
Consumer<ImmutableMap<VariableComponentKey, String>> validateRow = line -> {
lineCheckers.forEach(lineChecker -> {
ValidationCheckResult validationCheckResult = lineChecker.check(line);
@@ -293,7 +254,7 @@ public class OreSiService {
Configuration.ReferenceDescription ref = conf.getReferences().get(refType);
ReferenceValueRepository referenceValueRepository = repo.getRepository(app).referenceValue();
-
+ ImmutableMap<String, VariableComponentKey> variableComponentsByColumnName = getVariableComponentsByColumnName(ref);
CSVFormat csvFormat = CSVFormat.DEFAULT
.withDelimiter(ref.getSeparator())
.withSkipHeaderRecord();
@@ -301,10 +262,38 @@ public class OreSiService {
CSVParser csvParser = CSVParser.parse(csv, Charsets.UTF_8, csvFormat);
Iterator<CSVRecord> linesIterator = csvParser.iterator();
CSVRecord headerRow = linesIterator.next();
+ List<UUID> refsLinkedTo = new ArrayList<>();
ImmutableList<String> columns = Streams.stream(headerRow).collect(ImmutableList.toImmutableList());
Function<CSVRecord, Map<String, String>> csvRecordToLineAsMapFn = line -> {
Iterator<String> currentHeader = columns.iterator();
Map<String, String> recordAsMap = new LinkedHashMap<>();
+ ImmutableSet<LineChecker> lineCheckers = checkerFactory.getLineCheckersInReference(app, refType);
+ ImmutableMap.Builder<VariableComponentKey, String> variableComponentValues = ImmutableMap.builder();
+ variableComponentsByColumnName.entrySet().stream()
+ .forEach(e-> {
+ if(columns.indexOf(e.getKey())<0){
+ log.error("y'a une erreur");//la colonne e;get.getKey() n'existe pas dans la donnnée de référence refType
+ }else {
+ String value = line.get(columns.indexOf(e.getKey()));
+ variableComponentValues.put(e.getValue(), value);
+ }
+ });
+ ImmutableMap<VariableComponentKey, String> values = variableComponentValues.build();
+
+ if(!values.isEmpty()) {
+ lineCheckers.forEach(lineChecker -> {
+ ValidationCheckResult validationCheckResult = lineChecker.check(values);
+ if (validationCheckResult.isSuccess()) {
+ if (validationCheckResult instanceof ReferenceValidationCheckResult) {
+ UUID referenceId = ((ReferenceValidationCheckResult) validationCheckResult).getReferenceId();
+ refsLinkedTo.add(referenceId);
+ }
+ }/* else {
+ //erreur de validation du checker
+ errors.add(new CsvRowValidationCheckResult(validationCheckResult, rowWithData.getLineNumber()));
+ }*/
+ });
+ }
line.forEach(value -> {
String header = currentHeader.next();
recordAsMap.put(header, value);
@@ -326,6 +315,7 @@ public class OreSiService {
checkCompositeKey(key);
e.setBinaryFile(fileId);
e.setReferenceType(refType);
+ e.setRefsLinkedTo(refsLinkedTo);
e.setCompositeKey(key);
e.setApplication(app.getId());
e.setRefValues(refValues);
@@ -342,6 +332,29 @@ public class OreSiService {
return fileId;
}
+ private ImmutableMap<String, VariableComponentKey> getVariableComponentsByColumnName(Configuration.ReferenceDescription ref){
+ ImmutableMap.Builder<String, VariableComponentKey> variableComponentsBuilder = ImmutableMap.builder();
+ for (Map.Entry<String, Configuration.ColumnDescription> variableEntry : ref.getColumns().entrySet()) {
+ String variable = variableEntry.getKey();
+ Configuration.ColumnDescription variableDescription = variableEntry.getValue();
+ if (variableDescription == null) {
+ continue;
+ }
+ for (Map.Entry<String, Configuration.VariableComponentDescription> componentEntry : variableDescription.getComponents().entrySet()) {
+ String component = componentEntry.getKey();
+ VariableComponentKey variableComponentKey = new VariableComponentKey(variable, component);
+ String defaultValue;
+ if (componentEntry.getValue() == null) {
+ defaultValue = null;
+ } else {
+ defaultValue = componentEntry.getValue().getDefaultValue();
+ }
+ variableComponentsBuilder.put(variable + "_" + component, variableComponentKey);
+ }
+ }
+ return variableComponentsBuilder.build();
+ }
+
private void updateCompositeReference(Application application, String compositeReference) {
ReferenceValueRepository referenceValueRepository = repo.getRepository(application).referenceValue();
Configuration.CompositeReferenceDescription compositeReferenceDescription = application.getConfiguration().getCompositeReferences().get(compositeReference);
@@ -407,6 +420,7 @@ public class OreSiService {
/**
* Add a new datatype from csv file
+ *
* @param nameOrId
* @param dataType
* @param file
@@ -415,7 +429,7 @@ public class OreSiService {
* @throws CheckerException
*/
public List<CsvRowValidationCheckResult> addData(String nameOrId, String dataType, MultipartFile file) throws IOException, CheckerException {
- List<CsvRowValidationCheckResult> errors= new LinkedList<>();
+ List<CsvRowValidationCheckResult> errors = new LinkedList<>();
authenticationService.setRoleForClient();
Application app = getApplication(nameOrId);
@@ -426,7 +440,6 @@ public class OreSiService {
Configuration.FormatDescription formatDescription = dataTypeDescription.getFormat();
-
CSVParser csvParser = buildCSVParserForFile(file, formatDescription);
Iterator<CSVRecord> linesIterator = csvParser.iterator();
@@ -452,13 +465,14 @@ public class OreSiService {
/**
* return a function that transform each RowWithData to a stream of data entities
+ *
* @param app
* @param dataType
* @param fileId
* @return
*/
- private Function<RowWithData, Stream<Data>> buildLineValuesToEntityStreamFn(Application app, String dataType, UUID fileId, List<CsvRowValidationCheckResult> errors){
- ImmutableSet<LineChecker> lineCheckers = checkerFactory.getLineCheckers(app, dataType);
+ private Function<RowWithData, Stream<Data>> buildLineValuesToEntityStreamFn(Application app, String dataType, UUID fileId, List<CsvRowValidationCheckResult> errors) {
+ ImmutableSet<LineChecker> lineCheckers = checkerFactory.getLineCheckersInDatatype(app, dataType);
Configuration conf = app.getConfiguration();
Configuration.DataTypeDescription dataTypeDescription = conf.getDataTypes().get(dataType);
@@ -474,6 +488,7 @@ public class OreSiService {
/**
* build the function that transform each RowWithData to a stream of data entities
+ *
* @param app
* @param dataType
* @param fileId
@@ -547,7 +562,7 @@ public class OreSiService {
* @param defaultValues
* @return
*/
- private Function<RowWithData, RowWithData> buildReplaceMissingValuesByDefaultValuesFn(ImmutableMap<VariableComponentKey, String> defaultValues){
+ private Function<RowWithData, RowWithData> buildReplaceMissingValuesByDefaultValuesFn(ImmutableMap<VariableComponentKey, String> defaultValues) {
return rowWithData -> {
Maps.EntryTransformer<VariableComponentKey, String, String> replaceEmptyByDefaultValueFn =
(variableComponentKey, value) -> StringUtils.isBlank(value) ? defaultValues.get(variableComponentKey) : value;
@@ -559,10 +574,11 @@ public class OreSiService {
/**
* Return a function that add constantValues to rowWithdata
+ *
* @param constantValues
* @return
*/
- private Function<RowWithData, RowWithData> fillConstantValuesAndBuildMergeLineValuesAndConstantValuesFn(Map<VariableComponentKey, String> constantValues){
+ private Function<RowWithData, RowWithData> fillConstantValuesAndBuildMergeLineValuesAndConstantValuesFn(Map<VariableComponentKey, String> constantValues) {
return rowWithData -> {
ImmutableMap<VariableComponentKey, String> datum = ImmutableMap.<VariableComponentKey, String>builder()
.putAll(constantValues)
@@ -608,7 +624,7 @@ public class OreSiService {
* @return
*/
private Function<ParsedCsvRow, ImmutableSet<RowWithData>> buildLineAsMapWhenRepeatedColumnsToRecordsFn(Configuration.FormatDescription formatDescription) {
- return parsedCsvRow -> {
+ return parsedCsvRow -> {
List<Map.Entry<String, String>> line = parsedCsvRow.getColumns();
LinkedList<Map.Entry<String, String>> lineCopy = new LinkedList<>(line);
@@ -772,8 +788,7 @@ public class OreSiService {
* build a csvParser from file
*
* @param file
- * @param formatDesc
- * ription
+ * @param formatDesc ription
* @return
* @throws IOException
*/
@@ -907,10 +922,9 @@ public class OreSiService {
}
/**
- *
* @param nameOrId l'id de l'application
- * @param refType le type du referenciel
- * @param params les parametres query de la requete http. 'ANY' est utiliser pour dire n'importe quelle colonne
+ * @param refType le type du referenciel
+ * @param params les parametres query de la requete http. 'ANY' est utiliser pour dire n'importe quelle colonne
* @return la liste qui satisfont aux criteres
*/
public List<ReferenceValue> findReference(String nameOrId, String refType, MultiValueMap<String, String> params) {
diff --git a/src/main/resources/migration/application/V1__init_schema.sql b/src/main/resources/migration/application/V1__init_schema.sql
index 93876e5f8..fe3a5a5d6 100644
--- a/src/main/resources/migration/application/V1__init_schema.sql
+++ b/src/main/resources/migration/application/V1__init_schema.sql
@@ -14,6 +14,7 @@ create table ReferenceValue (
updateDate DateOrNow,
application EntityRef REFERENCES Application(id),
referenceType TEXT CHECK(name_check(application, 'referenceType', referenceType)),
+ refsLinkedTo ListEntityRef, --CHECK(refs_check('${applicationSchema}', application, refsLinkedTo)),
compositeKey ltree NOT NULL,
refValues jsonb,
binaryFile EntityRef REFERENCES BinaryFile(id)
diff --git a/src/test/java/fr/inra/oresing/rest/Fixtures.java b/src/test/java/fr/inra/oresing/rest/Fixtures.java
index 3eb3376d0..186d196a1 100644
--- a/src/test/java/fr/inra/oresing/rest/Fixtures.java
+++ b/src/test/java/fr/inra/oresing/rest/Fixtures.java
@@ -68,17 +68,17 @@ public class Fixtures {
public Map<String, String> getMonsoreReferentielFiles() {
Map<String, String> referentielFiles = new HashMap<>();
- referentielFiles.put("especes", "/data/monsore/refdatas/especes.csv");
referentielFiles.put("projet", "/data/monsore/refdatas/projet.csv");
+ referentielFiles.put("type_de_sites", "/data/monsore/refdatas/type_de_sites.csv");
referentielFiles.put("sites", "/data/monsore/refdatas/sites.csv");
+ referentielFiles.put("especes", "/data/monsore/refdatas/especes.csv");
referentielFiles.put("themes", "/data/monsore/refdatas/themes.csv");
- referentielFiles.put("type de fichiers", "/data/monsore/refdatas/type_de_fichiers.csv");
- referentielFiles.put("type_de_sites", "/data/monsore/refdatas/type_de_sites.csv");
referentielFiles.put("types_de_donnees_par_themes_de_sites_et_projet", "/data/monsore/refdatas/types_de_donnees_par_themes_de_sites_et_projet.csv");
referentielFiles.put("unites", "/data/monsore/refdatas/unites.csv");
- referentielFiles.put("valeurs_qualitatives", "/data/monsore/refdatas/valeurs_qualitatives.csv");
referentielFiles.put("variables", "/data/monsore/refdatas/variables.csv");
referentielFiles.put("variables_et_unites_par_types_de_donnees", "/data/monsore/refdatas/variables_et_unites_par_types_de_donnees.csv");
+ referentielFiles.put("valeurs_qualitatives", "/data/monsore/refdatas/valeurs_qualitatives.csv");
+ referentielFiles.put("type de fichiers", "/data/monsore/refdatas/type_de_fichiers.csv");
return referentielFiles;
}
diff --git a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
index dbc430eda..dfbee22df 100644
--- a/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
+++ b/src/test/java/fr/inra/oresing/rest/OreSiResourcesTest.java
@@ -122,7 +122,7 @@ public class OreSiResourcesTest {
Date now = new Date();
Assert.assertEquals("monsore", app2.getName());
- Assert.assertEquals(List.of("especes", "projet", "sites", "themes", "type de fichiers", "type_de_sites", "types_de_donnees_par_themes_de_sites_et_projet", "unites", "valeurs_qualitatives", "variables", "variables_et_unites_par_types_de_donnees"), app2.getReferenceType());
+ Assert.assertEquals(List.of("type_de_sites", "especes", "projet", "sites", "themes", "type de fichiers", "types_de_donnees_par_themes_de_sites_et_projet", "unites", "valeurs_qualitatives", "variables", "variables_et_unites_par_types_de_donnees"), app2.getReferenceType());
Assert.assertEquals(List.of("pem"), app2.getDataType());
// Ajout de referentiel
diff --git a/src/test/resources/data/monsore/monsore.yaml b/src/test/resources/data/monsore/monsore.yaml
index 9d86f26ff..d0f23f566 100644
--- a/src/test/resources/data/monsore/monsore.yaml
+++ b/src/test/resources/data/monsore/monsore.yaml
@@ -3,104 +3,174 @@ application:
name: MONSORE
version: 1
references:
+
+ type_de_sites:
+ keyColumns: [tze_nom_key]
+ columns:
+ tze_nom:
+ components:
+ key:
+ fr:
+ en:
+ tze_definition:
+ components:
+ fr:
+ en:
+
especes:
keyColumns: [esp_nom]
columns:
esp_nom:
- esp_definition_fr:
- esp_definition_en:
+ esp_definition:
+ components:
+ fr:
+ en:
colonne_homonyme_entre_referentiels:
projet:
keyColumns: [nom_key]
columns:
- nom_key:
- nom_fr:
- nom_en:
- definition_fr:
- definition_en:
+ nom:
+ components:
+ key:
+ fr:
+ en:
+ definition:
+ components:
+ fr:
+ en:
colonne_homonyme_entre_referentiels:
sites:
keyColumns: [chemin]
columns:
- tze_type_nom:
- zet_nom_key:
- zet_nom_fr:
- zet_nom_en:
- zet_description_fr:
- zet_description_en:
+ tze_type:
+ components:
+ nom:
+ checker:
+ name: Reference
+ params:
+ refType: type_de_sites
+ zet_nom:
+ components:
+ key:
+ fr:
+ en:
+ zet_description:
+ components:
+ fr:
+ en:
zet_chemin_parent:
chemin:
themes:
keyColumns: [nom_key]
columns:
- nom_key:
- nom_fr:
- nom_en:
- description_fr:
+ nom:
+ components:
+ key:
+ fr:
+ en:
+ description:
+ components:
+ fr:
+ en:
description_en:
type de fichiers:
keyColumns: [nom_key]
columns:
- nom_key:
- nom_fr:
- nom_en:
- description_fr:
+ nom:
+ components:
+ key:
+ fr:
+ en:
+ description:
+ components:
+ fr:
+ en:
description_en:
- type_de_sites:
- keyColumns: [tze_nom_key]
- columns:
- tze_nom_key:
- tze_nom_fr:
- tze_nom_en:
- tze_definition_fr:
- tze_definition_en:
-
types_de_donnees_par_themes_de_sites_et_projet:
columns:
- nom du projet:
- nom du site:
- nom du thème:
- nom du type de données:
+ nom du:
+ components:
+ projet:
+ checker:
+ name: Reference
+ params:
+ refType: projet
+ site:
+ checker:
+ name: Reference
+ params:
+ refType: sites
+ thème:
+ checker:
+ name: Reference
+ params:
+ refType: themes
+ type de données:
+ checker:
+ name: Reference
+ params:
+ refType: themes
unites:
columns:
- code_key:
- code_fr:
- code_en:
- nom_key:
- nom_fr:
- nom_en:
+ code:
+ components:
+ key:
+ fr:
+ en:
+ nom:
+ components:
+ key:
+ fr:
+ en:
valeurs_qualitatives:
keyColumns: [nom_key,valeur_key]
columns:
- nom_key:
- nom_fr:
- nom_en:
- valeur_key:
- valeur_fr:
- valeur_en:
+ nom:
+ components:
+ key:
+ fr:
+ en:
+ valeur:
+ components:
+ key:
+ fr:
+ en:
variables:
keyColumns: [nom_key]
columns:
- nom_key:
- nom_fr:
- nom_en:
- definition_fr:
- definition_en:
- isQualitative:
+ nom:
+ components:
+ key:
+ fr:
+ en:
+ definition:
+ components:
+ fr:
+ en:
variables_et_unites_par_types_de_donnees:
columns:
nom du type de données:
- nom de la variable:
- nom de l'unité:
+ nom:
+ components:
+ de la variable:
+ checker:
+ name: Reference
+ params:
+ refType: variables
+ de l'unité:
+ checker:
+ name: Reference
+ params:
+ refType: unites
dataTypes:
pem:
diff --git a/src/test/resources/data/monsore/refdatas/types_de_donnees_par_themes_de_sites_et_projet.csv b/src/test/resources/data/monsore/refdatas/types_de_donnees_par_themes_de_sites_et_projet.csv
index 3085ec604..993ec1cda 100644
--- a/src/test/resources/data/monsore/refdatas/types_de_donnees_par_themes_de_sites_et_projet.csv
+++ b/src/test/resources/data/monsore/refdatas/types_de_donnees_par_themes_de_sites_et_projet.csv
@@ -1,4 +1,4 @@
-nom du projet;nom du site;nom du thème;nom du type de données
+nom du_projet;nom du_site;nom du_thème;nom du_type de données
projet_manche;oir/p1;donnees_biologiques;piegeage_en_montee
projet_manche;oir/p2;donnees_biologiques;piegeage_en_montee
projet_manche;nivelle/p1;donnees_biologiques;piegeage_en_montee
diff --git a/src/test/resources/data/monsore/refdatas/variables_et_unites_par_types_de_donnees.csv b/src/test/resources/data/monsore/refdatas/variables_et_unites_par_types_de_donnees.csv
index de8b5190c..d674b2deb 100644
--- a/src/test/resources/data/monsore/refdatas/variables_et_unites_par_types_de_donnees.csv
+++ b/src/test/resources/data/monsore/refdatas/variables_et_unites_par_types_de_donnees.csv
@@ -1,3 +1,3 @@
-nom du type de données;nom de la variable;nom de l'unité
+nom du type de données;nom_de la variable;nom_de l'unité
piegeage_en_montee;nombre_d_individus;*
piegeage_en_montee;couleur_des_individus;*
--
GitLab