Skip to content

Commit

Permalink
Merge pull request #200 from opendatamesh-initiative/192-add-policy-i…
Browse files Browse the repository at this point in the history 
…d-to-error-message-for-non-compliant-product-versions

 192-add-policy-id-to-error-message-for-non-compliant-product-versions
  • Loading branch information
@gabrielefantini
gabrielefantini authored Dec 10, 2024
2 parents d0b57a9 + aee954e commit 7e5f7f0
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 110 deletions.
20 changes: 3 additions & 17 deletions ...c/main/java/org/opendatamesh/platform/pp/registry/server/services/DataProductService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,11 +97,7 @@ public DataProduct createDataProduct(DataProduct dataProduct) {
}
dataProduct.setId(uuid);

if (!registryPolicyServiceProxy.isCompliantWithPolicies(dataProductMapper.toResource(dataProduct))) {
throw new UnprocessableEntityException(
RegistryApiStandardErrors.SC422_03_DESCRIPTOR_NOT_COMPLIANT,
"The data product is not compliant to one or more global policies");
}
registryPolicyServiceProxy.validateDataProduct(dataProductMapper.toResource(dataProduct));

if (loadDataProduct(uuid) != null) {
throw new UnprocessableEntityException(
Expand Down Expand Up @@ -279,11 +275,7 @@ public DataProduct updateDataProduct(DataProduct dataProduct) {
"Data product [" + dataProduct.getFullyQualifiedName() + "] with id [" + dataProduct.getId() + "] doesn't exists");
}

if (!registryPolicyServiceProxy.isCompliantWithPolicies(dataProductMapper.toResource(oldDataProduct), dataProductMapper.toResource(dataProduct))) {
throw new UnprocessableEntityException(
RegistryApiStandardErrors.SC422_03_DESCRIPTOR_NOT_COMPLIANT,
"The data product is not compliant to one or more global policies");
}
registryPolicyServiceProxy.validateDataProduct(dataProductMapper.toResource(oldDataProduct), dataProductMapper.toResource(dataProduct));

dataProduct.setId(oldDataProduct.getId());
dataProduct.setFullyQualifiedName(oldDataProduct.getFullyQualifiedName());
Expand Down Expand Up @@ -406,13 +398,7 @@ private DataProductVersion addDataProductVersion (
ODMApiCommonErrors.SC500_00_SERVICE_ERROR,
"Data product version object cannot be null");
}

if(!dataProductVersionService.isCompliantWithGlobalPolicies(dataProductVersion)) {
throw new UnprocessableEntityException(
RegistryApiStandardErrors.SC422_03_DESCRIPTOR_NOT_COMPLIANT,
"Data product descriptor is not compliant with global policies");
}

dataProductVersionService.checkGlobalPoliciesCompliance(dataProductVersion);
DataProduct dataProduct = null;
String dataProductId = dataProductVersion.getInfo().getDataProductId();
if(dataProductExists(dataProductId)) {
Expand Down
10 changes: 4 additions & 6 deletions ...java/org/opendatamesh/platform/pp/registry/server/services/DataProductVersionService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,8 @@ protected DataProductVersion createDataProductVersion(
}

// TODO check schemas evolution rules
if (checkGlobalPolicies && !isCompliantWithGlobalPolicies(dataProductVersion)) {
throw new UnprocessableEntityException(
RegistryApiStandardErrors.SC422_03_DESCRIPTOR_NOT_COMPLIANT,
"The data product descriptor is not compliant to one or more global policies");
if (checkGlobalPolicies){
checkGlobalPoliciesCompliance(dataProductVersion);
}

try {
Expand Down Expand Up @@ -567,10 +565,10 @@ public String replaceVariables(
// OTHER
// ======================================================================================

public boolean isCompliantWithGlobalPolicies(DataProductVersion dataProductVersion) {
public void checkGlobalPoliciesCompliance(DataProductVersion dataProductVersion) {
DataProductVersionDPDS newDataProductVersionDPDS = dataProductVersionMapper.toResource(dataProductVersion);
DataProductVersionDPDS mostRecentDataProduct = getMostRecentDataProductVersion(dataProductVersion);
return policyServiceProxy.validateDataProductVersion(mostRecentDataProduct, newDataProductVersionDPDS);
policyServiceProxy.validateDataProductVersion(mostRecentDataProduct, newDataProductVersionDPDS);
}

private DataProductVersionDPDS getMostRecentDataProductVersion(DataProductVersion dataProductVersion) {
Expand Down
156 changes: 82 additions & 74 deletions ...opendatamesh/platform/pp/registry/server/services/proxies/RegistryPolicyServiceProxy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,22 @@
import org.opendatamesh.platform.core.commons.ObjectMapperFactory;
import org.opendatamesh.platform.core.commons.servers.exceptions.BadGatewayException;
import org.opendatamesh.platform.core.commons.servers.exceptions.ODMApiCommonErrors;
import org.opendatamesh.platform.core.commons.servers.exceptions.UnprocessableEntityException;
import org.opendatamesh.platform.pp.policy.api.clients.PolicyClientImpl;
import org.opendatamesh.platform.pp.policy.api.clients.PolicyValidationClient;
import org.opendatamesh.platform.pp.policy.api.mappers.utils.JsonNodeUtils;
import org.opendatamesh.platform.pp.policy.api.resources.PolicyEvaluationRequestResource;
import org.opendatamesh.platform.pp.policy.api.resources.PolicyEvaluationResultResource;
import org.opendatamesh.platform.pp.policy.api.resources.ValidationResponseResource;
import org.opendatamesh.platform.pp.registry.api.resources.DataProductResource;
import org.opendatamesh.platform.pp.registry.api.resources.RegistryApiStandardErrors;
import org.opendatamesh.platform.pp.registry.server.database.mappers.EventTypeMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;

@Service
public class RegistryPolicyServiceProxy {
Expand Down Expand Up @@ -44,94 +48,85 @@ public RegistryPolicyServiceProxy(
}
}

public boolean validateDataProductVersion(DataProductVersionDPDS oldDpds, DataProductVersionDPDS newDpds) {
public void validateDataProductVersion(DataProductVersionDPDS oldDpds, DataProductVersionDPDS newDpds) {
if (!this.policyServiceActive) {
logger.info("Policy Service is not active;");
return true;
return;
}
try {
PolicyEvaluationRequestResource evaluationRequest = new PolicyEvaluationRequestResource();
evaluationRequest.setResourceType(PolicyEvaluationRequestResource.ResourceType.DATA_PRODUCT_DESCRIPTOR);
evaluationRequest.setAfterState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(newDpds)));
evaluationRequest.setDataProductId(newDpds.getInfo().getDataProductId());
evaluationRequest.setDataProductVersion(newDpds.getInfo().getVersionNumber());
evaluationRequest.setEvent(PolicyEvaluationRequestResource.EventType.DATA_PRODUCT_VERSION_CREATION);
if (oldDpds != null) {
evaluationRequest.setCurrentState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(oldDpds)));
}
ValidationResponseResource evaluationResult = policyValidationClient.validateInputObject(evaluationRequest);

if (Boolean.FALSE.equals(evaluationResult.getResult())) {
logger.warn("Policy evaluation failed during DataProduct version creation");
}

return evaluationResult.getResult();
} catch (Exception e) {
throw new BadGatewayException(
ODMApiCommonErrors.SC502_71_POLICY_SERVICE_ERROR,
"An error occurred while invoking policy service to validate data product version: " + e.getMessage(),
e
);

PolicyEvaluationRequestResource evaluationRequest = new PolicyEvaluationRequestResource();
evaluationRequest.setResourceType(PolicyEvaluationRequestResource.ResourceType.DATA_PRODUCT_DESCRIPTOR);
evaluationRequest.setAfterState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(newDpds)));
evaluationRequest.setDataProductId(newDpds.getInfo().getDataProductId());
evaluationRequest.setDataProductVersion(newDpds.getInfo().getVersionNumber());
evaluationRequest.setEvent(PolicyEvaluationRequestResource.EventType.DATA_PRODUCT_VERSION_CREATION);
if (oldDpds != null) {
evaluationRequest.setCurrentState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(oldDpds)));
}
ValidationResponseResource evaluationResult = validateInputObject(evaluationRequest);
checkPoliciesValidationResults(evaluationResult);
}

public boolean isCompliantWithPolicies(DataProductResource dataProduct) {
public void validateDataProduct(DataProductResource dataProduct) {
if (!this.policyServiceActive) {
logger.info("Policy Service is not active;");
return true;
}
try {
PolicyEvaluationRequestResource evaluationRequest = new PolicyEvaluationRequestResource();
evaluationRequest.setResourceType(PolicyEvaluationRequestResource.ResourceType.DATA_PRODUCT_DESCRIPTOR);
DataProductVersionDPDS dpdsHead = descriptorFromDataProduct(dataProduct);

evaluationRequest.setAfterState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(dpdsHead)));
evaluationRequest.setDataProductId(dataProduct.getId());
evaluationRequest.setEvent(PolicyEvaluationRequestResource.EventType.DATA_PRODUCT_CREATION);
ValidationResponseResource evaluationResult = policyValidationClient.validateInputObject(evaluationRequest);

if (Boolean.FALSE.equals(evaluationResult.getResult())) {
logger.warn("Policy evaluation failed during DataProduct creation");
}

return evaluationResult.getResult();
} catch (Exception e) {
throw new BadGatewayException(
ODMApiCommonErrors.SC502_71_POLICY_SERVICE_ERROR,
"An error occurred while invoking policy service to validate data product: " + e.getMessage(),
e
);
return;
}
PolicyEvaluationRequestResource evaluationRequest = new PolicyEvaluationRequestResource();
evaluationRequest.setResourceType(PolicyEvaluationRequestResource.ResourceType.DATA_PRODUCT_DESCRIPTOR);
DataProductVersionDPDS dpdsHead = descriptorFromDataProduct(dataProduct);

evaluationRequest.setAfterState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(dpdsHead)));
evaluationRequest.setDataProductId(dataProduct.getId());
evaluationRequest.setEvent(PolicyEvaluationRequestResource.EventType.DATA_PRODUCT_CREATION);
ValidationResponseResource evaluationResult = validateInputObject(evaluationRequest);
checkPoliciesValidationResults(evaluationResult);
}

public boolean isCompliantWithPolicies(DataProductResource oldDataProduct, DataProductResource newDataProduct) {
public void validateDataProduct(DataProductResource oldDataProduct, DataProductResource newDataProduct) {
if (!this.policyServiceActive) {
logger.info("Policy Service is not active;");
return true;
return;
}
try {
PolicyEvaluationRequestResource evaluationRequest = new PolicyEvaluationRequestResource();
evaluationRequest.setResourceType(PolicyEvaluationRequestResource.ResourceType.DATA_PRODUCT_DESCRIPTOR);
DataProductVersionDPDS oldDpdsHead = descriptorFromDataProduct(oldDataProduct);
DataProductVersionDPDS newDpdsHead = descriptorFromDataProduct(newDataProduct);

evaluationRequest.setCurrentState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(oldDpdsHead)));
evaluationRequest.setAfterState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(newDpdsHead)));
evaluationRequest.setDataProductId(oldDataProduct.getId());
evaluationRequest.setEvent(PolicyEvaluationRequestResource.EventType.DATA_PRODUCT_UPDATE);
ValidationResponseResource evaluationResult = policyValidationClient.validateInputObject(evaluationRequest);

if (Boolean.FALSE.equals(evaluationResult.getResult())) {
logger.warn("Policy evaluation failed during DataProduct update");
}

return evaluationResult.getResult();
} catch (Exception e) {
throw new BadGatewayException(
ODMApiCommonErrors.SC502_71_POLICY_SERVICE_ERROR,
"An error occurred while invoking policy service to validate data product: " + e.getMessage(),
e
);

PolicyEvaluationRequestResource evaluationRequest = new PolicyEvaluationRequestResource();
evaluationRequest.setResourceType(PolicyEvaluationRequestResource.ResourceType.DATA_PRODUCT_DESCRIPTOR);
DataProductVersionDPDS oldDpdsHead = descriptorFromDataProduct(oldDataProduct);
DataProductVersionDPDS newDpdsHead = descriptorFromDataProduct(newDataProduct);

evaluationRequest.setCurrentState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(oldDpdsHead)));
evaluationRequest.setAfterState(JsonNodeUtils.toJsonNode(eventTypeMapper.toEventResource(newDpdsHead)));
evaluationRequest.setDataProductId(oldDataProduct.getId());
evaluationRequest.setEvent(PolicyEvaluationRequestResource.EventType.DATA_PRODUCT_UPDATE);
ValidationResponseResource evaluationResult = validateInputObject(evaluationRequest);
checkPoliciesValidationResults(evaluationResult);
}

private void checkPoliciesValidationResults(ValidationResponseResource validationResults) {
String allFailedPoliciesIds = "";
String failedBlockingPolicies = validationResults.getPolicyResults()
.stream()
.filter(policyResult -> Boolean.TRUE.equals(policyResult.getPolicy().getBlockingFlag()))
.map(PolicyEvaluationResultResource::getPolicyId)
.map(Object::toString)
.reduce("", (first, second) -> StringUtils.hasText(first) ? first + ", " + second : second);
if (StringUtils.hasText(failedBlockingPolicies)) {
allFailedPoliciesIds = allFailedPoliciesIds + " Blocking Policies IDs: [ " + failedBlockingPolicies + " ]";
}
String failedNonBlockingPolicies = validationResults.getPolicyResults()
.stream()
.filter(policyResult -> Boolean.FALSE.equals(policyResult.getPolicy().getBlockingFlag()))
.map(PolicyEvaluationResultResource::getPolicyId)
.map(Object::toString)
.reduce("", (first, second) -> StringUtils.hasText(first) ? first + ", " + second : second);
if (StringUtils.hasText(failedNonBlockingPolicies)) {
allFailedPoliciesIds = allFailedPoliciesIds + " Non-Blocking Policies IDs: [ " + failedNonBlockingPolicies + " ]";
}
logger.warn("The data product is not compliant to: {}", allFailedPoliciesIds);
if (StringUtils.hasText(failedBlockingPolicies)) {
throw new UnprocessableEntityException(
RegistryApiStandardErrors.SC422_03_DESCRIPTOR_NOT_COMPLIANT,
String.format("The data product is not compliant to: %s", allFailedPoliciesIds));
}
}

Expand All @@ -144,4 +139,17 @@ private DataProductVersionDPDS descriptorFromDataProduct(DataProductResource dat
dpdsHead.setInfo(dpdsHeadInfo);
return dpdsHead;
}

private ValidationResponseResource validateInputObject(PolicyEvaluationRequestResource evaluationRequest) {
try {
return policyValidationClient.validateInputObject(evaluationRequest);

} catch (Exception e) {
throw new BadGatewayException(
ODMApiCommonErrors.SC502_71_POLICY_SERVICE_ERROR,
"An error occurred while invoking policy service to validate data product version: " + e.getMessage(),
e
);
}
}
}
Loading

0 comments on commit 7e5f7f0

Please sign in to comment.