Improve: Analytics Event Data Masking

components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java

@@ -28,4 +28,13 @@ public class Constants {
     public static final String ANONYMOUS_VALUE = "anonymous";
     public static final String UNKNOWN_VALUE = "UNKNOWN";
     public static final int UNKNOWN_INT_VALUE = -1;
+    public static final String IPV4_PROP_TYPE = "IPV4";
+    public static final String IPV6_PROP_TYPE = "IPV6";
+    public static final String EMAIL_PROP_TYPE = "EMAIL";
+    public static final String USERNAME_PROP_TYPE = "USERNAME";
+
+    public static final String IPV4_MASK_VALUE = "***";
+    public static final String IPV6_MASK_VALUE = "**";
+    public static final String EMAIL_MASK_VALUE = "*****";
+    public static final String USERNAME_MASK_VALUE = "*****";
 }

components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java

@@ -73,4 +73,7 @@ public interface AnalyticsDataProvider {
     default Map<String, Object> getProperties() {
         return Collections.EMPTY_MAP;
     }
+
+    Map<String, String> getMaskProperties();
+
 }

components/apimgt/org.wso2.carbon.apimgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java

@@ -32,6 +32,18 @@
 import org.wso2.carbon.apimgt.common.analytics.publishers.dto.Target;
 import org.wso2.carbon.apimgt.common.analytics.publishers.impl.SuccessRequestDataPublisher;
 
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.wso2.carbon.apimgt.common.analytics.Constants.EMAIL_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.EMAIL_PROP_TYPE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV4_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV4_PROP_TYPE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV6_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV6_PROP_TYPE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.USERNAME_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.USERNAME_PROP_TYPE;
+
 /**
  * Success request data collector.
  */
@@ -58,6 +70,22 @@
 
         Event event = new Event();
         event.setProperties(provider.getProperties());
+
+        // Masking the configured data
+        Map<String, String> maskData = provider.getMaskProperties();
+        Iterator<Map.Entry<String, String>> iterator = maskData.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            Map<String, Object> props = event.getProperties();
+            if (props != null) {
+                Object value = props.get(entry.getKey());
+                if (value != null) {
+                    String maskStr = maskAnalyticsData(entry.getValue(), value);
+                    props.replace(entry.getKey(), maskStr);
+                }
+            }
+        }
+
         API api = provider.getApi();
         Operation operation = provider.getOperation();
         Target target = provider.getTarget();
@@ -70,14 +98,34 @@
         }
         Latencies latencies = provider.getLatencies();
         MetaInfo metaInfo = provider.getMetaInfo();
+
         String userAgent = provider.getUserAgentHeader();
         String userName = provider.getUserName();
+
+        // Mask UserName if configured
+        if (userName != null) {
+            if (maskData.containsKey("api.ut.userName")) {
+                userName = maskAnalyticsData(maskData.get("api.ut.userName"), userName);
+            } else if (maskData.containsKey("api.ut.userId")) {
+                userName = maskAnalyticsData(maskData.get("api.ut.userId"), userName);
+            }
+        }
+
         String userIp = provider.getEndUserIP();
         if (userIp == null) {
             userIp = Constants.UNKNOWN_VALUE;
+        } else {
+            // Mask User IP if configured
+            if (maskData.containsKey("api.analytics.user.ip")) {
+                userIp = maskAnalyticsData(maskData.get("api.analytics.user.ip"), userIp);
+            }
         }
         if (userAgent == null) {
             userAgent = Constants.UNKNOWN_VALUE;
+        } else {
+            if (maskData.containsKey("api.analytics.user.agent")) {
+                userAgent = maskAnalyticsData(maskData.get("api.analytics.user.agent"), userAgent);
+            }
         }
 
         event.setApi(api);
@@ -95,4 +143,33 @@
         this.processor.publish(event);
     }
 
+    private String maskAnalyticsData(String type, Object value) {
+        if (value instanceof String) {
+            switch (type) {
+                case IPV4_PROP_TYPE:
+                    String[] octets = value.toString().split("\\.");
+
+                    // Sample output: 192.168.***.98
+                    return octets[0] + "." + octets[1] + "." + IPV4_MASK_VALUE + "." + octets[3];
+                case IPV6_PROP_TYPE:
+                    octets = value.toString().split(":");
+
+                    // Sample output: 2001:0db8:85a3:****:****:****:****:7334
+                    return octets[0] + ":" + octets[1] + ":" + octets[2] + ":" + IPV6_MASK_VALUE + ":" + IPV6_MASK_VALUE
+                            + ":" + IPV6_MASK_VALUE + ":" + IPV6_MASK_VALUE + ":" + octets[7];
+                case EMAIL_PROP_TYPE:
+                    String[] email = value.toString().split("@");
+
+                    // Sample output: *****@gmail.com
+                    return EMAIL_MASK_VALUE + "@" + email[1];
+                case USERNAME_PROP_TYPE:
+                    return USERNAME_MASK_VALUE;
+                default:
+                    // Sample output: ********
+                    return USERNAME_MASK_VALUE;
+            }
+        }
+        return null;
+    }
+
 }

components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java

@@ -251,6 +251,13 @@ public MetaInfo getMetaInfo() {
         return metaInfo;
     }
 
+    @Override
+    public Map<String, String> getMaskProperties() {
+        Map<String, String> maskProperties = ServiceReferenceHolder.getInstance().getApiManagerConfigurationService()
+                .getAPIAnalyticsConfiguration().getMaskDataProperties();
+        return maskProperties;
+    }
+
     @Override
     public int getProxyResponseCode() {
 

components/apimgt/org.wso2.carbon.apimgt.gateway/src/main/java/org/wso2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java

@@ -254,6 +254,13 @@
         return metaInfo;
     }
 
+    @Override
+    public Map<String, String> getMaskProperties() {
+        Map<String, String> maskProperties = ServiceReferenceHolder.getInstance().getApiManagerConfigurationService()
+                .getAPIAnalyticsConfiguration().getMaskDataProperties();
+        return maskProperties;
+    }
+
     @Override
     public int getProxyResponseCode() {
         if (isSuccessRequest()) {

components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java

@@ -57,6 +57,7 @@ public class APIManagerAnalyticsConfiguration {
     private String responseSchemaName;
     private String faultSchemaName;
     private Map<String, String> reporterProperties;
+    private Map<String, String> maskDataProperties;
 
     private APIManagerAnalyticsConfiguration() {
     }
@@ -77,6 +78,7 @@ public void setAPIManagerConfiguration(APIManagerConfiguration config){
             this.responseSchemaName = config.getFirstProperty(APIConstants.API_ANALYTICS_RESPONSE_SCHEMA_NAME);
             this.faultSchemaName = config.getFirstProperty(APIConstants.API_ANALYTICS_FAULT_SCHEMA_NAME);
             this.reporterProperties = config.getAnalyticsProperties();
+            this.maskDataProperties = config.getAnalyticsMaskProperties();
         }
     }
 
@@ -235,6 +237,9 @@ public void setReporterClass(String reporterClass) {
     public Map<String, String> getReporterProperties() {
         return reporterProperties;
     }
+    public Map<String, String> getMaskDataProperties() {
+        return maskDataProperties;
+    }
 
     public void setReporterProperties(Map<String, String> reporterProperties) {
         this.reporterProperties = reporterProperties;

components/apimgt/org.wso2.carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java

@@ -127,6 +127,7 @@
     private RedisConfig redisConfig = new RedisConfig();
     private Map<String, List<String>> restApiJWTAuthAudiences = new HashMap<>();
     private JSONObject subscriberAttributes = new JSONObject();
+    private static Map<String, String> analyticsMaskProps;
 
     public Map<String, List<String>> getRestApiJWTAuthAudiences() {
         return restApiJWTAuthAudiences;
@@ -357,6 +358,19 @@
                         analyticsProps.put(name, value);
                     }
                 }
+
+                // Load all the mask properties
+                OMElement maskProperties = element.getFirstChildWithName(new QName("MaskProperties"));
+                Iterator maskPropertiesIterator = maskProperties.getChildrenWithLocalName("Property");
+                Map<String, String> maskProps = new HashMap<>();
+                while (maskPropertiesIterator.hasNext()) {
+                    OMElement propertyElem = (OMElement) maskPropertiesIterator.next();
+                    String name = propertyElem.getAttributeValue(new QName("name"));
+                    String value = propertyElem.getText();
+                    maskProps.put(name, value.toUpperCase());
+                }
+                analyticsMaskProps = maskProps;
+
                 OMElement authTokenElement = element.getFirstChildWithName(new QName("AuthToken"));
                 String resolvedAuthToken = MiscellaneousUtil.resolve(authTokenElement, secretResolver);
                 analyticsProps.put("auth.api.token", resolvedAuthToken);
@@ -2184,6 +2198,20 @@
             }
         }
 
+        OMElement properties = omElement.getFirstChildWithName(new
+                QName(APIConstants.API_GATEWAY_ADDITIONAL_PROPERTIES));
+        Map<String, String> additionalProperties = new HashMap<>();
+        if (properties != null) {
+            Iterator gatewayAdditionalProperties = properties.getChildrenWithLocalName
+                    (APIConstants.API_GATEWAY_ADDITIONAL_PROPERTY);
+            while (gatewayAdditionalProperties.hasNext()) {
+                OMElement propertyElem = (OMElement) gatewayAdditionalProperties.next();
+                String propName = propertyElem.getAttributeValue(new QName("name"));
+                String resolvedValue = MiscellaneousUtil.resolve(propertyElem, secretResolver);
+                additionalProperties.put(propName, resolvedValue);
+            }
+        }
+
         OMElement eventWaitingTimeElement = omElement
                 .getFirstChildWithName(new QName(APIConstants.EVENT_WAITING_TIME_CONFIG));
         if (eventWaitingTimeElement != null) {
@@ -2211,6 +2239,10 @@
     public static Map<String, String> getAnalyticsProperties() {
         return analyticsProperties;
     }
+
+    public static Map<String, String> getAnalyticsMaskProperties() {
+        return analyticsMaskProps;
+    }
 
     public static Map<String, String> getPersistenceProperties() {
         return persistenceProperties;

features/apimgt/org.wso2.carbon.apimgt.core.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2

@@ -304,6 +304,11 @@
             {% endfor %}
         </Properties>
 
+        <MaskProperties>
+            {% for key,value in apim.analytics.mask.items() %}
+            <Property name="{{key}}">{{value}}</Property>
+            {% endfor %}
+        </MaskProperties>
     </Analytics>
 
     <!--