From 29ef115f29d5597f64d3745e7b191d57cc853848 Mon Sep 17 00:00:00 2001
From: mrproliu <741550557@qq.com>
Date: Tue, 4 Jun 2024 09:11:09 +0000
Subject: [PATCH] Adding Access Log Service E2E Test (#127)

---
 .github/workflows/rover.yaml                  |   4 +
 CHANGES.md                                    |   1 +
 pkg/accesslog/collector/connect.go            |  26 +-
 pkg/accesslog/runner.go                       |   5 +-
 pkg/tools/ip/conntrack.go                     |  31 ++-
 test/e2e/base/env                             |   4 +-
 test/e2e/cases/access_log/e2e.yaml            | 231 ++++++++++++++++++
 .../expected/dependency-endpoint-reviews.yml  |  40 +++
 .../expected/dependency-instance.yml          |  41 ++++
 .../expected/dependency-services.yml          |  83 +++++++
 .../access_log/expected/metrics-has-value.yml |  31 +++
 .../expected/service-endpoint-reviews.yml     |  19 ++
 .../access_log/expected/service-instance.yml  |  22 ++
 .../e2e/cases/access_log/expected/service.yml |  45 ++++
 test/e2e/cases/access_log/kind.yaml           |  23 ++
 test/e2e/cases/access_log/rover.yaml          | 110 +++++++++
 test/e2e/cases/access_log/traffic-gen.yaml    |  40 +++
 .../http_avg_response_time/e2e.yaml           |   4 +-
 .../expected/metrics-has-value-labeld.yml     |  30 ++-
 .../continuous/http_error_rate/e2e.yaml       |   4 +-
 .../expected/metrics-has-value-labeld.yml     |  30 ++-
 .../profiling/task/network/base-cases.yaml    |  25 +-
 .../profiling/task/network/envoy/e2e.yaml     |  33 +--
 .../expected/metrics-has-value-label.yml      |  30 ++-
 .../expected/metrics-has-value-label.yml      |  30 ++-
 .../task/network/http1-metrics-cases.yaml     |  10 +-
 26 files changed, 857 insertions(+), 95 deletions(-)
 create mode 100644 test/e2e/cases/access_log/e2e.yaml
 create mode 100644 test/e2e/cases/access_log/expected/dependency-endpoint-reviews.yml
 create mode 100644 test/e2e/cases/access_log/expected/dependency-instance.yml
 create mode 100644 test/e2e/cases/access_log/expected/dependency-services.yml
 create mode 100644 test/e2e/cases/access_log/expected/metrics-has-value.yml
 create mode 100644 test/e2e/cases/access_log/expected/service-endpoint-reviews.yml
 create mode 100644 test/e2e/cases/access_log/expected/service-instance.yml
 create mode 100644 test/e2e/cases/access_log/expected/service.yml
 create mode 100644 test/e2e/cases/access_log/kind.yaml
 create mode 100644 test/e2e/cases/access_log/rover.yaml
 create mode 100644 test/e2e/cases/access_log/traffic-gen.yaml

diff --git a/.github/workflows/rover.yaml b/.github/workflows/rover.yaml
index ae63f970..964417bd 100644
--- a/.github/workflows/rover.yaml
+++ b/.github/workflows/rover.yaml
@@ -149,6 +149,10 @@ jobs:
             base: test/e2e/cases/process/istio
             config: e2e.yaml
             env: ISTIO_VERSION=1.13.1
+
+          - name: Access Log
+            base: test/e2e/cases/access_log
+            config: e2e.yaml
     steps:
       - uses: actions/checkout@v3
         with:
diff --git a/CHANGES.md b/CHANGES.md
index 5ea76ba2..9a39183a 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -8,6 +8,7 @@ Release Notes.
 * Upgrade LLVM to `18`.
 
 #### Bug Fixes
+* Fixed the issue where `conntrack` could not find the Reply IP in the access log module.
 
 #### Documentation
 
diff --git a/pkg/accesslog/collector/connect.go b/pkg/accesslog/collector/connect.go
index f315967d..ecd04a39 100644
--- a/pkg/accesslog/collector/connect.go
+++ b/pkg/accesslog/collector/connect.go
@@ -19,6 +19,7 @@ package collector
 
 import (
 	"encoding/binary"
+	"net"
 
 	"github.com/sirupsen/logrus"
 
@@ -99,6 +100,26 @@ func (c *ConnectCollector) Start(_ *module.Manager, context *common.AccessLogCon
 func (c *ConnectCollector) Stop() {
 }
 
+func (c *ConnectCollector) fixSocketFamilyIfNeed(event *events.SocketConnectEvent, result *ip.SocketPair) {
+	if result == nil {
+		return
+	}
+	if parseIP := net.ParseIP(result.SrcIP); parseIP != nil {
+		var actual uint32
+		if parseIP.To4() != nil {
+			actual = unix.AF_INET
+		} else {
+			actual = unix.AF_INET6
+		}
+
+		if result.Family != actual {
+			connectLogger.Debugf("fix the socket family from %d to %d, connection ID: %d, randomID: %d",
+				result.Family, actual, event.ConID, event.RandomID)
+			result.Family = actual
+		}
+	}
+}
+
 func (c *ConnectCollector) buildSocketFromConnectEvent(event *events.SocketConnectEvent) *ip.SocketPair {
 	if event.SocketFamily != unix.AF_INET && event.SocketFamily != unix.AF_INET6 && event.SocketFamily != enums.SocketFamilyUnknown {
 		// if not ipv4, ipv6 or unknown, ignore
@@ -122,6 +143,7 @@ func (c *ConnectCollector) buildSocketFromConnectEvent(event *events.SocketConne
 	connectLogger.Debugf("found the connection from the socket, connection ID: %d, randomID: %d",
 		event.ConID, event.RandomID)
 	pair.Role = enums.ConnectionRole(event.Role)
+	c.fixSocketFamilyIfNeed(event, pair)
 	c.tryToUpdateSocketFromConntrack(event, pair)
 	return pair
 }
@@ -193,12 +215,14 @@ func (c *ConnectCollector) buildSocketPair(event *events.SocketConnectEvent) *ip
 		return result
 	}
 
+	c.fixSocketFamilyIfNeed(event, result)
 	c.tryToUpdateSocketFromConntrack(event, result)
 	return result
 }
 
 func (c *ConnectCollector) tryToUpdateSocketFromConntrack(event *events.SocketConnectEvent, socket *ip.SocketPair) {
-	if socket != nil && socket.IsValid() && c.connTracker != nil && !tools.IsLocalHostAddress(socket.DestIP) {
+	if socket != nil && socket.IsValid() && c.connTracker != nil && !tools.IsLocalHostAddress(socket.DestIP) &&
+		event.FuncName != enums.SocketFunctionNameAccept { // accept event don't need to update the remote address
 		// if no contract and socket data is valid, then trying to get the remote address from the socket
 		// to encase the remote address is not the real remote address
 		originalIP := socket.DestIP
diff --git a/pkg/accesslog/runner.go b/pkg/accesslog/runner.go
index 2cab4f75..8762f62d 100644
--- a/pkg/accesslog/runner.go
+++ b/pkg/accesslog/runner.go
@@ -241,8 +241,9 @@ func (r *Runner) buildAccessLogMessage(firstLog, firstConnection bool, conn *com
 		rpcCon = conn.RPCConnection
 		if log.Enable(logrus.DebugLevel) {
 			log.Debugf("ready to sending access log with connection, connection ID: %d, random ID: %d, "+
-				"local: %s, remote: %s, role: %s",
-				conn.ConnectionID, conn.RandomID, rpcCon.Local, rpcCon.Remote, rpcCon.Role)
+				"local: %s, remote: %s, role: %s, kernel logs count: %d, contains protocol log: %t",
+				conn.ConnectionID, conn.RandomID, rpcCon.Local, rpcCon.Remote, rpcCon.Role,
+				len(kernelLogs), protocolLog != nil)
 		}
 	}
 	return &v3.EBPFAccessLogMessage{
diff --git a/pkg/tools/ip/conntrack.go b/pkg/tools/ip/conntrack.go
index 4d58d280..0ad7095f 100644
--- a/pkg/tools/ip/conntrack.go
+++ b/pkg/tools/ip/conntrack.go
@@ -30,7 +30,16 @@ import (
 
 var log = logger.GetLogger("tools", "ip")
 
-var numberStrategies = map[string]uint8{"tcp": syscall.IPPROTO_TCP, "udp": syscall.IPPROTO_UDP}
+var numberStrategies = []struct {
+	name  string
+	proto uint8
+}{{
+	name:  "tcp",
+	proto: syscall.IPPROTO_TCP,
+}, {
+	name:  "udp",
+	proto: syscall.IPPROTO_UDP,
+}}
 
 type ConnTrack struct {
 	tracker *conntrack.Nfct
@@ -51,13 +60,15 @@ func (c *ConnTrack) UpdateRealPeerAddress(addr *SocketPair) bool {
 	}
 
 	tuple := c.parseSocketToTuple(addr)
-	for name, strategy := range numberStrategies {
-		tuple.Proto.Number = &strategy
+	for _, info := range numberStrategies {
+		tuple.Proto.Number = &(info.proto)
 
+		// using get to query protocol
 		session, e := c.tracker.Get(conntrack.Conntrack, family, conntrack.Con{Origin: tuple})
 		if e != nil {
-			// try to get the reply session, if the strategy not exists or from accept events, have error is normal
-			log.Debugf("cannot get the conntrack session, strategy: %s, error: %v", name, e)
+			// try to get the reply session, if the info not exists or from accept events, have error is normal
+			log.Debugf("cannot get the conntrack session, type: %s, family: %d, origin src: %s:%d, origin dest: %s:%d, error: %v", info.name,
+				family, tuple.Src, *tuple.Proto.SrcPort, tuple.Dst, *tuple.Proto.DstPort, e)
 			continue
 		}
 
@@ -67,6 +78,16 @@ func (c *ConnTrack) UpdateRealPeerAddress(addr *SocketPair) bool {
 		}
 	}
 
+	// using dump to query protocol
+	dump, e := c.tracker.Dump(conntrack.Conntrack, family)
+	if e != nil {
+		log.Debug("cannot dump the conntrack session, error: ", e)
+		return false
+	}
+	if res := c.filterValidateReply(dump, tuple); res != nil {
+		addr.DestIP = res.Src.String()
+		return true
+	}
 	return false
 }
 
diff --git a/test/e2e/base/env b/test/e2e/base/env
index e417340c..8ee90056 100644
--- a/test/e2e/base/env
+++ b/test/e2e/base/env
@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-SW_CTL_COMMIT=6b2eb0011e38b630db6af7203db215806bd141ed
-SW_OAP_COMMIT=9ba0ad299139eeb4bb4e274c500cec1deaf84f79
+SW_CTL_COMMIT=ee371a210afe2dc6e65c2229b6a0519f8a4a2752
+SW_OAP_COMMIT=7a8bbacd20381bc780c48abf5706fd6e529c9872
 SW_KUBERNETES_COMMIT_SHA=a14f386063fffc61ed9b396e1328b76d33239aba
 
 SW_AGENT_GO_COMMIT=216f122d942cb683f48578d3014cc5ea83637582
diff --git a/test/e2e/cases/access_log/e2e.yaml b/test/e2e/cases/access_log/e2e.yaml
new file mode 100644
index 00000000..ff55ea42
--- /dev/null
+++ b/test/e2e/cases/access_log/e2e.yaml
@@ -0,0 +1,231 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+setup:
+  env: kind
+  file: kind.yaml
+  init-system-environment: ../../base/env
+  kind:
+    import-images:
+      - apache/skywalking-rover:latest
+    expose-ports:
+      - namespace: istio-system
+        resource: service/skywalking-ui
+        port: 80
+  steps:
+    - name: set PATH
+      command: export PATH=/tmp/skywalking-infra-e2e/bin:$PATH
+    - name: install swctl
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh swctl
+    - name: install kubectl
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh kubectl
+    - name: Install helm
+      command: bash test/e2e/base/scripts/prepare/setup-e2e-shell/install.sh helm
+    - name: Install SkyWalking
+      command: |
+        helm -n istio-system install skywalking --create-namespace \
+          oci://ghcr.io/apache/skywalking-helm/skywalking-helm \
+          --version "0.0.0-${SW_KUBERNETES_COMMIT_SHA}" \
+          --set fullnameOverride=skywalking \
+          --set elasticsearch.replicas=1 \
+          --set elasticsearch.minimumMasterNodes=1 \
+          --set oap.replicas=1 \
+          --set ui.image.repository=ghcr.io/apache/skywalking/ui \
+          --set ui.image.tag=${SW_OAP_COMMIT} \
+          --set oap.image.tag=${SW_OAP_COMMIT} \
+          --set oap.image.repository=ghcr.io/apache/skywalking/oap \
+          --set oap.storageType=elasticsearch \
+          -f test/e2e/base/kubernetes-values.yaml
+      wait:
+        - namespace: istio-system
+          resource: deployments/skywalking-oap
+          for: condition=available
+    - name: Deploy demo services
+      command: |
+        kubectl apply -f https://raw.githubusercontent.com/istio/istio/1.18.0/samples/bookinfo/platform/kube/bookinfo.yaml
+      wait:
+        - namespace: default
+          resource: pod
+          for: condition=Ready
+    - name: Install SkyWalking Rover
+      command: |
+        envsubst < test/e2e/cases/access_log/rover.yaml | kubectl apply -f -
+      wait:
+        - namespace: default
+          resource: pod
+          for: condition=Ready
+    - name: Generate traffic
+      path: traffic-gen.yaml
+      wait:
+        - namespace: default
+          resource: pod
+          for: condition=Ready
+  timeout: 25m
+
+verify:
+  retry:
+    count: 20
+    interval: 10s
+  cases:
+    # service list
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql service ls
+      expected: expected/service.yml
+    # service instance list
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance list --service-name=productpage.default
+      expected: expected/service-instance.yml
+
+    # service topology
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql dependency global --layer=K8S_SERVICE
+      expected: expected/dependency-services.yml
+    # service instance topology
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql dependency instance --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/dependency-instance.yml
+    # service endpoints
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql endpoint list --service-name=details.default
+      expected: expected/service-endpoint-reviews.yml
+    # endpoints topology
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql dependency endpoint --service-name=details.default --endpoint-name=/details/0
+      expected: expected/dependency-endpoint-reviews.yml
+
+    # service level metrics
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_connect_cpm --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_accept_cpm --service-name=reviews.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_write_cpm --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_read_cpm --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_write_l4_duration --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_read_l4_duration --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_write_package_cpm --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_read_package_cpm --service-name=productpage.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_http_call_cpm --service-name=reviews.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_http_call_duration --service-name=reviews.default
+      expected: expected/metrics-has-value.yml
+
+    # service instance level metrics
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_connect_cpm --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_write_cpm --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_write_l4_duration --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_write_package_cpm --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_read_cpm --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_read_l4_duration --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_read_package_cpm --service-name=productpage.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name reviews.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_http_call_cpm --service-name=reviews.default --instance-name=$instance_name
+      expected: expected/metrics-has-value.yml
+
+    # service endpoint level metrics
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_endpoint_call_cpm --service-name=details.default --endpoint-name=/details/0
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_endpoint_http_call_cpm --service-name=details.default --endpoint-name=/details/0
+      expected: expected/metrics-has-value.yml
+
+    # service relation metrics
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_connect_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_server_write_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_server_write_package_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_server_read_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_server_read_package_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_client_write_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_client_write_package_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_client_read_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+    - query: swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_relation_client_read_package_cpm --service-name=productpage.default --dest-service-name=details.default
+      expected: expected/metrics-has-value.yml
+
+    # service instance relation metrics
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_connect_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_server_write_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_server_write_package_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_server_read_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_server_read_package_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_client_write_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_client_write_package_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_client_read_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
+    - query: |
+        instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name productpage.default | yq '.[0].name' -)
+        dest_instance_name=$(swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql instance ls --service-name details.default | yq '.[0].name' -)
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=kubernetes_service_instance_relation_client_read_package_cpm --service-name=productpage.default --instance-name=$instance_name --dest-service-name=details.default --dest-instance-name=$dest_instance_name
+      expected: expected/metrics-has-value.yml
\ No newline at end of file
diff --git a/test/e2e/cases/access_log/expected/dependency-endpoint-reviews.yml b/test/e2e/cases/access_log/expected/dependency-endpoint-reviews.yml
new file mode 100644
index 00000000..a0bc6766
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/dependency-endpoint-reviews.yml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodes:
+{{- contains .nodes }}
+- id: {{ b64enc "productpage.default" }}.1_{{ b64enc "/details/0" }}
+  name: /details/0
+  serviceid: {{ b64enc "productpage.default" }}.1
+  servicename: productpage.default
+  type: ""
+  isreal: true
+- id: {{ b64enc "details.default" }}.1_{{ b64enc "/details/0" }}
+  name: /details/0
+  serviceid: {{ b64enc "details.default" }}.1
+  servicename: details.default
+  type: ""
+  isreal: true
+{{- end }}
+calls:
+{{- contains .calls }}
+- source: {{ b64enc "productpage.default" }}.1_{{ b64enc "/details/0" }}
+  sourcecomponents: []
+  target: {{ b64enc "details.default" }}.1_{{ b64enc "/details/0" }}
+  targetcomponents: []
+  id: {{ b64enc "productpage.default" }}.1-{{ b64enc "/details/0" }}-{{ b64enc "details.default" }}.1-{{ b64enc "/details/0" }}
+  detectpoints:
+    - SERVER
+{{- end }}
diff --git a/test/e2e/cases/access_log/expected/dependency-instance.yml b/test/e2e/cases/access_log/expected/dependency-instance.yml
new file mode 100644
index 00000000..f1045618
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/dependency-instance.yml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodes:
+{{- contains .nodes }}
+- id: {{ notEmpty .id }}
+  name: {{ notEmpty .name }}
+  serviceid: {{ b64enc "productpage.default" }}.1
+  servicename: productpage.default
+  type: ""
+  isreal: true
+- id: {{ notEmpty .id }}
+  name: {{ notEmpty .name }}
+  serviceid: {{ b64enc "details.default" }}.1
+  servicename: details.default
+  type: ""
+  isreal: true
+{{- end }}
+calls:
+{{- contains .calls }}
+- source: {{ notEmpty .source }}
+  sourcecomponents: []
+  target: {{ notEmpty .target }}
+  targetcomponents: []
+  id: {{ notEmpty .id }}
+  detectpoints:
+    - CLIENT
+    - SERVER
+{{- end }}
diff --git a/test/e2e/cases/access_log/expected/dependency-services.yml b/test/e2e/cases/access_log/expected/dependency-services.yml
new file mode 100644
index 00000000..bbe6f29a
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/dependency-services.yml
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodes:
+{{- contains .nodes }}
+- id: {{ b64enc "productpage.default"}}.1
+  name: productpage.default
+  type: null
+  isreal: true
+  layers: []
+- id: {{ b64enc "details.default"}}.1
+  name: details.default
+  type: http
+  isreal: true
+  layers: []
+- id: {{ b64enc "ratings.default" }}.1
+  name: ratings.default
+  type: http
+  isreal: true
+  layers: []
+- id: {{ b64enc "reviews.default" }}.1
+  name: reviews.default
+  type: http
+  isreal: true
+  layers: []
+{{- end }}
+calls:
+{{- contains .calls }}
+- source: {{ b64enc "productpage.default"}}.1
+  sourcecomponents:
+  {{- contains .sourcecomponents }}
+  - http
+  {{- end }}
+  target: {{ b64enc "details.default"}}.1
+  targetcomponents:
+  {{- contains .targetcomponents }}
+    - http
+  {{- end }}
+  id: {{ b64enc "productpage.default"}}.1-{{ b64enc "details.default"}}.1
+  detectpoints:
+    - CLIENT
+    - SERVER
+- source: {{ b64enc "productpage.default"}}.1
+  sourcecomponents:
+  {{- contains .sourcecomponents }}
+  - http
+  {{- end }}
+  target: {{ b64enc "reviews.default"}}.1
+  targetcomponents:
+  {{- contains .targetcomponents }}
+  - http
+  {{- end }}
+  id: {{ b64enc "productpage.default"}}.1-{{ b64enc "reviews.default"}}.1
+  detectpoints:
+    - CLIENT
+    - SERVER
+- source: {{ b64enc "reviews.default" }}.1
+  sourcecomponents:
+    {{- contains .sourcecomponents }}
+    - http
+    {{- end }}
+  target: {{ b64enc "ratings.default"}}.1
+  targetcomponents:
+  {{- contains .targetcomponents }}
+    - http
+  {{- end }}
+  id: {{ b64enc "reviews.default" }}.1-{{ b64enc "ratings.default"}}.1
+  detectpoints:
+    - CLIENT
+    - SERVER
+{{- end }}
diff --git a/test/e2e/cases/access_log/expected/metrics-has-value.yml b/test/e2e/cases/access_log/expected/metrics-has-value.yml
new file mode 100644
index 00000000..c4bbbafa
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/metrics-has-value.yml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels: []
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ .value }}
+        traceid: null
+      - id: {{ notEmpty .id }}
+        value: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
\ No newline at end of file
diff --git a/test/e2e/cases/access_log/expected/service-endpoint-reviews.yml b/test/e2e/cases/access_log/expected/service-endpoint-reviews.yml
new file mode 100644
index 00000000..fd5f37a4
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/service-endpoint-reviews.yml
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- contains . }}
+- id: {{ b64enc "details.default" }}.1_{{ b64enc "/details/0" }}
+  name: /details/0
+{{- end }}
diff --git a/test/e2e/cases/access_log/expected/service-instance.yml b/test/e2e/cases/access_log/expected/service-instance.yml
new file mode 100644
index 00000000..658cdbd5
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/service-instance.yml
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- contains . }}
+- id: {{ notEmpty .id }}
+  name: {{ notEmpty .name }}
+  attributes: []
+  language: UNKNOWN
+  instanceuuid: {{ notEmpty .instanceuuid }}
+{{- end }}
diff --git a/test/e2e/cases/access_log/expected/service.yml b/test/e2e/cases/access_log/expected/service.yml
new file mode 100644
index 00000000..872fbb60
--- /dev/null
+++ b/test/e2e/cases/access_log/expected/service.yml
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- contains . }}
+- id: {{ b64enc "details.default" }}.1
+  name: details.default
+  group: ""
+  shortname: details.default
+  layers:
+    - K8S_SERVICE
+  normal: true
+- id: {{ b64enc "reviews.default" }}.1
+  name: reviews.default
+  group: ""
+  shortname: reviews.default
+  layers:
+    - K8S_SERVICE
+  normal: true
+- id: {{ b64enc "productpage.default" }}.1
+  name: productpage.default
+  group: ""
+  shortname: productpage.default
+  layers:
+    - K8S_SERVICE
+  normal: true
+- id: {{ b64enc "ratings.default" }}.1
+  name: ratings.default
+  group: ""
+  shortname: ratings.default
+  layers:
+    - K8S_SERVICE
+  normal: true
+{{- end }}
\ No newline at end of file
diff --git a/test/e2e/cases/access_log/kind.yaml b/test/e2e/cases/access_log/kind.yaml
new file mode 100644
index 00000000..20a3fc74
--- /dev/null
+++ b/test/e2e/cases/access_log/kind.yaml
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8
+    extraMounts:
+      - hostPath: /
+        containerPath: /host
\ No newline at end of file
diff --git a/test/e2e/cases/access_log/rover.yaml b/test/e2e/cases/access_log/rover.yaml
new file mode 100644
index 00000000..e32f779a
--- /dev/null
+++ b/test/e2e/cases/access_log/rover.yaml
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: skywalking-rover
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: skywalking-rover
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: skywalking-rover
+subjects:
+  - kind: ServiceAccount
+    name: skywalking-rover
+    namespace: default
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: skywalking-rover
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "nodes", "services"]
+    verbs: ["get", "watch", "list"]
+---
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: skywalking-rover
+spec:
+  selector:
+    matchLabels:
+      name: skywalking-rover
+  template:
+    metadata:
+      labels:
+        name: skywalking-rover
+    spec:
+      serviceAccountName: skywalking-rover
+      serviceAccount: skywalking-rover
+      containers:
+        - name: skywalking-rover
+          # SkyWalking Rover image path
+          image: apache/skywalking-rover:latest
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            capabilities:
+              add:
+                - SYS_PTRACE
+                - SYS_ADMIN
+            privileged: true
+          volumeMounts:
+            - name: host
+              mountPath: /host
+              readOnly: true
+            - name: host-sys
+              mountPath: /sys
+              readOnly: true
+          env:
+            - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ACTIVE
+              value: "true"
+            - name: ROVER_LOGGER_LEVEL
+              value: "DEBUG"
+            - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: ROVER_BACKEND_ADDR
+              # backend OAP address
+              value: skywalking-oap.istio-system:11800
+            - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_ISTIO_ENVOY_ACTIVE
+              value: "false"
+            - name: ROVER_PROCESS_DISCOVERY_KUBERNETES_ANALYZER_ISTIO_APPLICATION_ACTIVE
+              value: "false"
+            - name: ROVER_HOST_MAPPING
+              value: /host
+            - name: ROVER_ACCESS_LOG_ACTIVE
+              value: "true"
+            - name: ROVER_ACCESS_LOG_FLUSH_PERIOD
+              value: 1s
+      hostPID: true
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      volumes:
+        - name: host
+          hostPath:
+            path: /host
+            type: Directory
+        - name: host-sys
+          hostPath:
+            path: /host/sys
+            type: Directory
\ No newline at end of file
diff --git a/test/e2e/cases/access_log/traffic-gen.yaml b/test/e2e/cases/access_log/traffic-gen.yaml
new file mode 100644
index 00000000..4e3b0502
--- /dev/null
+++ b/test/e2e/cases/access_log/traffic-gen.yaml
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trafficgenerator
+  labels:
+    app: trafficgenerator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: trafficgenerator
+  template:
+    metadata:
+      annotations:
+        sidecar.istio.io/inject: "false"
+      labels:
+        app: trafficgenerator
+    spec:
+      containers:
+        - name: trafficgenerator
+          image: elswork/wrk
+          command: ["wrk", "-t1", "-c1", "-d20m", "http://productpage:9080/productpage"]
+          resources:
+            requests:
+              cpu: 0.1
\ No newline at end of file
diff --git a/test/e2e/cases/profiling/continuous/http_avg_response_time/e2e.yaml b/test/e2e/cases/profiling/continuous/http_avg_response_time/e2e.yaml
index fd1e103e..5cfea254 100644
--- a/test/e2e/cases/profiling/continuous/http_avg_response_time/e2e.yaml
+++ b/test/e2e/cases/profiling/continuous/http_avg_response_time/e2e.yaml
@@ -98,8 +98,8 @@ verify:
 
     # check profiling metrics
     - query: |
-        swctl --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql --display yaml metrics multiple-linear \
-          --service-name test-continuous --instance-name test-instance --process-name response_timeout --name continuous_profiling_http_avg_response_time |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        swctl --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql --display yaml metrics exec --expression=continuous_profiling_http_avg_response_time \
+          --service-name test-continuous --instance-name test-instance --process-name response_timeout
       expected: expected/metrics-has-value-labeld.yml
 
     # check triggered profiling task
diff --git a/test/e2e/cases/profiling/continuous/http_avg_response_time/expected/metrics-has-value-labeld.yml b/test/e2e/cases/profiling/continuous/http_avg_response_time/expected/metrics-has-value-labeld.yml
index c57e8dda..fb41e2cd 100644
--- a/test/e2e/cases/profiling/continuous/http_avg_response_time/expected/metrics-has-value-labeld.yml
+++ b/test/e2e/cases/profiling/continuous/http_avg_response_time/expected/metrics-has-value-labeld.yml
@@ -13,13 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- contains . }}
-- key: {{ notEmpty .key }}
-  value:
-  {{- contains .value }}
-  - key: {{ notEmpty .key }}
-    value:
-      value: {{ ge .value.value 1 }}
-      isemptyvalue: false
-  {{- end }}
-{{- end }}
\ No newline at end of file
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels:
+        {{- contains .metric.labels }}
+        - key: {{ .key }}
+          value: {{ .value }}
+        {{- end}}
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ .value }}
+        traceid: null
+      - id: {{ notEmpty .id }}
+        value: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
diff --git a/test/e2e/cases/profiling/continuous/http_error_rate/e2e.yaml b/test/e2e/cases/profiling/continuous/http_error_rate/e2e.yaml
index 02941684..406da6ab 100644
--- a/test/e2e/cases/profiling/continuous/http_error_rate/e2e.yaml
+++ b/test/e2e/cases/profiling/continuous/http_error_rate/e2e.yaml
@@ -98,8 +98,8 @@ verify:
 
     # check profiling metrics
     - query: |
-        swctl --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql --display yaml metrics multiple-linear \
-          --service-name test-continuous --instance-name test-instance --process-name response_error --name continuous_profiling_http_error_rate |yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        swctl --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql --display yaml metrics exec --expression=continuous_profiling_http_error_rate \
+          --service-name test-continuous --instance-name test-instance --process-name response_error
       expected: expected/metrics-has-value-labeld.yml
 
     # check triggered profiling task
diff --git a/test/e2e/cases/profiling/continuous/http_error_rate/expected/metrics-has-value-labeld.yml b/test/e2e/cases/profiling/continuous/http_error_rate/expected/metrics-has-value-labeld.yml
index c57e8dda..fb41e2cd 100644
--- a/test/e2e/cases/profiling/continuous/http_error_rate/expected/metrics-has-value-labeld.yml
+++ b/test/e2e/cases/profiling/continuous/http_error_rate/expected/metrics-has-value-labeld.yml
@@ -13,13 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- contains . }}
-- key: {{ notEmpty .key }}
-  value:
-  {{- contains .value }}
-  - key: {{ notEmpty .key }}
-    value:
-      value: {{ ge .value.value 1 }}
-      isemptyvalue: false
-  {{- end }}
-{{- end }}
\ No newline at end of file
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels:
+        {{- contains .metric.labels }}
+        - key: {{ .key }}
+          value: {{ .value }}
+        {{- end}}
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ .value }}
+        traceid: null
+      - id: {{ notEmpty .id }}
+        value: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
diff --git a/test/e2e/cases/profiling/task/network/base-cases.yaml b/test/e2e/cases/profiling/task/network/base-cases.yaml
index 2a068ea0..7fe4adcb 100644
--- a/test/e2e/cases/profiling/task/network/base-cases.yaml
+++ b/test/e2e/cases/profiling/task/network/base-cases.yaml
@@ -63,32 +63,27 @@ cases:
 
   # histogram value of process relation, client side write_rtt/write_exe/read time
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_client_write_rtt_time_percentile \
       --service-name service --instance-name test --process-name service \
-      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE \
-      --name=process_relation_client_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE
     expected: expected/metrics-has-value-label.yml
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_client_write_exe_time_percentile \
       --service-name service --instance-name test --process-name service \
-      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE \
-      --name=process_relation_client_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE
     expected: expected/metrics-has-value-label.yml
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_server_write_rtt_time_percentile \
       --service-name service --instance-name test --process-name UNKNOWN_REMOTE \
-      --dest-service-name service --dest-instance-name test --dest-process-name service \
-      --name=process_relation_server_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name service
     expected: expected/metrics-has-value-label.yml
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_server_write_exe_time_percentile \
       --service-name service --instance-name test --process-name UNKNOWN_REMOTE \
-      --dest-service-name service --dest-instance-name test --dest-process-name service \
-      --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name service
     expected: expected/metrics-has-value-label.yml
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_server_write_exe_time_percentile \
       --service-name service --instance-name test --process-name UNKNOWN_REMOTE \
-      --dest-service-name service --dest-instance-name test --dest-process-name service \
-      --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name service
     expected: expected/metrics-has-value-label.yml
\ No newline at end of file
diff --git a/test/e2e/cases/profiling/task/network/envoy/e2e.yaml b/test/e2e/cases/profiling/task/network/envoy/e2e.yaml
index e3114db0..bfae32a8 100644
--- a/test/e2e/cases/profiling/task/network/envoy/e2e.yaml
+++ b/test/e2e/cases/profiling/task/network/envoy/e2e.yaml
@@ -144,40 +144,34 @@ verify:
 
     # histogram value of process relation, client side write_rtt/write_exe/read time
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_client_write_rtt_time_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
         --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name=process_relation_client_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
       expected: expected/metrics-has-value-label.yml
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_client_write_exe_time_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
-        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name=process_relation_client_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python
       expected: expected/metrics-has-value-label.yml
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_client_read_exe_time_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
-        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name=process_relation_client_read_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python
       expected: expected/metrics-has-value-label.yml
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_server_write_rtt_time_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
-        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name=process_relation_server_write_rtt_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python
       expected: expected/metrics-has-value-label.yml
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_server_write_exe_time_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
-        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python
       expected: expected/metrics-has-value-label.yml
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_server_write_exe_time_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
-        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name=process_relation_server_write_exe_time_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python
       expected: expected/metrics-has-value-label.yml
 
     # HTTP1 analyzer
@@ -200,10 +194,9 @@ verify:
         --name process_relation_http1_response_package_size|yq e 'to_entries' -
       expected: expected/metrics-has-value.yml
     - query: |
-        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+        swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_http1_request_package_size_percentile \
         --service-name productpage.default --instance-name productpage --process-name envoy \
-        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python \
-        --name process_relation_http1_request_package_size_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+        --dest-service-name productpage.default --dest-instance-name productpage --dest-process-name /usr/local/bin/python
       expected: expected/metrics-has-value-label.yml
     - query: |
         swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \
diff --git a/test/e2e/cases/profiling/task/network/envoy/expected/metrics-has-value-label.yml b/test/e2e/cases/profiling/task/network/envoy/expected/metrics-has-value-label.yml
index 8093415b..fb41e2cd 100644
--- a/test/e2e/cases/profiling/task/network/envoy/expected/metrics-has-value-label.yml
+++ b/test/e2e/cases/profiling/task/network/envoy/expected/metrics-has-value-label.yml
@@ -13,13 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- contains . }}
-- key: {{ notEmpty .key }}
-  value:
-  {{- contains .value }}
-  - key: {{ notEmpty .key }}
-    value:
-      value: {{ ge .value.value 1 }}
-      isemptyvalue: false
-  {{- end }}
-{{- end }}
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels:
+        {{- contains .metric.labels }}
+        - key: {{ .key }}
+          value: {{ .value }}
+        {{- end}}
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ .value }}
+        traceid: null
+      - id: {{ notEmpty .id }}
+        value: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
diff --git a/test/e2e/cases/profiling/task/network/expected/metrics-has-value-label.yml b/test/e2e/cases/profiling/task/network/expected/metrics-has-value-label.yml
index c57e8dda..fb41e2cd 100644
--- a/test/e2e/cases/profiling/task/network/expected/metrics-has-value-label.yml
+++ b/test/e2e/cases/profiling/task/network/expected/metrics-has-value-label.yml
@@ -13,13 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-{{- contains . }}
-- key: {{ notEmpty .key }}
-  value:
-  {{- contains .value }}
-  - key: {{ notEmpty .key }}
-    value:
-      value: {{ ge .value.value 1 }}
-      isemptyvalue: false
-  {{- end }}
-{{- end }}
\ No newline at end of file
+type: TIME_SERIES_VALUES
+results:
+  {{- contains .results }}
+  - metric:
+      labels:
+        {{- contains .metric.labels }}
+        - key: {{ .key }}
+          value: {{ .value }}
+        {{- end}}
+    values:
+      {{- contains .values }}
+      - id: {{ notEmpty .id }}
+        value: {{ .value }}
+        traceid: null
+      - id: {{ notEmpty .id }}
+        value: null
+        traceid: null
+      {{- end}}
+  {{- end}}
+error: null
diff --git a/test/e2e/cases/profiling/task/network/http1-metrics-cases.yaml b/test/e2e/cases/profiling/task/network/http1-metrics-cases.yaml
index 511d0b19..25733ddd 100644
--- a/test/e2e/cases/profiling/task/network/http1-metrics-cases.yaml
+++ b/test/e2e/cases/profiling/task/network/http1-metrics-cases.yaml
@@ -34,16 +34,14 @@ cases:
       --name process_relation_http1_response_package_size|yq e 'to_entries' -
     expected: expected/metrics-has-value.yml
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_http1_request_package_size_percentile \
       --service-name service --instance-name test --process-name service \
-      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE \
-      --name process_relation_http1_request_package_size_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE
     expected: expected/metrics-has-value-label.yml
   - query: |
-      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics multiple-linear \
+      swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics exec --expression=process_relation_http1_response_package_size_percentile \
       --service-name service --instance-name test --process-name service \
-      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE \
-      --name process_relation_http1_response_package_size_percentile|yq e 'to_entries | with(.[] ; .value=(.value | to_entries))' -
+      --dest-service-name service --dest-instance-name test --dest-process-name UNKNOWN_REMOTE
     expected: expected/metrics-has-value-label.yml
   - query: |
       swctl --display yaml --base-url=http://${service_skywalking_ui_host}:${service_skywalking_ui_80}/graphql metrics linear \