Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add decodeJson for DeidentifyJSONByResult and DeidentifyJSON #22

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions sdkdeidentify.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
package dlp

import (
"bytes"
"encoding/json"
"fmt"

Expand Down Expand Up @@ -63,7 +64,7 @@ func (I *Engine) DeidentifyJSON(jsonText string) (outStr string, retResults []*d
if results, kvMap, err := I.detectJSONImpl(jsonText); err == nil {
retResults = results
var jsonObj interface{}
if err := json.Unmarshal([]byte(jsonText), &jsonObj); err == nil {
if err := decodeJson([]byte(jsonText), &jsonObj); err == nil {
//kvMap := I.resultsToMap(results)
outObj := I.dfsJSON("", &jsonObj, kvMap, true)
if outJSON, err := json.Marshal(outObj); err == nil {
Expand Down Expand Up @@ -94,7 +95,7 @@ func (I *Engine) DeidentifyJSONByResult(jsonText string, detectResults []*dlphea
}
outStr = jsonText
var jsonObj interface{}
if err := json.Unmarshal([]byte(jsonText), &jsonObj); err == nil {
if err := decodeJson([]byte(jsonText), &jsonObj); err == nil {
kvMap := I.resultsToMap(detectResults)
outObj := I.dfsJSON("", &jsonObj, kvMap, true)
if outJSON, err := json.Marshal(outObj); err == nil {
Expand Down Expand Up @@ -177,3 +178,10 @@ func (I *Engine) resultsToMap(results []*dlpheader.DetectResult) map[string]stri
}
return kvMap
}

// decodeJson 应对精度丢失问题,json.Unmarshal到interface{}时,数字会被默认序列化为float64导致精度丢失
func decodeJson(data []byte, v interface{}) error {
d := json.NewDecoder(bytes.NewReader(data))
d.UseNumber()
return d.Decode(v)
}
23 changes: 23 additions & 0 deletions sdkdeidentify_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package dlp

import (
"encoding/json"
"testing"
)

func Test_decodeJson(t *testing.T) {
jsonText := `{"id":146310743121612001}`
var jsonObj interface{}
if err := json.Unmarshal([]byte(jsonText), &jsonObj); err != nil {
t.Error(err)
return
}
t.Logf("json.Unmarshal result:%v", jsonObj) //146310743121612000

err := decodeJson([]byte(jsonText), &jsonObj)
if err != nil {
t.Error(err)
return
}
t.Logf("decodeJson result:%v", jsonObj) //146310743121612001
}
2 changes: 1 addition & 1 deletion sdkdetect.go
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ func max(x, y int) int {
// detectJSONImpl implements detectJSON
func (I *Engine) detectJSONImpl(jsonText string) (retResults []*dlpheader.DetectResult, kvMap map[string]string, retErr error) {
var jsonObj interface{}
if err := json.Unmarshal([]byte(jsonText), &jsonObj); err == nil {
if err := decodeJson([]byte(jsonText), &jsonObj); err == nil {
//fmt.Printf("%+v\n", jsonObj)
kvMap = make(map[string]string, 0)
I.dfsJSON("", &jsonObj, kvMap, false)
Expand Down
2 changes: 1 addition & 1 deletion sdkinternal.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ func (I *Engine) dfsJSON(path string, ptr *interface{}, kvMap map[string]string,
if val, ok := (*ptr).(string); ok {
// try nested json Unmarshal
if I.maybeJSON(val) {
if err := json.Unmarshal([]byte(val), &subObj); err == nil {
if err := decodeJson([]byte(val), &subObj); err == nil {
obj := I.dfsJSON(path, &subObj, kvMap, isDeidentify)
if ret, err := json.Marshal(obj); err == nil {
retStr := string(ret)
Expand Down