NOT FOR MERGE - srl295/perfplus

.github/workflows/staging.yml

@@ -57,6 +57,11 @@ jobs:
           -DskipTests=true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: DataDog sourcemap upload
+        run: npx --package=@datadog/datadog-ci datadog-ci sourcemaps upload tools/cldr-apps/src/main/webapp/dist/ --minified-path-prefix=/cldr-apps/dist/ --release-version=r${{ github.event.inputs.git-ref }} --service=surveytool
+        env:
+          DATADOG_SITE: ${{ secrets.DATADOG_SITE }}
+          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
       - name: Build liberty server
         run: >
           mvn -s .github/workflows/mvn-settings.xml -B  -pl cldr-apps liberty:create liberty:deploy liberty:package -Dinclude=usr --file tools/pom.xml

tools/cldr-apps/js/package.json

@@ -40,6 +40,8 @@
   },
   "private": true,
   "dependencies": {
+    "@datadog/browser-logs": "^5.4.0",
+    "@datadog/browser-rum": "^5.4.0",
     "ant-design-vue": "^3.2.16",
     "browser-fs-access": "^0.34.1",
     "marked": "^4.3.0",

tools/cldr-apps/js/src/esm/cldrNotify.mjs

@@ -16,6 +16,10 @@
 
 // Reference: https://www.antdv.com/components/notification
 import { notification } from "ant-design-vue";
+import { datadogLogs } from "@datadog/browser-logs";
+
+/** if falsy, let sleeping dogs lie */
+const hasDataDog = !!window.dataDogClientToken;
 
 /**
  * For warnings and general notifications, automatically close after this many seconds
@@ -48,6 +52,9 @@ function open(message, description) {
  * @param {String} description the more detailed description
  */
 function warning(message, description) {
+  if (hasDataDog) {
+    datadogLogs.logger.warn(message, { description });
+  }
   notification.warning({
     message: message,
     description: description,
@@ -62,6 +69,9 @@ function warning(message, description) {
  * @param {String} description the more detailed description
  */
 function error(message, description) {
+  if (hasDataDog) {
+    datadogLogs.logger.error(message, { description });
+  }
   notification.error({
     message: message,
     description: description,
@@ -73,6 +83,9 @@ function error(message, description) {
  * Display an error notification, and when the user closes it, call the callback function
  */
 function errorWithCallback(message, description, callback) {
+  if (hasDataDog) {
+    datadogLogs.logger.error(message, { description });
+  }
   notification.error({
     message: message,
     description: description,
@@ -94,6 +107,9 @@ function exception(e, context) {
       message: e,
     };
   }
+  if (hasDataDog) {
+    datadogLogs.logger.error(context, {}, e);
+  }
   notification.error({
     message: "Internal error: " + e.name + " " + context,
     description: e.message,

tools/cldr-apps/js/src/index.js

@@ -12,6 +12,39 @@ import "ant-design-vue/dist/antd.min.css";
 import * as cldrGui from "./esm/cldrGui.mjs";
 import * as cldrVue from "./esm/cldrVue.mjs";
 
+import { datadogRum } from "@datadog/browser-rum";
+import { datadogLogs } from "@datadog/browser-logs";
+
+if (window.dataDogClientToken) {
+  datadogLogs.init({
+    clientToken: window.dataDogClientToken,
+    site: "us5.datadoghq.com",
+    forwardErrorsToLogs: true,
+    sessionSampleRate: 100,
+  });
+
+  datadogRum.init({
+    applicationId: window.dataDogAppId,
+    clientToken: window.dataDogClientToken,
+    site: "us5.datadoghq.com",
+    service: "surveytool",
+    env: window.dataDogEnv,
+    version: "r" + window.dataDogSha,
+    sessionSampleRate: 100,
+    sessionReplaySampleRate: 20,
+    trackUserInteractions: true,
+    trackResources: true,
+    trackLongTasks: true,
+    defaultPrivacyLevel: "allow",
+    allowedTracingUrls: [
+      {
+        match: /https:\/\/.*\.unicode\.org/,
+        propagatorTypes: ["tracecontext"],
+      },
+    ],
+  });
+}
+
 /**
  * This is called as cldrBundle.runGui by way of JavaScript embedded in HTML
  * embedded in Java code! See SurveyTool.java

tools/cldr-apps/src/main/java/org/unicode/cldr/web/DataPage.java

@@ -21,6 +21,7 @@
 import org.unicode.cldr.icu.LDMLConstants;
 import org.unicode.cldr.test.*;
 import org.unicode.cldr.test.CheckCLDR.CheckStatus;
+import org.unicode.cldr.test.CheckCLDR.CheckStatus.Subtype;
 import org.unicode.cldr.test.CheckCLDR.InputMethod;
 import org.unicode.cldr.test.CheckCLDR.Options;
 import org.unicode.cldr.test.CheckCLDR.StatusAction;
@@ -307,30 +308,19 @@ public String getPClass() {
              */
             private boolean setTests(List<CheckStatus> testList) {
                 tests = ImmutableList.copyOf(testList);
-                // only consider non-example tests as notable.
                 boolean weHaveTests = false;
                 int errorCount = 0;
                 int warningCount = 0;
-                for (CheckStatus status : tests) {
-                    if (!status.getType().equals(CheckStatus.exampleType)) {
-                        // skip codefallback exemplar complaints (i.e. 'JPY'
-                        // isn't in exemplars).. they'll show up in missing
-                        if (DEBUG)
-                            System.err.println(
-                                    "err: "
-                                            + status.getMessage()
-                                            + ", test: "
-                                            + status.getClass()
-                                            + ", cause: "
-                                            + status.getCause()
-                                            + " on "
-                                            + xpath);
-                        weHaveTests = true;
-                        if (status.getType().equals(CheckStatus.errorType)) {
-                            errorCount++;
-                        } else if (status.getType().equals(CheckStatus.warningType)) {
-                            warningCount++;
-                        }
+                for (final CheckStatus status : tests) {
+                    logger.finest(() -> status + " on " + xpath);
+                    if (status.getType() == CheckStatus.exampleType) {
+                        continue; // does not count as an error or warning but included in payload
+                    }
+                    weHaveTests = true;
+                    if (status.getType().equals(CheckStatus.errorType)) {
+                        errorCount++;
+                    } else if (status.getType().equals(CheckStatus.warningType)) {
+                        warningCount++;
                     }
                 }
                 if (weHaveTests) {
@@ -759,6 +749,7 @@ private void setShimTests(String base_xpath_string, TestResultBundle checkCldr)
                 CandidateItem shimItem = new CandidateItem(null);
                 List<CheckStatus> iTests = new ArrayList<>();
                 checkCldr.check(base_xpath_string, iTests, null);
+                STFactory.removeExcludedChecks(iTests);
                 if (!iTests.isEmpty()) {
                     // Got a bite.
                     if (shimItem.setTests(iTests)) {
@@ -875,6 +866,7 @@ private void updateInheritedValue(CLDRFile ourSrc, TestResultBundle checkCldr) {
                 List<CheckStatus> iTests = new ArrayList<>();
 
                 checkCldr.check(xpath, iTests, inheritedValue);
+                STFactory.removeExcludedChecks(iTests);
 
                 if (TRACE_TIME) {
                     System.err.println("@@6:" + (System.currentTimeMillis() - lastTime));
@@ -1355,7 +1347,8 @@ public static DataPage make(
             CookieSession session,
             CLDRLocale locale,
             String prefix,
-            XPathMatcher matcher) {
+            XPathMatcher matcher,
+            TestResultBundle checkCldr) {
 
         SurveyMain sm =
                 CookieSession
@@ -1379,8 +1372,9 @@ public static DataPage make(
             throw new InternalError("?!! ourSrc hsa no supplemental dir!");
         }
         synchronized (session) {
-            TestResultBundle checkCldr =
-                    sm.getSTFactory().getTestResult(locale, getOptions(session, locale));
+            if (checkCldr == null) {
+                checkCldr = sm.getSTFactory().getTestResult(locale, getSimpleOptions(locale));
+            }
             if (checkCldr == null) {
                 throw new InternalError("checkCldr == null");
             }
@@ -1419,10 +1413,24 @@ public static CheckCLDR.Options getOptions(CookieSession session, CLDRLocale loc
 
         final String org = session.getEffectiveCoverageLevel(locale.toString());
 
-        options = new Options(locale, SurveyMain.getTestPhase(), def, org);
+        options = getOptions(locale, def, org);
         return options;
     }
 
+    private static CheckCLDR.Options getOptions(
+            CLDRLocale locale, final String defaultLevel, final String org) {
+        return new Options(locale, SurveyMain.getTestPhase(), defaultLevel, org);
+    }
+
+    /** Get options, but don't try to check user preferences */
+    public static CheckCLDR.Options getSimpleOptions(CLDRLocale locale) {
+        return new Options(
+                locale,
+                SurveyMain.getTestPhase(),
+                Level.COMPREHENSIVE.name(), /* localeType is not used?! */
+                "NOT USED");
+    }
+
     private final BallotBox<User> ballotBox;
 
     /*
@@ -1935,6 +1943,7 @@ private void populateFromThisXpath(
         List<CheckStatus> examplesResult = new ArrayList<>();
         if (checkCldr != null) {
             checkCldr.check(xpath, checkCldrResult, isExtraPath ? null : ourValue);
+            STFactory.removeExcludedChecks(checkCldrResult);
             checkCldr.getExamples(xpath, isExtraPath ? null : ourValue, examplesResult);
         }
         if (ourValue != null && ourValue.length() > 0) {
@@ -1966,6 +1975,7 @@ private void populateFromThisXpathAddItemsForVotes(
             if (avalue != null && checkCldr != null) {
                 List<CheckStatus> item2Result = new ArrayList<>();
                 checkCldr.check(xpath, item2Result, avalue);
+                STFactory.removeExcludedChecks(item2Result);
                 if (!item2Result.isEmpty()) {
                     item2.setTests(item2Result);
                 }