WIP: Add RE2 regexp engine support

.travis.yml

@@ -30,6 +30,7 @@ env:
   - OPTS="--without-ssdeep $OPTS"
   - OPTS="--without-lua $OPTS"
   - OPTS="--without-maxmind $OPTS"
+  - OPTS="--without-re2 $OPTS"
 
 before_script:
   - echo $TRAVIS_OS_NAME
@@ -41,6 +42,7 @@ before_script:
   - '[ "$TRAVIS_OS_NAME" != linux ] || sudo apt-get update'
   - '[ "$TRAVIS_OS_NAME" != linux ] || sudo apt-cache search maxmind'
   - '[ "$TRAVIS_OS_NAME" != linux ] || sudo apt-get install -y libmaxminddb-dev'
+  - '[ "$OPTS" == "*--without-re2*" ] || ./.travis/install_re2.sh'
 
 script:
   - ./build.sh

.travis/install_re2.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# Ubuntu 14.04 doesn't have RE2 package altogether,
+# and Ubuntu 16.04 RE2 package is too old.
+#
+# Ubuntu 18.04 RE2 package might work, but this Ubuntu verison it's not
+# supported by Travis yet.
+
+RELEASE=2019-01-01
+
+set -ex
+
+cd /tmp
+wget https://github.com/google/re2/archive/$RELEASE.tar.gz -O re2.tar.gz
+tar -xf re2.tar.gz
+(cd re2-$RELEASE && make && sudo make install)

build/pcre.m4

@@ -23,6 +23,16 @@ AC_ARG_WITH(
 
 AC_MSG_CHECKING([for libpcre config script])
 
+if test "x${with_pcre}" == "xno"; then
+    AC_MSG_NOTICE([Support for PCRE was disabled by the utilization of --without-pcre or --with-pcre=no])
+    PCRE_DISABLED=yes
+else
+    if test "x${with_pcre}" == "xyes"; then
+        PCRE_MANDATORY=yes
+        AC_MSG_NOTICE([PCRE support was marked as mandatory by the utilization of --with-pcre=yes])
+        test_paths="/usr/local/libpcre /usr/local/pcre /usr/local /opt/libpcre /opt/pcre /opt /usr /opt/local"
+    fi
+
 for x in ${test_paths}; do
     dnl # Determine if the script was specified and use it directly
     if test ! -d "$x" -a -e "$x"; then
@@ -55,7 +65,7 @@ if test -n "${pcre_path}"; then
     AC_MSG_RESULT([${PCRE_CONFIG}])
     PCRE_VERSION="`${PCRE_CONFIG} --version`"
     if test ! -z "${PCRE_VERSION}"; then AC_MSG_NOTICE(pcre VERSION: $PCRE_VERSION); fi
-    PCRE_CFLAGS="`${PCRE_CONFIG} --cflags`"
+    PCRE_CFLAGS="`${PCRE_CONFIG} --cflags` -DWITH_PCRE"
     if test ! -z "${PCRE_CFLAGS}"; then AC_MSG_NOTICE(pcre CFLAGS: $PCRE_CFLAGS); fi
     PCRE_LDADD="`${PCRE_CONFIG} --libs`"
     if test ! -z "${PCRE_LDADD}"; then AC_MSG_NOTICE(pcre LDADD: $PCRE_LDADD); fi
@@ -90,20 +100,31 @@ if test -n "${PCRE_VERSION}"; then
     LDFLAGS=$save_$LDFLAGS
 fi
 
-AC_SUBST(PCRE_CONFIG)
-AC_SUBST(PCRE_VERSION)
-AC_SUBST(PCRE_CPPFLAGS)
-AC_SUBST(PCRE_CFLAGS)
-AC_SUBST(PCRE_LDFLAGS)
-AC_SUBST(PCRE_LDADD)
-AC_SUBST(PCRE_LD_PATH)
+fi
+
 
-if test -z "${PCRE_VERSION}"; then
-    AC_MSG_NOTICE([*** pcre library not found.])
-    ifelse([$2], , AC_MSG_ERROR([pcre library is required]), $2)
+if test -z "${PCRE_LDADD}"; then
+    if test -z "${PCRE_MANDATORY}"; then
+        if test -z "${PCRE_DISABLED}"; then
+            PCRE_FOUND=0
+        else
+            PCRE_FOUND=2
+        fi
+    else
+        AC_MSG_ERROR([PCRE was explicitly referenced but it was not found])
+        PCRE_FOUND=-1
+    fi
 else
-    AC_MSG_NOTICE([using pcre v${PCRE_VERSION}])
-    ifelse([$1], , , $1) 
-    PCRE_LDADD="${PCRE_LDADD} -lpcre"
-fi 
+    PCRE_FOUND=1
+    AC_SUBST(PCRE_CONFIG)
+    AC_SUBST(PCRE_VERSION)
+    AC_SUBST(PCRE_CPPFLAGS)
+    AC_SUBST(PCRE_CFLAGS)
+    AC_SUBST(PCRE_LDFLAGS)
+    AC_SUBST(PCRE_LDADD)
+    AC_SUBST(PCRE_LD_PATH)
+    PCRE_DISPLAY="${PCRE_LDADD}, ${PCRE_CFLAGS}"
+    AC_SUBST(PCRE_DISPLAY)
+fi
+
 ])

build/re2.m4

@@ -0,0 +1,166 @@
+dnl Check for RE2 Libraries
+dnl CHECK_RE2(ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND])
+
+AC_DEFUN([PROG_RE2], [
+
+# Needed if pkg-config will be used.
+AC_REQUIRE([PKG_PROG_PKG_CONFIG])
+
+# Possible names for the re2 library/package (pkg-config)
+RE2_POSSIBLE_LIB_NAMES="re2"
+
+# Possible extensions for the library
+RE2_POSSIBLE_EXTENSIONS="so la sl dll dylib"
+
+# Possible paths (if pkg-config was not found, proceed with the file lookup)
+RE2_POSSIBLE_PATHS="/usr/lib /usr/local/lib /usr/local/libre2 /usr/local/re2 /usr/local /opt/libre2 /opt/re2 /opt /usr /usr/lib64"
+
+# Variables to be set by this very own script.
+RE2_VERSION=""
+RE2_CFLAGS=""
+RE2_CPPFLAGS=""
+RE2_LDADD=""
+RE2_LDFLAGS=""
+
+AC_ARG_WITH(
+    re2,
+    AC_HELP_STRING(
+      [--with-re2=PATH],
+      [Path to re2 prefix or config script]
+    )
+)
+
+if test "x${with_re2}" == "xno"; then
+    AC_DEFINE(HAVE_RE2, 0, [Support for RE2 was disabled by the utilization of --without-re2 or --with-re2=no])
+    AC_MSG_NOTICE([Support for RE2 was disabled by the utilization of --without-re2 or --with-re2=no])
+    RE2_DISABLED=yes
+else
+    if test "x${with_re2}" == "xyes"; then
+        RE2_MANDATORY=yes
+        AC_MSG_NOTICE([RE2 support was marked as mandatory by the utilization of --with-re2=yes])
+    fi
+    if test "x${with_re2}" == "x" || test "x${with_re2}" == "xyes"; then
+        # Nothing about RE2 was informed, using the pkg-config to figure things out.
+        if test -n "${PKG_CONFIG}"; then
+            RE2_PKG_NAME=""
+            for x in ${RE2_POSSIBLE_LIB_NAMES}; do
+                if ${PKG_CONFIG} --exists ${x}; then
+                    RE2_PKG_NAME="$x"
+                    break
+                fi
+            done
+        fi
+        AC_MSG_NOTICE([Nothing about RE2 was informed during the configure phase. Trying to detect it on the platform...])
+        if test -n "${RE2_PKG_NAME}"; then
+            # Package was found using the pkg-config scripts
+            RE2_VERSION="`${PKG_CONFIG} ${RE2_PKG_NAME} --modversion`"
+            RE2_CFLAGS="`${PKG_CONFIG} ${RE2_PKG_NAME} --cflags`"
+            RE2_LDADD="`${PKG_CONFIG} ${RE2_PKG_NAME} --libs-only-l`"
+            RE2_LDFLAGS="`${PKG_CONFIG} ${RE2_PKG_NAME} --libs-only-L --libs-only-other`"
+            RE2_DISPLAY="${RE2_LDADD}, ${RE2_CFLAGS}"
+        else
+            # If pkg-config did not find anything useful, go over file lookup.
+            for x in ${RE2_POSSIBLE_PATHS}; do
+                CHECK_FOR_RE2_AT(${x})
+                if test -n "${RE2_VERSION}"; then
+                    break
+                fi
+            done
+        fi
+    fi
+    if test "x${with_re2}" != "x"; then
+        # An specific path was informed, lets check.
+        RE2_MANDATORY=yes
+        CHECK_FOR_RE2_AT(${with_re2})
+    fi
+fi
+
+if test -z "${RE2_LDADD}"; then
+    if test -z "${RE2_MANDATORY}"; then
+        if test -z "${RE2_DISABLED}"; then
+            AC_MSG_NOTICE([RE2 library was not found])
+            RE2_FOUND=0
+        else
+            RE2_FOUND=2
+        fi
+    else
+        AC_MSG_ERROR([RE2 was explicitly referenced but it was not found])
+        RE2_FOUND=-1
+    fi
+else
+    RE2_FOUND=1
+    AC_MSG_NOTICE([using RE2 v${RE2_VERSION}])
+    RE2_CFLAGS="-DWITH_RE2 ${RE2_CFLAGS}"
+    RE2_DISPLAY="${RE2_LDADD}, ${RE2_CFLAGS}"
+    AC_SUBST(RE2_VERSION)
+    AC_SUBST(RE2_LDADD)
+    AC_SUBST(RE2_LIBS)
+    AC_SUBST(RE2_LDFLAGS)
+    AC_SUBST(RE2_CFLAGS)
+    AC_SUBST(RE2_DISPLAY)
+fi
+
+
+
+AC_SUBST(RE2_FOUND)
+
+]) # AC_DEFUN [PROG_RE2]
+
+
+AC_DEFUN([CHECK_FOR_RE2_AT], [
+    path=$1
+    for y in ${RE2_POSSIBLE_EXTENSIONS}; do
+        for z in ${RE2_POSSIBLE_LIB_NAMES}; do
+           if test -e "${path}/${z}.${y}"; then
+               re2_lib_path="${path}/"
+               re2_lib_name="${z}"
+               re2_lib_file="${re2_lib_path}/${z}.${y}"
+               break
+           fi
+           if test -e "${path}/lib${z}.${y}"; then
+               re2_lib_path="${path}/"
+               re2_lib_name="${z}"
+               re2_lib_file="${re2_lib_path}/lib${z}.${y}"
+               break
+           fi
+           if test -e "${path}/lib/lib${z}.${y}"; then
+               re2_lib_path="${path}/lib/"
+               re2_lib_name="${z}"
+               re2_lib_file="${re2_lib_path}/lib${z}.${y}"
+               break
+           fi
+           if test -e "${path}/lib/x86_64-linux-gnu/lib${z}.${y}"; then
+               re2_lib_path="${path}/lib/x86_64-linux-gnu/"
+               re2_lib_name="${z}"
+               re2_lib_file="${re2_lib_path}/lib${z}.${y}"
+               break
+           fi
+       done
+       if test -n "$re2_lib_path"; then
+           break
+       fi
+    done
+    if test -e "${path}/include/re2.h"; then
+        re2_inc_path="${path}/include"
+    elif test -e "${path}/re2.h"; then
+        re2_inc_path="${path}"
+    elif test -e "${path}/include/re2/re2.h"; then
+        re2_inc_path="${path}/include"
+    fi
+
+    if test -n "${re2_lib_path}"; then
+        AC_MSG_NOTICE([RE2 library found at: ${re2_lib_file}])
+    fi
+
+    if test -n "${re2_inc_path}"; then
+        AC_MSG_NOTICE([RE2 headers found at: ${re2_inc_path}])
+    fi
+
+    if test -n "${re2_lib_path}" -a -n "${re2_inc_path}"; then
+        # TODO: Compile a piece of code to check the version.
+        RE2_CFLAGS="-I${re2_inc_path}"
+        RE2_LDADD="-l${re2_lib_name}"
+        RE2_LDFLAGS="-L${re2_lib_path}"
+        RE2_DISPLAY="${re2_lib_file}, ${re2_inc_path}"
+    fi
+]) # AC_DEFUN [CHECK_FOR_RE2_AT]

configure.ac

@@ -105,6 +105,9 @@ AM_CONDITIONAL([SSDEEP_CFLAGS], [test "SSDEEP_CFLAGS" != ""])
 CHECK_LUA
 AM_CONDITIONAL([LUA_CFLAGS], [test "LUA_CFLAGS" != ""])
 
+# Check for RE2
+PROG_RE2
+AM_CONDITIONAL([RE2_CFLAGS], [test "RE2_CFLAGS" != ""])
 
 #
 # Check for curl
@@ -549,6 +552,68 @@ if test "x$LUA_FOUND" = "x2"; then
 fi
 
 
+## REGEX
+echo "   + Regular expression engine"
+
+REGEX_SELECTED_RE2="          "
+REGEX_SELECTED_PCRE="          "
+
+if test "x$RE2_FOUND" = "x1"; then
+    MSC_REGEX_ENG="re2"
+fi
+if test "x$PCRE_FOUND" = "x1"; then
+    MSC_REGEX_ENG="pcre"
+fi
+
+AC_SUBST(MSC_REGEX_ENG)
+
+if test "x$MSC_REGEX_ENG" = "xpcre"; then
+    MSC_REGEX_ENG_IS_PCRE=1
+    REGEX_SELECTED_PCRE="[[selected]]"
+    AC_SUBST(MSC_REGEX_ENG_IS_PCRE)
+fi
+if test "x$MSC_REGEX_ENG" = "xre2"; then
+    MSC_REGEX_ENG_IS_RE2=1
+    REGEX_SELECTED_PCRE="[[selected]]"
+    AC_SUBST(MSC_REGEX_ENG_IS_RE2)
+fi
+
+
+#### RE2
+if test "x$RE2_FOUND" = "x0"; then
+    echo "      * RE2 (experimental)                         ....not found"
+fi
+if test "x$RE2_FOUND" = "x1"; then
+    echo -n "      * RE2 (experimental)             ${REGEX_SELECTED_RE2}  ....found "
+    if ! test "x$RE2_VERSION" = "x"; then
+        echo "v${RE2_VERSION}"
+    else
+        echo ""
+    fi
+    echo "         ${RE2_DISPLAY}"
+fi
+if test "x$RE2_FOUND" = "x2"; then
+    echo "      * RE2 (experimental)                        ....disabled"
+fi
+
+#### PCRE
+if test "x$PCRE_FOUND" = "x0"; then
+    echo "      * PCRE                                       ....not found"
+fi
+if test "x$PCRE_FOUND" = "x1"; then
+    echo -n "      * PCRE                           ${REGEX_SELECTED_PCRE}  ....found "
+    if ! test "x$PCRE_VERSION" = "x"; then
+        echo "v${PCRE_VERSION}"
+    else
+        echo ""
+    fi
+    echo "         ${PCRE_DISPLAY}"
+fi
+if test "x$PCRE_FOUND" = "x2"; then
+    echo "      * PCRE                                       ....disabled"
+fi
+
+
 echo " "
 echo " Other Options"
 if test $buildTestUtilities = true; then

examples/reading_logs_via_rule_message/Makefile.am

@@ -15,7 +15,8 @@ simple_request_LDADD = \
 	$(LUA_LDADD) \
 	$(PCRE_LDADD) \
 	$(SSDEEP_LDADD) \
-	$(YAJL_LDADD)
+	$(YAJL_LDADD) \
+	$(RE2_LDADD)
 
 simple_request_LDFLAGS = \
 	-L$(top_builddir)/src/.libs/ \
@@ -28,7 +29,8 @@ simple_request_LDFLAGS = \
 	$(LUA_LDFLAGS) \
 	$(MAXMIND_LDFLAGS) \
 	$(SSDEEP_LDFLAGS) \
-	$(YAJL_LDFLAGS)
+	$(YAJL_LDFLAGS) \
+	$(RE2_LDFLAGS)
 
 simple_request_CPPFLAGS = \
 	$(GLOBAL_CFLAGS) \
@@ -46,7 +48,8 @@ simple_request_CPPFLAGS = \
 	$(LMDB_CFLAGS) \
 	$(LUA_CFLAGS) \
 	$(PCRE_CFLAGS) \
-	$(LIBXML2_CFLAGS)
+	$(LIBXML2_CFLAGS) \
+	$(RE2_CFLAGS)
 
 
 MAINTAINERCLEANFILES = \