Merge branch develop to master

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 This is a list of notable changes to Hyperscan, in reverse chronological order.
 
+## [5.1.0] 2019-01-17
+- Improve DFA state compression by wide-state optimization to reduce bytecode
+  size.
+- Create specific interpreter runtime handling to boost the performance of pure
+  literal matching.
+- Optimize original presentation of interpreter (the "Rose" engine ) to
+  increase overall performance.
+- Bugfix for logical combinations: fix error reporting combination's match in
+  case of sub-expression has EOD match under streaming mode.
+- Bugfix for logical combinations: fix miss reporting combination's match under
+  vacuous input.
+- Bugfix for issue #104: fix compile error with Boost 1.68.0.
+- Bugfix for issue #127: avoid pcre error for hscollider with installed PCRE
+  package.
+- Update version of PCRE used by testing tools as a syntax and semantic
+  reference to PCRE 8.41 or above.
+- Fix github repo address in doc.
+
 ## [5.0.0] 2018-07-09
 - Introduce chimera hybrid engine of Hyperscan and PCRE, to fully support
   PCRE syntax as well as to take advantage of the high performance nature of

CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required (VERSION 2.8.11)
 project (hyperscan C CXX)
 
 set (HS_MAJOR_VERSION 5)
-set (HS_MINOR_VERSION 0)
+set (HS_MINOR_VERSION 1)
 set (HS_PATCH_VERSION 0)
 set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION})
 
@@ -456,7 +456,7 @@ set(PCRE_REQUIRED_MINOR_VERSION 41)
 set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
 include (${CMAKE_MODULE_PATH}/pcre.cmake)
 if (NOT CORRECT_PCRE_VERSION)
-    message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
+    message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} or above not found")
 endif()
 
 # we need static libs for Chimera - too much deep magic for shared libs
@@ -508,7 +508,7 @@ set(PCRE_REQUIRED_MINOR_VERSION 41)
 set(PCRE_REQUIRED_VERSION ${PCRE_REQUIRED_MAJOR_VERSION}.${PCRE_REQUIRED_MINOR_VERSION})
 include (${CMAKE_MODULE_PATH}/pcre.cmake)
 if (NOT CORRECT_PCRE_VERSION)
-    message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} not found")
+    message(STATUS "PCRE ${PCRE_REQUIRED_VERSION} or above not found")
 endif()
 
 # we need static libs for Chimera - too much deep magic for shared libs

chimera/ch_compile.cpp

@@ -714,7 +714,7 @@ ch_error_t HS_CDECL ch_compile(const char *expression, unsigned flags,
                                                   (int)e.index : -1);
         return CH_COMPILER_ERROR;
     }
-    catch (std::bad_alloc) {
+    catch (std::bad_alloc &) {
         *db = nullptr;
         *comp_error = const_cast<ch_compile_error_t *>(&ch_enomem);
         return CH_COMPILER_ERROR;
@@ -782,7 +782,7 @@ ch_error_t HS_CDECL ch_compile_multi(const char *const *expressions,
                                                   (int)e.index : -1);
         return CH_COMPILER_ERROR;
     }
-    catch (std::bad_alloc) {
+    catch (std::bad_alloc &) {
         *db = nullptr;
         *comp_error = const_cast<ch_compile_error_t *>(&ch_enomem);
         return CH_COMPILER_ERROR;
@@ -855,7 +855,7 @@ ch_error_t HS_CDECL ch_compile_ext_multi(
                                                   (int)e.index : -1);
         return CH_COMPILER_ERROR;
     }
-    catch (std::bad_alloc) {
+    catch (std::bad_alloc &) {
         *db = nullptr;
         *comp_error = const_cast<ch_compile_error_t *>(&ch_enomem);
         return CH_COMPILER_ERROR;

chimera/ch_scratch.c

@@ -216,7 +216,6 @@ ch_error_t HS_CDECL ch_alloc_scratch(const ch_database_t *hydb,
     }
 
     if (db->flags & CHIMERA_FLAG_NO_MULTIMATCH) {
-        (*scratch)->multi_scratch = NULL;
         return CH_SUCCESS;
     }
 

cmake/pcre.cmake

@@ -27,18 +27,18 @@ if (PCRE_BUILD_SOURCE)
 
     # first, check version number
     CHECK_C_SOURCE_COMPILES("#include <pcre.h.generic>
-    #if PCRE_MAJOR != ${PCRE_REQUIRED_MAJOR_VERSION} || PCRE_MINOR != ${PCRE_REQUIRED_MINOR_VERSION}
+    #if PCRE_MAJOR != ${PCRE_REQUIRED_MAJOR_VERSION} || PCRE_MINOR < ${PCRE_REQUIRED_MINOR_VERSION}
     #error Incorrect pcre version
     #endif
     main() {}" CORRECT_PCRE_VERSION)
     set (CMAKE_REQUIRED_INCLUDES "${saved_INCLUDES}")
 
     if (NOT CORRECT_PCRE_VERSION)
         unset(CORRECT_PCRE_VERSION CACHE)
-        message(STATUS "Incorrect version of pcre - version ${PCRE_REQUIRED_VERSION} is required")
+        message(STATUS "Incorrect version of pcre - version ${PCRE_REQUIRED_VERSION} or above is required")
         return ()
     else()
-        message(STATUS "PCRE version ${PCRE_REQUIRED_VERSION} - building from source.")
+        message(STATUS "PCRE version ${PCRE_REQUIRED_VERSION} or above - building from source.")
     endif()
 
     # PCRE compile options
@@ -52,12 +52,12 @@ if (PCRE_BUILD_SOURCE)
 else ()
     # pkgconf should save us
     find_package(PkgConfig)
-    pkg_check_modules(PCRE libpcre=${PCRE_REQUIRED_VERSION})
+    pkg_check_modules(PCRE libpcre>=${PCRE_REQUIRED_VERSION})
     if (PCRE_FOUND)
         set(CORRECT_PCRE_VERSION TRUE)
-        message(STATUS "PCRE version ${PCRE_REQUIRED_VERSION}")
+        message(STATUS "PCRE version ${PCRE_REQUIRED_VERSION} or above")
     else ()
-        message(STATUS "PCRE version ${PCRE_REQUIRED_VERSION} not found")
+        message(STATUS "PCRE version ${PCRE_REQUIRED_VERSION} or above not found")
         return ()
     endif ()
 endif (PCRE_BUILD_SOURCE)

doc/dev-reference/compilation.rst

@@ -64,7 +64,7 @@ libpcre are supported. The use of unsupported constructs will result in
 compilation errors.
 
 The version of PCRE used to validate Hyperscan's interpretation of this syntax
-is 8.41.
+is 8.41 or above.
 
 ====================
 Supported Constructs

doc/dev-reference/getting_started.rst

@@ -10,7 +10,7 @@ Very Quick Start
 #. Clone Hyperscan ::
 
      cd <where-you-want-hyperscan-source>
-     git clone git://github/intel/hyperscan
+     git clone git://github.com/intel/hyperscan
 
 #. Configure Hyperscan
 

src/fdr/fdr_compile.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2019, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -807,6 +807,9 @@ void findIncludedLits(vector<hwlmLiteral> &lits,
         for (size_t i = 0; i < cnt; i++) {
             u32 bucket1 = group[i].first;
             u32 id1 = group[i].second;
+            if (lits[id1].pure) {
+                continue;
+            }
             buildSquashMask(lits, id1, bucket1, i + 1, group, parent_map,
                             exception_map);
         }

src/fdr/fdr_confirm.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2019, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -62,6 +62,7 @@ struct LitInfo {
     u8 size;
     u8 flags; //!< bitfield of flags from FDR_LIT_FLAG_* above.
     u8 next;
+    u8 pure; //!< The pass-on of pure flag from hwlmLiteral.
 };
 
 #define FDRC_FLAG_NO_CONFIRM 1

src/fdr/fdr_confirm_compile.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2019, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -87,6 +87,7 @@ void fillLitInfo(const vector<hwlmLiteral> &lits, vector<LitInfo> &tmpLitInfo,
         info.flags = flags;
         info.size = verify_u8(max(lit.msk.size(), lit.s.size()));
         info.groups = lit.groups;
+        info.pure = lit.pure;
 
         // these are built up assuming a LE machine
         CONF_TYPE msk = all_ones;

src/fdr/fdr_confirm_runtime.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2019, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -65,6 +65,7 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
     u8 oldNext; // initialized in loop
     do {
         assert(ISALIGNED(li));
+        scratch->pure = li->pure;
 
         if (unlikely((conf_key & li->msk) != li->v)) {
             goto out;
@@ -99,6 +100,7 @@ void confWithBit(const struct FDRConfirm *fdrc, const struct FDR_Runtime_Args *a
         li++;
     } while (oldNext);
     scratch->fdr_conf = NULL;
+    scratch->pure = 0;
 }
 
 #endif

src/grey.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2018, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -82,6 +82,7 @@ Grey::Grey(void) :
                    onlyOneOutfix(false),
                    allowShermanStates(true),
                    allowMcClellan8(true),
+                   allowWideStates(true), // enable wide state for McClellan8
                    highlanderPruneDFA(true),
                    minimizeDFA(true),
                    accelerateDFA(true),
@@ -197,7 +198,15 @@ void applyGreyOverrides(Grey *g, const string &s) {
 
         string::const_iterator ve = find(ke, pe, ';');
 
-        unsigned int value = lexical_cast<unsigned int>(string(ke + 1, ve));
+        unsigned int value = 0;
+        try {
+            value = lexical_cast<unsigned int>(string(ke + 1, ve));
+        } catch (boost::bad_lexical_cast &e) {
+            printf("Invalid grey override key %s:%s\n", key.c_str(),
+                   string(ke + 1, ve).c_str());
+            invalid_key_seen = true;
+            break;
+        }
         bool done = false;
 
         /* surely there exists a nice template to go with this macro to make
@@ -251,6 +260,7 @@ void applyGreyOverrides(Grey *g, const string &s) {
         G_UPDATE(onlyOneOutfix);
         G_UPDATE(allowShermanStates);
         G_UPDATE(allowMcClellan8);
+        G_UPDATE(allowWideStates);
         G_UPDATE(highlanderPruneDFA);
         G_UPDATE(minimizeDFA);
         G_UPDATE(accelerateDFA);

src/grey.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2018, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -87,6 +87,7 @@ struct Grey {
 
     bool allowShermanStates;
     bool allowMcClellan8;
+    bool allowWideStates; // enable wide state for McClellan8
     bool highlanderPruneDFA;
     bool minimizeDFA;
 

src/hwlm/hwlm_literal.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2019, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -83,9 +83,10 @@ bool maskIsConsistent(const std::string &s, bool nocase, const vector<u8> &msk,
  * \ref HWLM_MASKLEN. */
 hwlmLiteral::hwlmLiteral(const std::string &s_in, bool nocase_in,
                          bool noruns_in, u32 id_in, hwlm_group_t groups_in,
-                         const vector<u8> &msk_in, const vector<u8> &cmp_in)
+                         const vector<u8> &msk_in, const vector<u8> &cmp_in,
+                         bool pure_in)
     : s(s_in), id(id_in), nocase(nocase_in), noruns(noruns_in),
-      groups(groups_in), msk(msk_in), cmp(cmp_in) {
+      groups(groups_in), msk(msk_in), cmp(cmp_in), pure(pure_in) {
     assert(s.size() <= HWLM_LITERAL_MAX_LEN);
     assert(msk.size() <= HWLM_MASKLEN);
     assert(msk.size() == cmp.size());

src/hwlm/hwlm_literal.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2019, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -113,15 +113,20 @@ struct hwlmLiteral {
      */
     std::vector<u8> cmp;
 
+    bool pure; //!< \brief The pass-on of pure flag from LitFragment.
+
     /** \brief Complete constructor, takes group information and msk/cmp.
      *
      * This constructor takes a msk/cmp pair. Both must be vectors of length <=
      * \ref HWLM_MASKLEN. */
     hwlmLiteral(const std::string &s_in, bool nocase_in, bool noruns_in,
                 u32 id_in, hwlm_group_t groups_in,
-                const std::vector<u8> &msk_in, const std::vector<u8> &cmp_in);
+                const std::vector<u8> &msk_in, const std::vector<u8> &cmp_in,
+                bool pure_in = false);
 
-    /** \brief Simple constructor: no group information, no msk/cmp. */
+    /** \brief Simple constructor: no group information, no msk/cmp.
+     *
+     * This constructor is only used in internal unit test. */
     hwlmLiteral(const std::string &s_in, bool nocase_in, u32 id_in)
         : hwlmLiteral(s_in, nocase_in, false, id_in, HWLM_ALL_GROUPS, {}, {}) {}
 };

src/nfa/accel_dfa_build_strat.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015-2017, Intel Corporation
+ * Copyright (c) 2015-2018, Intel Corporation
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
@@ -40,6 +40,11 @@ namespace ue2 {
 
 class ReportManager;
 struct Grey;
+enum DfaType {
+    McClellan,
+    Sheng,
+    Gough
+};
 
 class accel_dfa_build_strat : public dfa_build_strat {
 public:
@@ -53,6 +58,8 @@ class accel_dfa_build_strat : public dfa_build_strat {
     virtual void buildAccel(dstate_id_t this_idx, const AccelScheme &info,
                             void *accel_out);
     virtual std::map<dstate_id_t, AccelScheme> getAccelInfo(const Grey &grey);
+    virtual DfaType getType() const = 0;
+
 private:
     bool only_accel_init;
 };

src/nfa/goughcompile.cpp

@@ -91,6 +91,7 @@ class gough_build_strat : public mcclellan_build_strat {
     void buildAccel(dstate_id_t this_idx, const AccelScheme &info,
                     void *accel_out) override;
     u32 max_allowed_offset_accel() const override { return 0; }
+    DfaType getType() const override { return Gough; }
 
     raw_som_dfa &rdfa;
     const GoughGraph ≫

src/nfa/limex_compile.cpp

@@ -980,7 +980,7 @@ u32 addSquashMask(const build_info &args, const NFAVertex &v,
     // see if we've already seen it, otherwise add a new one.
     auto it = find(squash.begin(), squash.end(), sit->second);
     if (it != squash.end()) {
-        return verify_u32(distance(squash.begin(), it));
+        return verify_u32(std::distance(squash.begin(), it));
     }
     u32 idx = verify_u32(squash.size());
     squash.push_back(sit->second);
@@ -1007,7 +1007,7 @@ u32 addReports(const flat_set<ReportID> &r, vector<ReportID> &reports,
     auto it = search(begin(reports), end(reports), begin(my_reports),
                      end(my_reports));
     if (it != end(reports)) {
-        u32 offset = verify_u32(distance(begin(reports), it));
+        u32 offset = verify_u32(std::distance(begin(reports), it));
         DEBUG_PRINTF("reusing found report list at %u\n", offset);
         return offset;
     }