Merge pull request #151 from hubblestack/develop

Merge to master (prep v2.2.1, for real this time)
hubblestack · Aug 8, 2017 · 0c60e38 · 0c60e38
2 parents 731e165 + e4e27b9
commit 0c60e38
Show file tree

Hide file tree

Showing 9 changed files with 42 additions and 13 deletions.
diff --git a/hubblestack/files/hubblestack_nova/grep.py b/hubblestack/files/hubblestack_nova/grep.py
@@ -28,6 +28,7 @@
               pattern: '/tmp'  # grep pattern
               match_output: 'nodev'  # string to check for in output of grep command (optional)
               match_output_regex: True  # whether to use regex when matching output (default: False)
+              match_output_multiline: False  # whether to use multiline flag for regex matching (default: True)
               grep_args:  # extra args to grep
                 - '-E'
                 - '-i'
@@ -122,8 +123,12 @@ def audit(data_list, tags, debug=False, **kwargs):
                         if tag_data['match_output'] not in grep_ret:
                             found = False
                     else:  # match with regex
-                        if not re.match(tag_data['match_output'], grep_ret):
-                            found = False
+                        if tag_data.get('match_output_multiline', True):
+                            if not re.search(tag_data['match_output'], grep_ret, re.MULTILINE):
+                                found = False
+                        else:
+                            if not re.search(tag_data['match_output'], grep_ret):
+                                found = False
 
                 if not os.path.exists(name) and 'match_on_file_missing' in tag_data:
                     if tag_data['match_on_file_missing']:

diff --git a/pkg/amazonlinux2016.09/Dockerfile b/pkg/amazonlinux2016.09/Dockerfile
@@ -37,13 +37,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+ && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -63,7 +66,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz
 ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9
 ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
-RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \
+RUN mkdir -p "$LIBGIT2TEMP" \
+ && cd "$LIBGIT2TEMP" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \

diff --git a/pkg/amazonlinux2017.03/Dockerfile b/pkg/amazonlinux2017.03/Dockerfile
@@ -37,13 +37,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+ && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -63,7 +66,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz
 ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9
 ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
-RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \
+RUN mkdir -p "$LIBGIT2TEMP" \
+ && cd "$LIBGIT2TEMP" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \

diff --git a/pkg/centos6/Dockerfile b/pkg/centos6/Dockerfile
@@ -37,13 +37,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+ && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -65,7 +68,6 @@ ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
 RUN mkdir -p "$LIBGIT2TEMP" \
  && cd "$LIBGIT2TEMP" \
- && echo "$LIBGIT2_SRC_URL" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \
@@ -124,8 +126,6 @@ CMD [ "scl enable python27 'pyinstaller --onedir --noconfirm --log-level ${_BINA
     && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
     && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \
     && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \
-    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \
-    && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \
     && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \
 #during container run, if a configuration file exists in a /data copy it over the existing one so it would be
 #possile to optionally include a custom one with the package

diff --git a/pkg/centos7/Dockerfile b/pkg/centos7/Dockerfile
@@ -37,13 +37,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+&& sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -63,7 +66,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz
 ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9
 ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
-RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \
+RUN mkdir -p "$LIBGIT2TEMP" \
+ && cd "$LIBGIT2TEMP" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \

diff --git a/pkg/coreos/Dockerfile b/pkg/coreos/Dockerfile
@@ -38,13 +38,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+ && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -64,7 +67,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz
 ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9
 ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
-RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \
+RUN mkdir -p "$LIBGIT2TEMP" \
+ && cd "$LIBGIT2TEMP" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \

diff --git a/pkg/debian8/Dockerfile b/pkg/debian8/Dockerfile
@@ -41,13 +41,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+ && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -67,7 +70,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz
 ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9
 ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
-RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \
+RUN mkdir -p "$LIBGIT2TEMP" \
+ && cd "$LIBGIT2TEMP" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \
@@ -124,7 +128,7 @@ CMD [ "pyinstaller --onedir --noconfirm --log-level ${_BINARY_LOG_LEVEL} --addit
     && tar -xzvf /data/hubblestack-${HUBBLE_VERSION}.tar.gz -C /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION} \
     && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d \
     && cp /hubble_build/pkg/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/init.d/ \
-    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSIOHUBBLE_VERSION}/usr/lib/systemd/system \
+    && mkdir -p /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system \
     && cp /hubble_build/pkg/hubble.service /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/usr/lib/systemd/system/ \
     && cp -f /hubble_build/conf/hubble /hubble_build/debbuild/hubblestack-${HUBBLE_VERSION}/etc/hubble/ \
 #during container run, if a configuration file exists in a /data copy it over the existing one so it would be

diff --git a/pkg/debian8/pyinstaller-requirements.txt b/pkg/debian8/pyinstaller-requirements.txt
@@ -12,5 +12,5 @@ pygit2
 salt-ssh
 gitpython
 pyinotify
-cffi
+cffi>=1.10.0
 azure
diff --git a/pkg/debian9/Dockerfile b/pkg/debian9/Dockerfile
@@ -38,13 +38,16 @@ RUN cd /home/"$OSQUERY_BUILD_USER" \
  && sed -i 's,^\(HOMEBREW_BREW=\).*,\1'ac2cbd2137006ebfe84d8584ccdcb5d78c1130d9',' tools/provision.sh \
  && sed -i 's,^\(LINUXBREW_BREW=\).*,\1'20bcce2c176469cec271b46d523eef1510217436',' tools/provision.sh \
  && make sysprep \
+#have the default augeas lenses directory point to /opt/osquery/lenses, must be done after sysprep
+ && sed -i '/augeas_lenses,/,/\"Directory\ that\ contains\ augeas\ lenses\ files\"\\)\;/ s/\/usr\/share\/osquery\/lenses/\/opt\/osquery\/lenses/' osquery/tables/system/posix/augeas.cpp \
  && make deps \
  && make \
  && make strip
 USER root
 RUN cp -pr /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryi /home/"$OSQUERY_BUILD_USER"/osquery/build/linux/osquery/osqueryd /opt/osquery \
  && chown -R root. /opt/osquery \
  && chmod -R 500 /opt/osquery/* \
+#put augeas lenses into the default directory that we changed earlier
  && mkdir -p /opt/osquery/lenses \
  && cp -r /usr/local/osquery/share/augeas/lenses/dist/* /opt/osquery/lenses \
  && chmod -R 400 /opt/osquery/lenses/*
@@ -64,7 +67,8 @@ ENV LIBGIT2_SRC_URL=https://github.com/libgit2/libgit2/archive/v0.26.0.tar.gz
 ENV LIBGIT2_SRC_SHA256=4ac70a2bbdf7a304ad2a9fb2c53ad3c8694be0dbec4f1fce0f3cd0cda14fb3b9
 ENV LIBGIT2_SRC_VERSION=0.26.0
 ENV LIBGIT2TEMP=/tmp/libgit2temp
-RUN mkdir -p "$LIBGIT2TEMP" && cd "$LIBGIT2TEMP" \
+RUN mkdir -p "$LIBGIT2TEMP" \
+ && cd "$LIBGIT2TEMP" \
  && wget -q "$LIBGIT2_SRC_URL" -O libgit2.tar.gz \
  && echo "$LIBGIT2_SRC_SHA256  libgit2.tar.gz" | sha256sum -c - \
  && tar xzf libgit2.tar.gz \