Merge pull request #26 from hubblestack/develop

Merge to master (prep v2.1.1 release)

.gitignore

@@ -1,3 +1,7 @@
+scripts/temp/
+hubble.tar.gz
+hubble.spec
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -28,7 +32,6 @@ var/
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
 
 # Installer logs
 pip-log.txt

README.md

@@ -6,14 +6,16 @@ SaltStack infrastructure.
 # Building packages (CentOS)
 
 ```bash
-yum install rpm-build
-git clone git://github.com/hubblestack/hubble
-cd hubble
-python setup.py clean && python setup.py bdist_rpm
+sudo yum install git
+git clone git://github.com/hubblestack/hubble ~/hubble
+cd ~/hubble/pkg
+./build_rpms.sh  # note the lack of sudo, that is important
 ```
 
-Package will be in the `dist/` directory.
-
+Package will be in the `~/el6/` and `~/el7` directory. The only difference
+between the packages is the inclusion of `/etc/init.d/hubble` for el6 and
+the inclusion of a systemd unit file for el7. There's no guarantee of glibc
+compatibility.
 
 # Installation/testing
 

hook-salt.py

@@ -77,33 +77,7 @@
             DATAS.remove(entry)
 
 # Some packages salt required, which we should include that are not discovered by PyInstaller
-PACKAGES = [
- 'ssl',
- 'crypto',
- 'OpenSSL',
- 'argparse',
- 'base64',
- 'HTMLParser',
- 'json',
- 'logging',
- 'requests',
- 'functools',
- 'BaseHTTPServer',
- 'argparse',
- 'logging',
- 'pprint',
- 'os',
- 'random',
- 'git',
- 'daemon',
- 'boto3',
- 'botocore',
- 'six',
- 'inspect',
- 'yaml',
- 'traceback',
- 'pygit2'
-]
+PACKAGES = []
 
 for pkg in PACKAGES:
     DATAS.extend(collect_data_files(pkg, include_py_files=True))

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.1.0'
+__version__ = '2.1.1'

hubblestack/daemon.py

@@ -198,7 +198,7 @@ def schedule():
             run = True
 
         if run:
-            log.info('Executing scheduled function {0}'.format(func))
+            log.debug('Executing scheduled function {0}'.format(func))
             jobdata['last_run'] = time.time()
             ret = __salt__[func](*args, **kwargs)
             log.debug('Job returned:\n{0}'.format(ret))
@@ -207,7 +207,7 @@ def schedule():
                 if returner not in __returners__:
                     log.error('Could not find {0} returner.'.format(returner))
                     continue
-                log.info('Returning job data to {0}'.format(returner))
+                log.debug('Returning job data to {0}'.format(returner))
                 returner_ret = {'id': __grains__['id'],
                                 'jid': salt.utils.jid.gen_jid(),
                                 'fun': func,
@@ -265,6 +265,7 @@ def load_config():
     salt.config.DEFAULT_MINION_OPTS['cachedir'] = '/var/cache/hubble'
     salt.config.DEFAULT_MINION_OPTS['pidfile'] = '/var/run/hubble.pid'
     salt.config.DEFAULT_MINION_OPTS['log_file'] = '/var/log/hubble'
+    salt.config.DEFAULT_MINION_OPTS['log_level'] = None
     salt.config.DEFAULT_MINION_OPTS['file_client'] = 'local'
     salt.config.DEFAULT_MINION_OPTS['fileserver_update_frequency'] = 60
 
@@ -279,11 +280,12 @@ def load_config():
     __opts__.update(parsed_args)
 
     # Convert -vvv to log level
-    # Default to 'error'
-    __opts__['log_level'] = 'error'
-    # Default to more verbose if we're daemonizing
-    if __opts__['daemonize']:
-        __opts__['log_level'] = 'info'
+    if __opts__['log_level'] is None:
+        # Default to 'error'
+        __opts__['log_level'] = 'error'
+        # Default to more verbose if we're daemonizing
+        if __opts__['daemonize']:
+            __opts__['log_level'] = 'info'
     # Handle the explicit -vvv settings
     if __opts__['verbose'] == 1:
         __opts__['log_level'] = 'warning'

hubblestack/extmods/returners/slack_pulsar_returner.py

@@ -276,11 +276,11 @@ def returner(ret):
         log.error('slack_pulsar.api_key not defined in salt config')
         return
 
-    if isinstance(ret, dict):
+    if ret and isinstance(ret, dict):
         message = ('id: {0}\r\n'
                    'return: {1}\r\n').format(__opts__['id'],
                                              pprint.pformat(ret.get('return')))
-    elif isinstance(ret, list):
+    elif ret and isinstance(ret, list):
         message = 'id: {0}\r\n'
         for r in ret:
             message += pprint.pformat(r.get('return'))

init_pkg.sh

@@ -1,33 +1,42 @@
 #!/bin/bash
 
+_user=`id -u`
+
 # Installing minimum of python 2.7
 _python_version=`python -c 'import sys; version=sys.version_info[:3]; print("{0}.{1}.{2}".format(*version))'`
 
-read -r -d "" _PYTHON_HELP <<EOF
-
-You are running an older version python ... \n \n
-
-Please run the following commands to setup python27 \n \n
-
-sudo yum update \n
-sudo yum install centos-release-scl \n
-sudo yum install python27 \n
-scl enable python27 bash \n
-
-EOF
+# Check if the current user is root
+if [ "$_user" == "0" ]
+then
+  echo "This script should not be run as root ..."
+  echo "Please run this script as regular user with sudo privileges ..."
+  echo "Exiting ..."
+  exit
+fi
 
+# Check if the python version is 2.6.6
 if [ "$_python_version" == "2.6.6" ]
 then
-  echo -e $_PYTHON_HELP
+  sudo yum update -y
+  sudo yum install centos-release-scl -y
+  sudo yum install python27 -y
+  cd scripts/
+  sudo scl enable python27 'bash installessentials.sh'
+  sudo scl enable python27 "bash pyinstaller-hubble.sh pkg_clean $1"
+  sudo scl enable python27 'bash libgit2-build.sh'
+  bash osquery-build.sh
+  sudo scl enable python27 'bash pip-install.sh'
+  sudo scl enable python27 'bash pyinstaller-hubble.sh pkg_init'
+  sudo scl enable python27 'bash pyinstaller-hubble.sh pkg_create'
   exit
 fi
 
-
+# Normal install for python 2.7
 cd scripts/
 sudo bash installessentials.sh
+sudo bash pyinstaller-hubble.sh pkg_clean $1
 sudo bash libgit2-build.sh
 bash osquery-build.sh
 sudo bash pip-install.sh
-sudo bash pyinstaller-hubble.sh pkg_clean
 sudo bash pyinstaller-hubble.sh pkg_init
 sudo bash pyinstaller-hubble.sh pkg_create

pkg/build_rpms.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -x # echo on
+
+_user=`id -u`
+
+# Check if the current user is root
+if [ "$_user" == "0" ]
+then
+  echo "This script should not be run as root ..."
+  echo "Please run this script as regular user with sudo privileges ..."
+  echo "Exiting ..."
+  exit
+fi
+
+cd ..
+
+bash init_pkg.sh -y
+cp hubble.tar.gz ~/hubble.tar.gz
+rm -rf ~/hubblestack-2.1.1
+rm -rf ~/hubblestack-2.1.1.tar.gz
+mkdir ~/hubblestack-2.1.1
+tar -xzvf ~/hubble.tar.gz -C ~/hubblestack-2.1.1
+mkdir -p ~/hubblestack-2.1.1/etc/init.d
+cp pkg/hubble ~/hubblestack-2.1.1/etc/init.d
+mkdir -p ~/hubblestack-2.1.1/usr/lib/systemd/system
+cp pkg/hubble.service ~/hubblestack-2.1.1/usr/lib/systemd/system
+cp -f conf/hubble ~/hubblestack-2.1.1/etc/hubble/hubble
+cd ~
+tar -czvf hubblestack-2.1.1.tar.gz hubblestack-2.1.1/
+rm -rf ~/rpmbuild
+mkdir -p ~/rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp}
+
+cat <<EOF >~/.rpmmacros
+%_topdir   %(echo $HOME)/rpmbuild
+%_tmppath  %{_topdir}/tmp
+EOF
+
+cp ~/hubblestack-2.1.1.tar.gz ~/rpmbuild/SOURCES/
+cd ~/rpmbuild
+
+cp ~/hubble/pkg/specs/* SPECS/
+
+rpmbuild -ba SPECS/hubblestack-el6.spec
+rm -rf ~/el6
+mkdir ~/el6
+cp ~/rpmbuild/RPMS/x86_64/* ~/el6/
+rpmbuild -ba SPECS/hubblestack-el7.spec
+rm -rf ~/el7
+mkdir ~/el7
+cp ~/rpmbuild/RPMS/x86_64/* ~/el7/

pkg/specs/hubblestack-el6.spec

@@ -0,0 +1,60 @@
+# Don't try fancy stuff like debuginfo, which is useless on binary-only
+# packages. Don't strip binary too
+# Be sure buildpolicy set to do nothing
+%define        __spec_install_post %{nil}
+%define          debug_package %{nil}
+%define        __os_install_post %{_dbpath}/brp-compress
+# Don't fail out because we're not packaging the other distro's service files
+%define        _unpackaged_files_terminate_build 0
+
+Summary: Hubblestack is a module, open-source security compliance framework
+Name: hubblestack
+Version: 2.1.1
+Release: 1
+License: Apache 2.0
+Group: Development/Tools
+SOURCE0: %{name}-%{version}.tar.gz
+URL: https://hubblestack.io
+Autoreq: 0
+Requires: git
+
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
+
+%description
+%{summary}
+
+%prep
+%setup -q
+
+%build
+# Empty section.
+
+%install
+rm -rf %{buildroot}
+mkdir -p  %{buildroot}
+mkdir -p  %{buildroot}/usr/bin
+ln -s /opt/hubble/hubble %{buildroot}/usr/bin/hubble
+ln -s /opt/osquery/osqueryi %{buildroot}/usr/bin/osqueryi
+ln -s /opt/osquery/osqueryd %{buildroot}/usr/bin/osqueryd
+
+# in builddir
+cp -a * %{buildroot}
+
+
+%clean
+rm -rf %{buildroot}
+
+
+%files
+%{_sysconfdir}/hubble
+%{_sysconfdir}/osquery
+%{_sysconfdir}/init.d/hubble
+/opt/*
+/usr/bin/*
+
+%changelog
+* Wed Feb 8 2017  Colton Myers <[email protected]> 2.1.0-1
+- First Build
+
+* Mon Feb 13 2017  Colton Myers <[email protected]> 2.1.1-1
+- Remove autoreq, add unit files

pkg/specs/hubblestack-el7.spec

@@ -0,0 +1,60 @@
+# Don't try fancy stuff like debuginfo, which is useless on binary-only
+# packages. Don't strip binary too
+# Be sure buildpolicy set to do nothing
+%define        __spec_install_post %{nil}
+%define          debug_package %{nil}
+%define        __os_install_post %{_dbpath}/brp-compress
+# Don't fail out because we're not packaging the other distro's service files
+%define        _unpackaged_files_terminate_build 0
+
+Summary: Hubblestack is a module, open-source security compliance framework
+Name: hubblestack
+Version: 2.1.1
+Release: 1
+License: Apache 2.0
+Group: Development/Tools
+SOURCE0: %{name}-%{version}.tar.gz
+URL: https://hubblestack.io
+Autoreq: 0
+Requires: git
+
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
+
+%description
+%{summary}
+
+%prep
+%setup -q
+
+%build
+# Empty section.
+
+%install
+rm -rf %{buildroot}
+mkdir -p  %{buildroot}
+mkdir -p  %{buildroot}/usr/bin
+ln -s /opt/hubble/hubble %{buildroot}/usr/bin/hubble
+ln -s /opt/osquery/osqueryi %{buildroot}/usr/bin/osqueryi
+ln -s /opt/osquery/osqueryd %{buildroot}/usr/bin/osqueryd
+
+# in builddir
+cp -a * %{buildroot}
+
+
+%clean
+rm -rf %{buildroot}
+
+
+%files
+%{_sysconfdir}/hubble
+%{_sysconfdir}/osquery
+/opt/*
+/usr/bin/*
+/usr/lib/*
+
+%changelog
+* Wed Feb 8 2017  Colton Myers <[email protected]> 2.1.0-1
+- First Build
+
+* Mon Feb 13 2017  Colton Myers <[email protected]> 2.1.1-1
+- Remove autoreq, add unit files

scripts/installessentials.sh

@@ -2,10 +2,10 @@
 
 if [ -f "/usr/bin/apt-get" ]
 then
-  apt-get install python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev make cmake python-setuptools gcc -y
+  apt-get install python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev libjpeg8-dev zlib1g-dev make cmake python-setuptools gcc wget git -y
 elif [ -f "/usr/bin/yum" ]
 then
-  yum install python27-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libjpeg-devel zlib-devel make cmake python27-setuptools gcc -y
+  yum install python27-devel libffi-devel openssl-devel libxml2-devel libxslt-devel libjpeg-devel zlib-devel make cmake python27-setuptools python27-python-setuptools gcc python-devel python-setuptools git wget -y
 else
   echo "No package managers found ..."
   exit
@@ -14,4 +14,4 @@ fi
 easy_install pip
 
 # Creating required directories
-mkdir -p /etc/osquery /var/log/osquery /etc/hubble /opt/hubble /opt/osquery
+mkdir -p /etc/osquery /var/log/osquery /etc/hubble /opt/hubble /opt/osquery /var/osquery

scripts/osquery-build.sh

@@ -6,9 +6,9 @@ git clone https://github.com/facebook/osquery.git
 cd osquery
 make sysprep
 make deps
-make
+SKIP_TESTS=1 make -j 4
 make strip
-sudo mv ./build/linux/osquery/osqueryi ./build/linux/osquery/osqueryd /opt/osquery
+sudo cp -pr ./build/linux/osquery/osqueryi ./build/linux/osquery/osqueryd /opt/osquery
 sudo chown -R root. /opt/osquery
 cd ../../
 sudo cp ../conf/osquery.conf /etc/osquery/