Merge pull request #53 from hubblestack/develop

Merge to master (prep v2.1.7)

README.md

@@ -3,45 +3,51 @@
 An alternate version of Hubblestack which can be run without an existing
 SaltStack infrastructure.
 
-# Building standalone packages (CentOS)
+# Packaging / Installing
+
+## Installing using setup.py
+
+```bash
+sudo yum install git -y
+git clone https://github.com/hubblestack/hubble
+cd hubble
+sudo python setup.py install
+```
+
+Installs a `hubble` "binary" into `/usr/bin/`.
+
+## Building standalone packages (CentOS)
 
 ```bash
 sudo yum install git -y
-git clone git://github.com/hubblestack/hubble ~/hubble
-cd ~/hubble/pkg
+git clone https://github.com/hubblestack/hubble
+cd hubble/pkg
 ./build_rpms.sh  # note the lack of sudo, that is important
 ```
 
-Package will be in the `~/el6/` and `~/el7` directory. The only difference
+Packages will be in the `hubble/pkg/dist/` directory. The only difference
 between the packages is the inclusion of `/etc/init.d/hubble` for el6 and
 the inclusion of a systemd unit file for el7. There's no guarantee of glibc
 compatibility.
 
-# Building dep-heavy cross-platform packages
+## Building standalone packages (Debian)
 
 ```bash
 sudo yum install git -y
-git clone git://github.com/hubblestack/hubble
-cd hubble
-python setup.py bdist_rpm
+git clone https://github.com/hubblestack/hubble
+cd hubble/pkg
+./build_debs.sh  # note the lack of sudo, that is important
 ```
 
-You'll find the generated RPM in the `dist/` folder.
+Package will be in the `hubble/pkg/dist/` directory. There's no guarantee of
+glibc compatibility.
 
+# Usage
 
-# Testing
+A config template has been placed in `/etc/hubble/hubble`. Modify it to your
+specifications and needs.
 
-You can do `hubble -h` to see the available options. Here's a sample working
-config you can place in `/etc/hubble/hubble`. Note that you'll need to install
-python-pygit2 to get gitfs working:
-
-```
-gitfs_remotes:
-  - https://github.com/hubblestack/hubblestack_data.git
-fileserver_backend:
-  - roots
-  - git
-```
+You can do `hubble -h` to see the available options.
 
 The first two commands you should run to make sure things are set up correctly
 are `hubble --version` and `hubble test.ping`. If those run without issue
@@ -98,8 +104,8 @@ schedule:
     run_on_start: True
 ```
 
-Note that you need to have your splunk_nova_return configured in order to use
-the above block:
+Note that you need to have your hubblestack splunk returner configured in order
+to use the above block:
 
 ```
 hubblestack:
@@ -112,3 +118,10 @@ hubblestack:
         sourcetype_nebula: hubble_osquery
         sourcetype_pulsar: hubble_fim
 ```
+
+When using the scheduler, you can just run `hubble` in the foreground, or use
+the included sysvinit and systemd files to run it as a service in the
+background. You can also start it as a daemon without any scripts by using the
+`-d` argument.
+
+Use `-vvv` to turn on debug logging.

conf/hubble

@@ -10,8 +10,8 @@
 gitfs_remotes:
   - https://github.com/hubblestack/hubblestack_data.git
 fileserver_backend:
-  - git
   - roots
+  - git
 
 #################################
 ## Scheduler Config

hubblestack/__init__.py

@@ -1 +1 @@
-__version__ = '2.1.6'
+__version__ = '2.1.7'

hubblestack/daemon.py

@@ -311,6 +311,9 @@ def load_config():
     salt.log.setup.setup_console_logger(__opts__['log_level'])
     salt.log.setup.setup_logfile_logger(__opts__['log_file'],
                                         __opts__['log_level'])
+    # 384 is 0o600 permissions, written without octal for python 2/3 compat
+    os.chmod(__opts__['log_file'], 384)
+    os.chmod(parsed_args.get('configfile'), 384)
 
     __grains__ = salt.loader.grains(__opts__)
     __pillar__ = {}

hubblestack/extmods/returners/splunk_nebula_return.py

@@ -89,6 +89,11 @@ def returner(ret):
             fqdn_ip4 = __grains__['fqdn_ip4'][0]
         except IndexError:
             fqdn_ip4 = __grains__['ipv4'][0]
+        if fqdn_ip4.startswith('127.'):
+            for ip4_addr in __grains__['ipv4']:
+                if ip4_addr and not ip4_addr.startswith('127.'):
+                    fqdn_ip4 = ip4_addr
+                    break
 
         if not data:
             return

hubblestack/extmods/returners/splunk_nova_return.py

@@ -88,6 +88,11 @@ def returner(ret):
             fqdn_ip4 = __grains__['fqdn_ip4'][0]
         except IndexError:
             fqdn_ip4 = __grains__['ipv4'][0]
+        if fqdn_ip4.startswith('127.'):
+            for ip4_addr in __grains__['ipv4']:
+                if ip4_addr and not ip4_addr.startswith('127.'):
+                    fqdn_ip4 = ip4_addr
+                    break
 
         if __grains__['master']:
             master = __grains__['master']

hubblestack/extmods/returners/splunk_pulsar_return.py

@@ -97,6 +97,11 @@ def returner(ret):
             fqdn_ip4 = __grains__['fqdn_ip4'][0]
         except IndexError:
             fqdn_ip4 = __grains__['ipv4'][0]
+        if fqdn_ip4.startswith('127.'):
+            for ip4_addr in __grains__['ipv4']:
+                if ip4_addr and not ip4_addr.startswith('127.'):
+                    fqdn_ip4 = ip4_addr
+                    break
 
         alerts = []
         for item in data:

pkg/build_debs.sh

@@ -22,14 +22,14 @@ mkdir -p dist
 bash ./init_pkg.sh -y
 cp ../hubble.tar.gz dist/hubble.tar.gz
 mv ../hubble.tar.gz build/hubble.tar.gz
-mkdir build/hubblestack-2.1.6
-tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.6
-mkdir -p build/hubblestack-2.1.6/etc/init.d
-cp ./hubble build/hubblestack-2.1.6/etc/init.d
-mkdir -p build/hubblestack-2.1.6/usr/lib/systemd/system
-cp ./hubble.service build/hubblestack-2.1.6/usr/lib/systemd/system
-cp -f ../conf/hubble build/hubblestack-2.1.6/etc/hubble/hubble
-cd build/hubblestack-2.1.6
+mkdir build/hubblestack-2.1.7
+tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.7
+mkdir -p build/hubblestack-2.1.7/etc/init.d
+cp ./hubble build/hubblestack-2.1.7/etc/init.d
+mkdir -p build/hubblestack-2.1.7/usr/lib/systemd/system
+cp ./hubble.service build/hubblestack-2.1.7/usr/lib/systemd/system
+cp -f ../conf/hubble build/hubblestack-2.1.7/etc/hubble/hubble
+cd build/hubblestack-2.1.7
 
 sudo apt-get install -y ruby ruby-dev rubygems gcc make
 sudo gem install --no-ri --no-rdoc fpm
@@ -39,9 +39,9 @@ ln -s /opt/osquery/osqueryd usr/bin/osqueryd
 ln -s /opt/osquery/osqueryi usr/bin/osqueryi
 fpm -s dir -t deb \
     -n hubblestack \
-    -v 2.1.6-1 \
+    -v 2.1.7-1 \
     -d 'git' \
     --config-files /etc/hubble/hubble --config-files /etc/osquery/osquery.conf \
     --deb-no-default-config-files \
     etc/hubble etc/osquery etc/init.d opt usr/bin
-cp hubblestack_2.1.6-1_amd64.deb ../../dist/
+cp hubblestack_2.1.7-1_amd64.deb ../../dist/

pkg/build_rpms.sh

@@ -22,23 +22,23 @@ mkdir -p dist
 bash ./init_pkg.sh -y
 cp ../hubble.tar.gz dist/hubble.tar.gz
 mv ../hubble.tar.gz build/hubble.tar.gz
-mkdir build/hubblestack-2.1.6
-tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.6
-mkdir -p build/hubblestack-2.1.6/etc/init.d
-cp ./hubble build/hubblestack-2.1.6/etc/init.d
-mkdir -p build/hubblestack-2.1.6/usr/lib/systemd/system
-cp ./hubble.service build/hubblestack-2.1.6/usr/lib/systemd/system
-cp -f ../conf/hubble build/hubblestack-2.1.6/etc/hubble/hubble
+mkdir build/hubblestack-2.1.7
+tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.7
+mkdir -p build/hubblestack-2.1.7/etc/init.d
+cp ./hubble build/hubblestack-2.1.7/etc/init.d
+mkdir -p build/hubblestack-2.1.7/usr/lib/systemd/system
+cp ./hubble.service build/hubblestack-2.1.7/usr/lib/systemd/system
+cp -f ../conf/hubble build/hubblestack-2.1.7/etc/hubble/hubble
 cd build
-tar -czvf hubblestack-2.1.6.tar.gz hubblestack-2.1.6/
+tar -czvf hubblestack-2.1.7.tar.gz hubblestack-2.1.7/
 mkdir -p rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp}
 
-cp hubblestack-2.1.6.tar.gz rpmbuild/SOURCES/
+cp hubblestack-2.1.7.tar.gz rpmbuild/SOURCES/
 cd rpmbuild
 
 cp ../../specs/* SPECS/
 
 rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el6.spec
-cp RPMS/x86_64/hubblestack-2.1.6-1.x86_64.rpm ../../dist/hubblestack-2.1.6-1.el6.x86_64.rpm
+cp RPMS/x86_64/hubblestack-2.1.7-1.x86_64.rpm ../../dist/hubblestack-2.1.7-1.el6.x86_64.rpm
 rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el7.spec
-cp RPMS/x86_64/hubblestack-2.1.6-1.x86_64.rpm ../../dist/hubblestack-2.1.6-1.el7.x86_64.rpm
+cp RPMS/x86_64/hubblestack-2.1.7-1.x86_64.rpm ../../dist/hubblestack-2.1.7-1.el7.x86_64.rpm

pkg/hubble

@@ -1,17 +1,16 @@
 #!/bin/sh
 ### BEGIN INIT INFO
-# Provides:
+# Provides:          hubble
 # Required-Start:    $remote_fs $syslog
 # Required-Stop:     $remote_fs $syslog
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
-# Short-Description: Start daemon at boot time
-# Description:       Enable service provided by daemon.
+# Short-Description: Start hubble daemon
+# Description:       Start hubble daemon
 ### END INIT INFO
 
 dir="/etc/hubble"
 cmd="hubble -d"
-user="root"
 
 name=`basename $0`
 pid_file="/var/run/$name.pid"
@@ -32,11 +31,7 @@ case "$1" in
     else
         echo "Starting $name"
         cd "$dir"
-        if [ -z "$user" ]; then
-            sudo $cmd
-        else
-            sudo -u "$user" $cmd
-        fi
+        $cmd
         sleep 0.1
         if ! is_running; then
             echo "Unable to start, see $log_file"

pkg/scripts/osquery-build.sh

@@ -4,8 +4,13 @@ sudo chown -R $USER. temp
 cd temp
 git clone https://github.com/facebook/osquery.git
 cd osquery
+git checkout 2.3.2
 make sysprep
 make deps
+if [[ -n "$(python -mplatform | grep debian-7)" ]]; then
+    /usr/local/osquery/bin/brew untap homebrew/dupes
+    /usr/local/osquery/bin/brew link ncurses
+fi
 SKIP_TESTS=1 make -j 4
 make strip
 sudo cp -pr ./build/linux/osquery/osqueryi ./build/linux/osquery/osqueryd /opt/osquery

pkg/scripts/pip-install.sh

@@ -1 +1,5 @@
-  pip install -r pyinstaller-requirements.txt
+if [[ -n "$(python -mplatform | grep debian-7)" ]]; then
+    pip install -r pyinstaller-requirements-debian7.txt
+else
+    pip install -r pyinstaller-requirements.txt
+fi

pkg/scripts/pyinstaller-requirements-debian7.txt

@@ -0,0 +1,14 @@
+pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception
+Crypto
+pyopenssl
+argparse
+requests>=2.13.0
+logging
+pprint
+daemon
+boto3
+botocore
+salt-ssh
+gitpython
+pyinotify
+cffi

pkg/scripts/pyinstaller-requirements.txt

@@ -2,7 +2,7 @@ pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore
 Crypto
 pyopenssl
 argparse
-requests
+requests>=2.13.0
 logging
 pprint
 daemon

pkg/specs/hubblestack-el6.spec

@@ -9,7 +9,7 @@
 
 Summary: Hubblestack is a module, open-source security compliance framework
 Name: hubblestack
-Version: 2.1.6
+Version: 2.1.7
 Release: 1
 License: Apache 2.0
 Group: Development/Tools
@@ -54,6 +54,10 @@ rm -rf %{buildroot}
 /usr/bin/*
 
 %changelog
+* Fri Apr 7 2017  Colton Myers <[email protected]> 2.1.7-1
+- Force config and logs to 600 permissions to hide tokens
+- Splunk returners: Fix for hosts with misconfigured FQDN (no localhost IPs, please!)
+
 * Mon Apr 3 2017  Colton Myers <[email protected]> 2.1.6-1
 - Fix pulsar loading
 - Fix splay in scheduler

pkg/specs/hubblestack-el7.spec

@@ -9,7 +9,7 @@
 
 Summary: Hubblestack is a module, open-source security compliance framework
 Name: hubblestack
-Version: 2.1.6
+Version: 2.1.7
 Release: 1
 License: Apache 2.0
 Group: Development/Tools
@@ -54,6 +54,10 @@ rm -rf %{buildroot}
 /usr/lib/*
 
 %changelog
+* Fri Apr 7 2017  Colton Myers <[email protected]> 2.1.7-1
+- Force config and logs to 600 permissions to hide tokens
+- Splunk returners: Fix for hosts with misconfigured FQDN (no localhost IPs, please!)
+
 * Mon Apr 3 2017  Colton Myers <[email protected]> 2.1.6-1
 - Fix pulsar loading
 - Fix splay in scheduler

setup.py

@@ -42,7 +42,9 @@
         ],
     },
     install_requires=[
-        'salt >= 2015.5.0',
+        'salt-ssh >= 2015.8.0',
+        'gitpython',
+        'pyinotify',
     ],
     data_files=data_files,
     options={