diff --git a/README.md b/README.md index 751b1941f..8ce8902ae 100644 --- a/README.md +++ b/README.md @@ -3,45 +3,51 @@ An alternate version of Hubblestack which can be run without an existing SaltStack infrastructure. -# Building standalone packages (CentOS) +# Packaging / Installing + +## Installing using setup.py + +```bash +sudo yum install git -y +git clone https://github.com/hubblestack/hubble +cd hubble +sudo python setup.py install +``` + +Installs a `hubble` "binary" into `/usr/bin/`. + +## Building standalone packages (CentOS) ```bash sudo yum install git -y -git clone git://github.com/hubblestack/hubble ~/hubble -cd ~/hubble/pkg +git clone https://github.com/hubblestack/hubble +cd hubble/pkg ./build_rpms.sh # note the lack of sudo, that is important ``` -Package will be in the `~/el6/` and `~/el7` directory. The only difference +Packages will be in the `hubble/pkg/dist/` directory. The only difference between the packages is the inclusion of `/etc/init.d/hubble` for el6 and the inclusion of a systemd unit file for el7. There's no guarantee of glibc compatibility. -# Building dep-heavy cross-platform packages +## Building standalone packages (Debian) ```bash sudo yum install git -y -git clone git://github.com/hubblestack/hubble -cd hubble -python setup.py bdist_rpm +git clone https://github.com/hubblestack/hubble +cd hubble/pkg +./build_debs.sh # note the lack of sudo, that is important ``` -You'll find the generated RPM in the `dist/` folder. +Package will be in the `hubble/pkg/dist/` directory. There's no guarantee of +glibc compatibility. +# Usage -# Testing +A config template has been placed in `/etc/hubble/hubble`. Modify it to your +specifications and needs. -You can do `hubble -h` to see the available options. Here's a sample working -config you can place in `/etc/hubble/hubble`. Note that you'll need to install -python-pygit2 to get gitfs working: - -``` -gitfs_remotes: - - https://github.com/hubblestack/hubblestack_data.git -fileserver_backend: - - roots - - git -``` +You can do `hubble -h` to see the available options. The first two commands you should run to make sure things are set up correctly are `hubble --version` and `hubble test.ping`. If those run without issue @@ -98,8 +104,8 @@ schedule: run_on_start: True ``` -Note that you need to have your splunk_nova_return configured in order to use -the above block: +Note that you need to have your hubblestack splunk returner configured in order +to use the above block: ``` hubblestack: @@ -112,3 +118,10 @@ hubblestack: sourcetype_nebula: hubble_osquery sourcetype_pulsar: hubble_fim ``` + +When using the scheduler, you can just run `hubble` in the foreground, or use +the included sysvinit and systemd files to run it as a service in the +background. You can also start it as a daemon without any scripts by using the +`-d` argument. + +Use `-vvv` to turn on debug logging. diff --git a/conf/hubble b/conf/hubble index 2badbdeb1..6dbe16f6e 100644 --- a/conf/hubble +++ b/conf/hubble @@ -10,8 +10,8 @@ gitfs_remotes: - https://github.com/hubblestack/hubblestack_data.git fileserver_backend: - - git - roots + - git ################################# ## Scheduler Config diff --git a/hubblestack/__init__.py b/hubblestack/__init__.py index da04cc33c..bc6379c18 100644 --- a/hubblestack/__init__.py +++ b/hubblestack/__init__.py @@ -1 +1 @@ -__version__ = '2.1.6' +__version__ = '2.1.7' diff --git a/hubblestack/daemon.py b/hubblestack/daemon.py index 5452858f7..3081f793e 100644 --- a/hubblestack/daemon.py +++ b/hubblestack/daemon.py @@ -311,6 +311,9 @@ def load_config(): salt.log.setup.setup_console_logger(__opts__['log_level']) salt.log.setup.setup_logfile_logger(__opts__['log_file'], __opts__['log_level']) + # 384 is 0o600 permissions, written without octal for python 2/3 compat + os.chmod(__opts__['log_file'], 384) + os.chmod(parsed_args.get('configfile'), 384) __grains__ = salt.loader.grains(__opts__) __pillar__ = {} diff --git a/hubblestack/extmods/returners/splunk_nebula_return.py b/hubblestack/extmods/returners/splunk_nebula_return.py index 50a3a15cf..5015d3c48 100644 --- a/hubblestack/extmods/returners/splunk_nebula_return.py +++ b/hubblestack/extmods/returners/splunk_nebula_return.py @@ -89,6 +89,11 @@ def returner(ret): fqdn_ip4 = __grains__['fqdn_ip4'][0] except IndexError: fqdn_ip4 = __grains__['ipv4'][0] + if fqdn_ip4.startswith('127.'): + for ip4_addr in __grains__['ipv4']: + if ip4_addr and not ip4_addr.startswith('127.'): + fqdn_ip4 = ip4_addr + break if not data: return diff --git a/hubblestack/extmods/returners/splunk_nova_return.py b/hubblestack/extmods/returners/splunk_nova_return.py index 2aed38873..acb220138 100644 --- a/hubblestack/extmods/returners/splunk_nova_return.py +++ b/hubblestack/extmods/returners/splunk_nova_return.py @@ -88,6 +88,11 @@ def returner(ret): fqdn_ip4 = __grains__['fqdn_ip4'][0] except IndexError: fqdn_ip4 = __grains__['ipv4'][0] + if fqdn_ip4.startswith('127.'): + for ip4_addr in __grains__['ipv4']: + if ip4_addr and not ip4_addr.startswith('127.'): + fqdn_ip4 = ip4_addr + break if __grains__['master']: master = __grains__['master'] diff --git a/hubblestack/extmods/returners/splunk_pulsar_return.py b/hubblestack/extmods/returners/splunk_pulsar_return.py index 1f713aae1..7989514f4 100644 --- a/hubblestack/extmods/returners/splunk_pulsar_return.py +++ b/hubblestack/extmods/returners/splunk_pulsar_return.py @@ -97,6 +97,11 @@ def returner(ret): fqdn_ip4 = __grains__['fqdn_ip4'][0] except IndexError: fqdn_ip4 = __grains__['ipv4'][0] + if fqdn_ip4.startswith('127.'): + for ip4_addr in __grains__['ipv4']: + if ip4_addr and not ip4_addr.startswith('127.'): + fqdn_ip4 = ip4_addr + break alerts = [] for item in data: diff --git a/pkg/build_debs.sh b/pkg/build_debs.sh index 74abce611..585f6e58f 100755 --- a/pkg/build_debs.sh +++ b/pkg/build_debs.sh @@ -22,14 +22,14 @@ mkdir -p dist bash ./init_pkg.sh -y cp ../hubble.tar.gz dist/hubble.tar.gz mv ../hubble.tar.gz build/hubble.tar.gz -mkdir build/hubblestack-2.1.6 -tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.6 -mkdir -p build/hubblestack-2.1.6/etc/init.d -cp ./hubble build/hubblestack-2.1.6/etc/init.d -mkdir -p build/hubblestack-2.1.6/usr/lib/systemd/system -cp ./hubble.service build/hubblestack-2.1.6/usr/lib/systemd/system -cp -f ../conf/hubble build/hubblestack-2.1.6/etc/hubble/hubble -cd build/hubblestack-2.1.6 +mkdir build/hubblestack-2.1.7 +tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.7 +mkdir -p build/hubblestack-2.1.7/etc/init.d +cp ./hubble build/hubblestack-2.1.7/etc/init.d +mkdir -p build/hubblestack-2.1.7/usr/lib/systemd/system +cp ./hubble.service build/hubblestack-2.1.7/usr/lib/systemd/system +cp -f ../conf/hubble build/hubblestack-2.1.7/etc/hubble/hubble +cd build/hubblestack-2.1.7 sudo apt-get install -y ruby ruby-dev rubygems gcc make sudo gem install --no-ri --no-rdoc fpm @@ -39,9 +39,9 @@ ln -s /opt/osquery/osqueryd usr/bin/osqueryd ln -s /opt/osquery/osqueryi usr/bin/osqueryi fpm -s dir -t deb \ -n hubblestack \ - -v 2.1.6-1 \ + -v 2.1.7-1 \ -d 'git' \ --config-files /etc/hubble/hubble --config-files /etc/osquery/osquery.conf \ --deb-no-default-config-files \ etc/hubble etc/osquery etc/init.d opt usr/bin -cp hubblestack_2.1.6-1_amd64.deb ../../dist/ +cp hubblestack_2.1.7-1_amd64.deb ../../dist/ diff --git a/pkg/build_rpms.sh b/pkg/build_rpms.sh index 44d264801..93e1cdabe 100755 --- a/pkg/build_rpms.sh +++ b/pkg/build_rpms.sh @@ -22,23 +22,23 @@ mkdir -p dist bash ./init_pkg.sh -y cp ../hubble.tar.gz dist/hubble.tar.gz mv ../hubble.tar.gz build/hubble.tar.gz -mkdir build/hubblestack-2.1.6 -tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.6 -mkdir -p build/hubblestack-2.1.6/etc/init.d -cp ./hubble build/hubblestack-2.1.6/etc/init.d -mkdir -p build/hubblestack-2.1.6/usr/lib/systemd/system -cp ./hubble.service build/hubblestack-2.1.6/usr/lib/systemd/system -cp -f ../conf/hubble build/hubblestack-2.1.6/etc/hubble/hubble +mkdir build/hubblestack-2.1.7 +tar -xzvf build/hubble.tar.gz -C build/hubblestack-2.1.7 +mkdir -p build/hubblestack-2.1.7/etc/init.d +cp ./hubble build/hubblestack-2.1.7/etc/init.d +mkdir -p build/hubblestack-2.1.7/usr/lib/systemd/system +cp ./hubble.service build/hubblestack-2.1.7/usr/lib/systemd/system +cp -f ../conf/hubble build/hubblestack-2.1.7/etc/hubble/hubble cd build -tar -czvf hubblestack-2.1.6.tar.gz hubblestack-2.1.6/ +tar -czvf hubblestack-2.1.7.tar.gz hubblestack-2.1.7/ mkdir -p rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp} -cp hubblestack-2.1.6.tar.gz rpmbuild/SOURCES/ +cp hubblestack-2.1.7.tar.gz rpmbuild/SOURCES/ cd rpmbuild cp ../../specs/* SPECS/ rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el6.spec -cp RPMS/x86_64/hubblestack-2.1.6-1.x86_64.rpm ../../dist/hubblestack-2.1.6-1.el6.x86_64.rpm +cp RPMS/x86_64/hubblestack-2.1.7-1.x86_64.rpm ../../dist/hubblestack-2.1.7-1.el6.x86_64.rpm rpmbuild --define "_topdir $(pwd)" --define "_tmppath %{_topdir}/tmp" -ba SPECS/hubblestack-el7.spec -cp RPMS/x86_64/hubblestack-2.1.6-1.x86_64.rpm ../../dist/hubblestack-2.1.6-1.el7.x86_64.rpm +cp RPMS/x86_64/hubblestack-2.1.7-1.x86_64.rpm ../../dist/hubblestack-2.1.7-1.el7.x86_64.rpm diff --git a/pkg/hubble b/pkg/hubble index 8713430be..fe940e84b 100755 --- a/pkg/hubble +++ b/pkg/hubble @@ -1,17 +1,16 @@ #!/bin/sh ### BEGIN INIT INFO -# Provides: +# Provides: hubble # Required-Start: $remote_fs $syslog # Required-Stop: $remote_fs $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: Start daemon at boot time -# Description: Enable service provided by daemon. +# Short-Description: Start hubble daemon +# Description: Start hubble daemon ### END INIT INFO dir="/etc/hubble" cmd="hubble -d" -user="root" name=`basename $0` pid_file="/var/run/$name.pid" @@ -32,11 +31,7 @@ case "$1" in else echo "Starting $name" cd "$dir" - if [ -z "$user" ]; then - sudo $cmd - else - sudo -u "$user" $cmd - fi + $cmd sleep 0.1 if ! is_running; then echo "Unable to start, see $log_file" diff --git a/pkg/scripts/osquery-build.sh b/pkg/scripts/osquery-build.sh index 12f8f6c0e..9dbd30fb0 100644 --- a/pkg/scripts/osquery-build.sh +++ b/pkg/scripts/osquery-build.sh @@ -4,8 +4,13 @@ sudo chown -R $USER. temp cd temp git clone https://github.com/facebook/osquery.git cd osquery +git checkout 2.3.2 make sysprep make deps +if [[ -n "$(python -mplatform | grep debian-7)" ]]; then + /usr/local/osquery/bin/brew untap homebrew/dupes + /usr/local/osquery/bin/brew link ncurses +fi SKIP_TESTS=1 make -j 4 make strip sudo cp -pr ./build/linux/osquery/osqueryi ./build/linux/osquery/osqueryd /opt/osquery diff --git a/pkg/scripts/pip-install.sh b/pkg/scripts/pip-install.sh index 70b5ca717..9753025aa 100644 --- a/pkg/scripts/pip-install.sh +++ b/pkg/scripts/pip-install.sh @@ -1 +1,5 @@ - pip install -r pyinstaller-requirements.txt +if [[ -n "$(python -mplatform | grep debian-7)" ]]; then + pip install -r pyinstaller-requirements-debian7.txt +else + pip install -r pyinstaller-requirements.txt +fi diff --git a/pkg/scripts/pyinstaller-requirements-debian7.txt b/pkg/scripts/pyinstaller-requirements-debian7.txt new file mode 100644 index 000000000..0c24be762 --- /dev/null +++ b/pkg/scripts/pyinstaller-requirements-debian7.txt @@ -0,0 +1,14 @@ +pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore exception +Crypto +pyopenssl +argparse +requests>=2.13.0 +logging +pprint +daemon +boto3 +botocore +salt-ssh +gitpython +pyinotify +cffi diff --git a/pkg/scripts/pyinstaller-requirements.txt b/pkg/scripts/pyinstaller-requirements.txt index e20bf321d..52ab646f0 100644 --- a/pkg/scripts/pyinstaller-requirements.txt +++ b/pkg/scripts/pyinstaller-requirements.txt @@ -2,7 +2,7 @@ pyinstaller==3.2 # currently 3.2.1 version is not supported because of botocore Crypto pyopenssl argparse -requests +requests>=2.13.0 logging pprint daemon diff --git a/pkg/specs/hubblestack-el6.spec b/pkg/specs/hubblestack-el6.spec index 461b9d82f..87dd1b083 100644 --- a/pkg/specs/hubblestack-el6.spec +++ b/pkg/specs/hubblestack-el6.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.1.6 +Version: 2.1.7 Release: 1 License: Apache 2.0 Group: Development/Tools @@ -54,6 +54,10 @@ rm -rf %{buildroot} /usr/bin/* %changelog +* Fri Apr 7 2017 Colton Myers 2.1.7-1 +- Force config and logs to 600 permissions to hide tokens +- Splunk returners: Fix for hosts with misconfigured FQDN (no localhost IPs, please!) + * Mon Apr 3 2017 Colton Myers 2.1.6-1 - Fix pulsar loading - Fix splay in scheduler diff --git a/pkg/specs/hubblestack-el7.spec b/pkg/specs/hubblestack-el7.spec index 26780eb2c..e34d93139 100644 --- a/pkg/specs/hubblestack-el7.spec +++ b/pkg/specs/hubblestack-el7.spec @@ -9,7 +9,7 @@ Summary: Hubblestack is a module, open-source security compliance framework Name: hubblestack -Version: 2.1.6 +Version: 2.1.7 Release: 1 License: Apache 2.0 Group: Development/Tools @@ -54,6 +54,10 @@ rm -rf %{buildroot} /usr/lib/* %changelog +* Fri Apr 7 2017 Colton Myers 2.1.7-1 +- Force config and logs to 600 permissions to hide tokens +- Splunk returners: Fix for hosts with misconfigured FQDN (no localhost IPs, please!) + * Mon Apr 3 2017 Colton Myers 2.1.6-1 - Fix pulsar loading - Fix splay in scheduler diff --git a/setup.py b/setup.py index 8ee5f6256..e28e07381 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,9 @@ ], }, install_requires=[ - 'salt >= 2015.5.0', + 'salt-ssh >= 2015.8.0', + 'gitpython', + 'pyinotify', ], data_files=data_files, options={