Updates to dockerfile for dev, framework handling of dev appliance

pytest_store handling of no SSH client address
dockerignore for pytest_cache
iputils and iproute in quickstart system packages
SSHDummy property handling, explicitly None
building cfme-tests docker on cfmeqe_sel_stable

.dockerignore

@@ -0,0 +1 @@
+.pytest_cache

cfme/fixtures/pytest_store.py

@@ -133,13 +133,20 @@ def slaveid(self):
 
     @cached_property
     def my_ip_address(self):
+        """Make a few guesses at what the pytest host's address is
+        Last guess defaulting to localhost
+        """
         try:
             # Check the environment first
-            return os.environ['CFME_MY_IP_ADDRESS']
+            env_or_ssh = os.environ.get(
+                'CFME_MY_IP_ADDRESS',
+                self.current_appliance.ssh_client.client_address
+            )
+            return env_or_ssh or '127.0.0.1'
         except KeyError:
             # Fall back to having an appliance tell us what it thinks our IP
             # address is
-            return self.current_appliance.ssh_client.client_address()
+            return
 
     def write_line(self, line, **kwargs):
         return write_line(line, **kwargs)

cfme/scripting/quickstart/system.py

@@ -12,6 +12,7 @@
 HAS_DNF = os.path.exists('/usr/bin/dnf')
 HAS_YUM = os.path.exists('/usr/bin/yum')
 HAS_APT = os.path.exists('/usr/bin/apt-get')
+HAS_SUDO = os.path.exists('/usr/bin/sudo')
 
 OS_RELEASE_FILE = '/etc/os-release'
 OS_NAME_REGEX = r'^NAME="?([\w\s\/]*?)"?$'
@@ -36,7 +37,7 @@
 else:
     INSTALL_COMMAND = ''
 
-if not IS_ROOT:
+if not IS_ROOT and HAS_SUDO:
     INSTALL_COMMAND = 'sudo ' + INSTALL_COMMAND
     DEBUG_INSTALL_COMMAND = 'sudo ' + DEBUG_INSTALL_COMMAND if DEBUG_INSTALL_COMMAND else None
 
@@ -58,7 +59,7 @@
 RH_BASE = (
     " python2-virtualenv gcc postgresql-devel libxml2-devel libxslt-devel"
     " zeromq3-devel libcurl-devel redhat-rpm-config gcc-c++ openssl-devel"
-    " libffi-devel python-devel tesseract freetype-devel"
+    " libffi-devel python-devel tesseract freetype-devel iputils iproute"
     " python2-debuginfo python3-debuginfo git"
 )
 

cfme/utils/appliance/__init__.py

@@ -1800,8 +1800,7 @@ def wait_for_ssh(self, timeout=600):
         """
         if self.is_dev:
             return
-        wait_for(func=getattr,
-                 func_args=[self, 'is_ssh_running'],
+        wait_for(func=lambda: self.is_ssh_running,
                  message='appliance.is_ssh_running',
                  delay=5,
                  num_sec=timeout)
@@ -1841,8 +1840,7 @@ def wait_for_embedded_ansible(self, timeout=1200):
             timeout *= 2
 
         wait_for(
-            func=getattr,
-            func_args=[self, 'is_embedded_ansible_running'],
+            func=lambda: self.is_embedded_ansible_running,
             message='appliance.is_embedded_ansible_running',
             delay=60,
             num_sec=timeout
@@ -1860,8 +1858,7 @@ def get_host_address(self):
 
     def wait_for_host_address(self):
         try:
-            wait_for(func=getattr,
-                     func_args=[self, 'get_host_address'],
+            wait_for(func=lambda: self.get_host_address,
                      fail_condition=None,
                      delay=5,
                      num_sec=120,
@@ -1915,6 +1912,8 @@ def is_idle(self):
 
     @cached_property
     def build_datetime(self):
+        if self.is_dev:
+            return datetime.now()
         build_datetime_string = self.build.split('_', 1)[0]
         return datetime.strptime(build_datetime_string, '%Y%m%d%H%M%S')
 

cfme/utils/browser.py

@@ -146,6 +146,7 @@ def _firefox_profile(self):
 
     def processed_browser_args(self):
         self._add_missing_options()
+        log.info('BROWSER KWARGS: {}'.format(self.browser_kwargs))
         return self.browser_kwargs
 
     def create(self, url_key):
@@ -258,7 +259,7 @@ def from_conf(cls, browser_conf):
                 if browser_conf[
                         'webdriver_options'][
                             'desired_capabilities']['browserName'].lower() == 'chrome':
-                    browser_kwargs['desired_capabilities']['chromeOptions'] = {}
+                    browser_kwargs['desired_capabilities']['chromeOptions'] = {'w3c': False}
                     browser_kwargs[
                         'desired_capabilities']['chromeOptions']['args'] = ['--no-sandbox',
                                                                             '--start-maximized',

cfme/utils/ssh.py

@@ -174,6 +174,18 @@ def vmdb_version(self):
     def username(self):
         return self._connect_kwargs.get('username')
 
+    @property
+    def connected(self):
+        return self._transport and self._transport.active
+
+    @property
+    def client_address(self):
+        res = self.run_command('echo $SSH_CLIENT', ensure_host=True, ensure_user=True)
+        # SSH_CLIENT format is 'clientip clientport serverport', we want clientip
+        if not res.output:
+            raise Exception('unable to get client address via SSH')
+        return res.output.split()[0]
+
     def __repr__(self):
         return "<SSHClient hostname={} port={}>".format(
             repr(self._connect_kwargs.get("hostname")),
@@ -218,10 +230,6 @@ def close(self):
         except (AttributeError, ValueError):
             pass
 
-    @property
-    def connected(self):
-        return self._transport and self._transport.active
-
     def connect(self, hostname=None, **kwargs):
         if self.is_dev:
             raise Exception('SSH is not allowed using a dev appliance!')
@@ -655,13 +663,6 @@ def uptime(self):
 
         return 0
 
-    def client_address(self):
-        res = self.run_command('echo $SSH_CLIENT', ensure_host=True, ensure_user=True)
-        # SSH_CLIENT format is 'clientip clientport serverport', we want clientip
-        if not res.output:
-            raise Exception('unable to get client address via SSH')
-        return res.output.split()[0]
-
     def appliance_has_netapp(self):
         return self.run_command("stat /var/www/miq/vmdb/HAS_NETAPP").success
 
@@ -793,6 +794,9 @@ class SSHDummy(object):
     This is for dev appliances that don't support ssh
     Framework/tests that use appliance.ssh_client will end up just logging what they would have run
     """
+    def __repr__(self):
+        return "<SSHDummy for dev appliance>"
+
     def __enter__(self, *args, **kwargs):
         return self
 
@@ -804,12 +808,16 @@ def is_appliance_downstream(self):
         return False
 
     def __getattr__(self, item):
+        run_warn = 'Cannot run ssh against a dev appliance'
         if item in ['run_command', 'run_rails_command', 'run_rails_console', 'run_rake_command']:
-            logger.error('Cannot run ssh against a dev appliance, SSH function call skipped: %s',
-                         item)
+            logger.warning('%s, SSH function call skipped: %s', run_warn, item)
             from functools import partial
             return partial(SSHResultDummy, rc=1, output=None)
 
+        if item in [name for name, p in SSHClient.__dict__.items() if isinstance(p, property)]:
+            logger.warning('%s, SSH property call skipped: %s', run_warn, item)
+            return None
+
 
 def keygen():
     """Generate temporary ssh keypair for appliance SSH auth

conf/env.yaml.template

@@ -8,7 +8,7 @@ browser:
             browserName: 'chrome'
             unexpectedAlertBehaviour: 'ignore'
 github:
-    default_repo: foo/bar
+    default_repo: manageiq/integration_tests
     token: abcdef0123456789
 bugzilla:
     url: https://bugzilla.redhat.com/xmlrpc.cgi

dockerfiles/cfme_tests_dev/Dockerfile

@@ -1,100 +1,27 @@
-FROM registry.fedoraproject.org/fedora:26
+# Intended build context is integration_tests top level
 
-RUN echo "tsflags=nodocs" >> /etc/yum.conf
+FROM dima-sel-base:latest
 
-COPY ../docker-assets/google-chrome.repo /etc/yum.repos.d/
+ENV PYCURL_SSL_LIBRARY=nss \
+    PYTHONDONTWRITEBYTECODE=1 \
+    VENV=/cfme_venv \
+    CFME_ENV=/integration_tests
 
-RUN dnf install -y http://linuxdownload.adobe.com/linux/x86_64/adobe-release-x86_64-1.0-1.noarch.rpm && \
-    rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-adobe-linux && \
-    dnf install -y \
-    # Base packages
-    bzip2 \
-    dejavu* \
-    findutils \
-    flash-plugin \
-    fluxbox \
-    freetype-devel \
-    gcc \
-    gcc-c++ \
-    git \
-    gtk2 \
-    java-1.8.0-openjdk.x86_64 \
-    libcurl-devel \
-    libffi-devel \
-    libpng-devel \
-    libxml2-devel \
-    libxslt-devel \
-    openssl-devel \
-    passwd \
-    postgresql-devel \
-    python-devel \
-    python-netaddr \
-    python-pip \
-    python-setuptools \
-    python2-virtualenv \
-    redhat-rpm-config \
-    sshpass \
-    sudo \
-    tigervnc-server \
-    unzip \
-    which \
-    xorg-x11-fonts-* \
-    xterm \
-    zeromq-devel && \
-    dnf install -y https://dl.google.com/linux/direct/google-chrome-stable_current_x86_64.rpm && \
-    dnf clean all
+COPY ./cfme/scripting $CFME_ENV/cfme/scripting/
+COPY ./requirements.txt $CFME_ENV/
+COPY requirements/ $CFME_ENV/
+WORKDIR $CFME_ENV
+# for package installs through quickstart
+USER 0
+RUN python2 -m cfme.scripting.quickstart --mk-virtualenv $VENV
+RUN source $VENV/bin/activate && pip freeze && pip install -e .
 
-ENV CHROME_PATH /opt/google/chrome
-ENV CHROMEDRIVER_VERSION 2.35
-ENV CHROMEDRIVER_PATH /opt/chromedriver/
-ENV SELENIUM_VERSION 2.53
-ENV SELENIUM_PATH /opt/selenium
-ENV FIREFOX_VERSION 45.5.0esr
-ENV FIREFOX_PATH /opt/firefox
+# integration_tests volume mounted at run time to pickup real-time changes
+VOLUME $CFME_ENV
+WORKDIR $CFME_ENV
 
-# Chrome driver
-RUN mkdir -p ${CHROMEDRIVER_PATH} && \
-    cd ${CHROMEDRIVER_PATH} && \
-    curl -O http://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip && \
-    unzip chromedriver_linux64.zip && \
-    chmod a+x chromedriver && \
-    rm -f chromedriver_linux64.zip
+COPY ./dockerfiles/cfme_tests_dev/docker-assets/entrypoint /entrypoint
 
-# Selenium
-RUN mkdir -p ${SELENIUM_PATH} && \
-    cd ${SELENIUM_PATH} && \
-    curl http://selenium-release.storage.googleapis.com/${SELENIUM_VERSION}/selenium-server-standalone-${SELENIUM_VERSION}.0.jar -o selenium_latest.jar && \
-    chmod ugo+r selenium_latest.jar
 EXPOSE 5999
 
-# Firefox
-RUN mkdir -p ${FIREFOX_PATH} && \
-    cd ${FIREFOX_PATH} && \
-    curl https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -o firefox.tar.bz2 && \
-    tar -C . -xjvf firefox.tar.bz2 --strip-components 1 && \
-    rm -f firefox.tar.bz2
-
-ENV PATH="${FIREFOX_PATH}:${CHROME_PATH}:${CHROMEDRIVER_PATH}:${SELENIUM_PATH}:${PATH}"
-
-ENV PYCURL_SSL_LIBRARY=nss
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV CFME_ENV=/
-
-RUN virtualenv /cfme_venv
-RUN echo "source /cfme_venv/bin/activate" >> /root/.bashrc
-
-# Preinstall any python dependencies to keep it in an early layer
-COPY ../requirements/frozen.py2.txt frozen.py2.txt
-RUN /cfme_venv/bin/pip install --no-cache-dir -U pip wheel setuptools_scm docutils && \
-    /cfme_venv/bin/pip install --no-cache-dir -r frozen.py2.txt --no-binary pycurl && \
-    rm -rf ~/.cache/pip && \
-    find . -name *.pyc -delete && \
-    find . -name __pycache__ -delete
-
-VOLUME /projects/cfme_env/cfme_vol
-WORKDIR /projects/cfme_env/cfme_vol
-
-COPY ../docker-assets/xstartup /xstartup
-COPY ../docker-assets/entrypoint /entrypoint
-
 ENTRYPOINT ["/entrypoint"]

dockerfiles/cfme_tests_dev/docker-assets/entrypoint

@@ -1,14 +1,13 @@
 #!/bin/bash
+nginx &
+HOME=/home/selenium vncserver :99 -fg -Log *:stderr:100 &
 
 # Add host.docker.internal if it does not already exist
 ping -c 1 host.docker.internal > /dev/null
 [[ $? -ne 0 ]] && ip -4 route list match 0/0 |awk '{print $3 " host.docker.internal"}' >> /etc/hosts
 
-touch .yaml_key
-mkdir -p /cfme-qe-yamls/complete
-cp conf/* /cfme-qe-yamls/complete
-
-vncserver :99 -xstartup /xstartup -SecurityTypes None -geometry 1280x1024
+cd /integration_tests
+git init
 
 source /cfme_venv/bin/activate
 pip install -e .