initial Ios compatibility

.gitignore

@@ -2,3 +2,4 @@
 report
 bin/sampleinput/limobilelogin.json
 env/
+*.swp

README.md

@@ -47,6 +47,11 @@ Mobster provides a simple way for developers to record crucial web performance d
 
 4. In the Chrome settings, enable USB Debugging (Advanced -> Developer Tools -> Enable USB Web Debugging)
 
+#### iOS-Specific Prerequisites ####
+1. Install the [iOS WebKit Debugging Proxy](https://github.com/google/ios-webkit-debug-proxy) on your computer. 
+
+2. In the Safari settings, enable Web Inspector (Safari Settings -> Advanced -> Web Inspector)
+
 ### Run Mobster with an Android Device ###
 
 1. Execute the command below under the Android SDK folder:
@@ -87,6 +92,16 @@ Mobster provides a simple way for developers to record crucial web performance d
 **Important Note:**
 If you use Chrome as your web browser normally, it will be annoying to run Mobster with Chrome because Mobster by default uses one of the currently open tab(s) for testing and also clears cookies, etc. This means that, at the end of a test, one of your open tab(s) will be showing the final web page from your test and you will be logged out of all websites. **An easy way to avoid this problem is to run Mobster with [Chromium](http://www.chromium.org/Home) or [Chrome Canary](https://www.google.com/intl/en/chrome/browser/canary.html) so your normal browsing is not affected.** Chrome, Chromium, and Chrome Canary can all be installed side-by-side.
 
+### Run Mobster with an iOS Device ###
+
+
+1. Open Safari on the device. Note that **Mobster will clear cookies** and other browsing data. The tab currently being viewed will be used by Mobster to navigate to webpages. Currently, Safari must be visible on the screen for Mobster to work.
+
+2. Run the main Mobster script with a sample flow in your Mobster home directory. Note the --ios argument, which tells Mobster to start the iOS webkit debugging proxy (which **must** be installed).
+
+    <pre>./bin/mobster.py -t bin/sampleinput/sample.json -p -b --ios</pre>
+
+    Mobster reports will be generated in the MOBSTER_HOME/report folder if no folder is specified. Run mobster.py with the "-h" option to learn more about command-line options. To learn how to make your own flows, look at the scripts in the bin/sampleinput/ directory.
 
 ## Contribution ##
 

bin/mobster.py

@@ -8,6 +8,7 @@
 import json
 import logging
 import os
+import signal
 import subprocess
 import sys
 import time
@@ -34,18 +35,34 @@ def run(args):
   Run a test with the specified parameters, and return the HTTP Archive
   (HAR) File represented as a dictionary.
   """
+  if args.ios:
+    kill_existing_proxy_processes()
+    DEVNULL = open(os.devnull, 'wb')
+    p = subprocess.Popen('ios_webkit_debug_proxy', stdout=DEVNULL, stderr=DEVNULL)
+    logging.info("Giving proxy time to start") 
+    time.sleep(2)
   har_gen = FlowProfiler(args.testfile, int(args.iterations))   \
             if args.iterations else FlowProfiler(args.testfile)
 
   # profiling_results is a list of lists containing HARs for each page in a run
   profiling_results = har_gen.profile()
-
+  if args.ios:
+    p.terminate()
   if args.average:
     return [merge_by_average(page_results) \
             for page_results in zip(*profiling_results)]
   else:
     return profiling_results[-1]
 
+def kill_existing_proxy_processes():
+  p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
+  out, err = p.communicate()
+
+  for line in out.splitlines():
+    if 'ios_webkit' in line:
+      pid = int(line.split(None, 1)[0])
+      os.kill(pid, signal.SIGKILL)
+
 def write_report(args):
   """
   Autogenerates an HTML report and writes it to a file, with location and input
@@ -158,6 +175,8 @@ def parse_args():
 
   arg_parser.add_argument('-b', '--browser', action='store_true', \
     help='Open HTML report in browser after creation')
+
+  arg_parser.add_argument('--ios', action='store_true', help='Run iOS debug proxy')
 
   # Used if and only if generating report with results from previous test
   arg_parser.add_argument('-r', '--har', \

bin/sampleinput/googlesearch.json → bin/sampleinput/googlesearch_dchrome.json

@@ -1,3 +1,7 @@
+/*
+ * Searches for "LinkedIn" in Google. Only tested on Desktop Chrome.
+ */
+
 {
     "name" : "Google Search Flow",
     "navigations": [

src/linkedin/mobster/webkitcommunicator.py → src/linkedin/mobster/comm/webkitcommunicator.py

@@ -1,3 +1,4 @@
+from abc import ABCMeta, abstractmethod
 from collections import defaultdict
 import json
 import logging
@@ -11,7 +12,7 @@
 
 from linkedin.mobster.mobsterconfig import config
 
-class RemoteWebKitCommunicator(WebSocketClient):
+class WebKitCommunicator(WebSocketClient):
   """
   Asynchronous interface for communicating with a remote WebKit-based browser 
   via remote debugging protocol. Currently tested only on desktop and Android 
@@ -27,14 +28,22 @@ class RemoteWebKitCommunicator(WebSocketClient):
   """
 
   def __init__(self, page_num = 0):
+    self._page_num = page_num
     self._counter = 0
     self._response_callbacks = {}
     self._domain_callbacks = defaultdict(lambda: {})
     self._stopped = False
     self._command_queue = Queue()
+    debug_ws_url = self.get_ws_debug_url()
+    super(WebKitCommunicator, self).__init__(debug_ws_url)
+    self.start()
+
+  def get_ws_debug_url(self):
+    """
+    Returns the WebSocket debugging URL used by the browser for the specified
+    page number.
+    """
 
-    # Access list of open browser pages and pick the page with the specified 
-    # index
     url = 'http://localhost:{0}/json'.format(config["WS_DEBUG_PORT"])
     try:
       response = urllib2.urlopen(url).read()
@@ -44,15 +53,12 @@ def __init__(self, page_num = 0):
       sys.exit()
 
     page_info = json.loads(response)
-    page = page_info[page_num]
-    debug_ws_url = page['webSocketDebuggerUrl']
+    return page_info[self._page_num]['webSocketDebuggerUrl']
 
-    super(RemoteWebKitCommunicator, self).__init__(debug_ws_url)
-    self.start()
 
   def opened(self): pass
   def closed(self, code, reason=None): pass
-
+  
   def received_message(self, messageData):
     """Called whenever the WebSocket receives a message"""
     response = json.loads(str(messageData))

src/linkedin/mobster/har/flowprofiler.py

@@ -1,19 +1,21 @@
 import json
+import logging
+import time
 
+from linkedin.mobster.comm.webkitcommunicator import WebKitCommunicator
 from linkedin.mobster.har.css import CSSProfileParser
 from linkedin.mobster.har.network import NetworkEventHandler
 from linkedin.mobster.har.page import PageEventHandler, PageLoadNotifier
 from linkedin.mobster.har.timeline import TimelineEventHandler
 from linkedin.mobster.utils import wait_until, format_time
 from linkedin.mobster.webkitclient import RemoteWebKitClient
-from linkedin.mobster.webkitcommunicator import RemoteWebKitCommunicator
 
 class FlowProfiler(RemoteWebKitClient):
   """
   Runs a flow and records profiling information for each navigation. Generates HAR file.
   """
   def __init__(self, test_file, iterations=1):
-    super(FlowProfiler, self).__init__(RemoteWebKitCommunicator())
+    super(FlowProfiler, self).__init__(WebKitCommunicator())
     assert iterations > 0, "iterations must be a positive integer"
     self._iterations = iterations
     self._page_event_handler = None
@@ -39,7 +41,10 @@ def profile(self):
 
     for x in range(0, self._iterations):
       hars = []
-      self.clear_http_cache()
+      if self.can_clear_http_cache():
+        self.clear_http_cache()
+      else:
+        logging.info('Not clearing browser cache')
       self.clear_cookies()
 
       # Navigate to about:blank, to reset memory, etc.
@@ -51,7 +56,7 @@ def profile(self):
 
       for navigation in self._test['navigations']:
         assert len(navigation) > 0, 'Each navigation must have at least one action'
-
+        self._start_time = time.time()
         # do all the actions except the last one, because the last action causes the actual page navigation
         for i in range(0, len(navigation) - 1):
           self.process_action(navigation[i])
@@ -150,10 +155,23 @@ def get_page_timings(self):
     Returns a dictionary containing onContentLoad and onLoad times, both in (whole) milliseconds from
     the start of the request
     """
-    browser_timings = self.get_window_performance()
+    #FIXME the two different methods of calculating timings need to be merged
     first_request_start = self._network_event_handler.get_first_request_time()
-    return {
-      'onContentLoad': max(int(browser_timings['domContentLoadedEventEnd'] - (first_request_start * 1000)), -1),
-      'onLoad' : max(int(browser_timings['loadEventEnd'] - (first_request_start * 1000)), -1)
-    }
+    if self.get_os_name() == 'iOS':
+      onload = int(1000 * \
+        (self._page_event_handler.dom_content_load - self._start_time))
+      domcontent = int(1000 * \
+        (self._page_event_handler.on_load - self._start_time))
+
+      return {
+        'onContentLoad': max(domcontent, -1),
+        'onLoad' : max(onload, -1)
+      } 
+    else:
+      browser_timings = self.get_window_performance()
+      return {
+        'onContentLoad': max(int(browser_timings['domContentLoadedEventEnd'] - (first_request_start * 1000)), -1),
+        'onLoad' : max(int(browser_timings['loadEventEnd'] - (first_request_start * 1000)), -1)
+      }
+
 

src/linkedin/mobster/har/network.py

@@ -169,7 +169,7 @@ def process_response_received(self, message):
     # timings are not included for about: url's
     if not self._requests[request_id]['url'].startswith('about:'):
       if not 'timing' in params['response']:
-        log.info('No timing information in message')
+        logging.info('No timing information in message')
         return
 
       provided_timings = params['response']['timing']

src/linkedin/mobster/har/page.py

@@ -10,6 +10,10 @@ def process_event(self, message):
     if message['method'] == 'Page.loadEventFired':
       logging.info('Page.loadEventFired recorded')
       self.page_loaded = True
+      self.on_load = message['params']['timestamp']
+    elif message['method'] == 'Page.domContentEventFired':
+      logging.info('Page.domContentEventFired recorded')
+      self.dom_content_load = message['params']['timestamp']
 
 
 class PageLoadNotifier(object):
@@ -30,7 +34,6 @@ def process_page_event(self, message):
       logging.info('Page.loadEventFired recorded')
       self._received_page_load_event = True
 
-
   def process_network_event(self, message):
     self._last_network_event_time = time.time()
 

src/linkedin/mobster/webkitclient.py

@@ -72,8 +72,11 @@ def response_handler(response):
       elif 'result' in response and 'result' in response['result']:
         if 'wasThrown' in response['result']['result']:
           logging.error('Received error after running JS: {0}\n{1}'.format(js, response['result']))
-
-        self._js_result = response['result']['result']['value']
+
+        try:
+          self._js_result = response['result']['result']['value']
+        except KeyError:
+          logging.error('Javascript failed: {0}'.format(js))
         self._got_js_result = True
 
     self._communicator.send_cmd('Runtime.evaluate',
@@ -474,8 +477,9 @@ def get_os_version(self):
     if self._device_is_android():
       return re.search('Android\s+([\d\.]+)',self.get_user_agent()).groups()[0]
     elif self._device_is_ios():
-      start = self.get_user_agent().index('OS')
-      return self.get_user_agent()[start + 3, start + 6].replace('_', '.')
+      user_agent = self.get_user_agent()
+      start = user_agent.index('OS')
+      return user_agent[start + 3:start + 8].replace('_', '.')
     else:
       return ''