From 4e526024ad7c7f11785a7caca1c4861b9d6b7c84 Mon Sep 17 00:00:00 2001 From: Arman Boehm Date: Wed, 27 Mar 2013 21:32:29 -0700 Subject: [PATCH 1/3] initial ios compatibility --- .gitignore | 1 + bin/mobster.py | 21 ++++++++++- ...esearch.json => googlesearch_dchrome.json} | 4 +++ src/linkedin/mobster/comm/__init__.py | 0 .../mobster/{ => comm}/webkitcommunicator.py | 22 +++++++----- src/linkedin/mobster/har/flowprofiler.py | 36 ++++++++++++++----- src/linkedin/mobster/har/network.py | 2 +- src/linkedin/mobster/har/page.py | 5 ++- src/linkedin/mobster/webkitclient.py | 12 ++++--- 9 files changed, 79 insertions(+), 24 deletions(-) rename bin/sampleinput/{googlesearch.json => googlesearch_dchrome.json} (91%) create mode 100644 src/linkedin/mobster/comm/__init__.py rename src/linkedin/mobster/{ => comm}/webkitcommunicator.py (90%) diff --git a/.gitignore b/.gitignore index f89767e..f362f79 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ report bin/sampleinput/limobilelogin.json env/ +*.swp diff --git a/bin/mobster.py b/bin/mobster.py index 365cab2..4654e1b 100755 --- a/bin/mobster.py +++ b/bin/mobster.py @@ -8,6 +8,7 @@ import json import logging import os +import signal import subprocess import sys import time @@ -34,18 +35,34 @@ def run(args): Run a test with the specified parameters, and return the HTTP Archive (HAR) File represented as a dictionary. """ + if args.ios: + kill_existing_proxy_processes() + DEVNULL = open(os.devnull, 'wb') + p = subprocess.Popen('ios_webkit_debug_proxy', stdout=DEVNULL, stderr=DEVNULL) + logging.info("Giving proxy time to start") + time.sleep(2) har_gen = FlowProfiler(args.testfile, int(args.iterations)) \ if args.iterations else FlowProfiler(args.testfile) # profiling_results is a list of lists containing HARs for each page in a run profiling_results = har_gen.profile() - + if args.ios: + p.terminate() if args.average: return [merge_by_average(page_results) \ for page_results in zip(*profiling_results)] else: return profiling_results[-1] +def kill_existing_proxy_processes(): + p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE) + out, err = p.communicate() + + for line in out.splitlines(): + if 'ios_webkit' in line: + pid = int(line.split(None, 1)[0]) + os.kill(pid, signal.SIGKILL) + def write_report(args): """ Autogenerates an HTML report and writes it to a file, with location and input @@ -158,6 +175,8 @@ def parse_args(): arg_parser.add_argument('-b', '--browser', action='store_true', \ help='Open HTML report in browser after creation') + + arg_parser.add_argument('--ios', action='store_true', help='Run iOS debug proxy') # Used if and only if generating report with results from previous test arg_parser.add_argument('-r', '--har', \ diff --git a/bin/sampleinput/googlesearch.json b/bin/sampleinput/googlesearch_dchrome.json similarity index 91% rename from bin/sampleinput/googlesearch.json rename to bin/sampleinput/googlesearch_dchrome.json index 446876a..7bc6e9d 100644 --- a/bin/sampleinput/googlesearch.json +++ b/bin/sampleinput/googlesearch_dchrome.json @@ -1,3 +1,7 @@ +/* + * Searches for "LinkedIn" in Google. Only tested on Desktop Chrome. + */ + { "name" : "Google Search Flow", "navigations": [ diff --git a/src/linkedin/mobster/comm/__init__.py b/src/linkedin/mobster/comm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/linkedin/mobster/webkitcommunicator.py b/src/linkedin/mobster/comm/webkitcommunicator.py similarity index 90% rename from src/linkedin/mobster/webkitcommunicator.py rename to src/linkedin/mobster/comm/webkitcommunicator.py index 5a21311..9c02a27 100644 --- a/src/linkedin/mobster/webkitcommunicator.py +++ b/src/linkedin/mobster/comm/webkitcommunicator.py @@ -1,3 +1,4 @@ +from abc import ABCMeta, abstractmethod from collections import defaultdict import json import logging @@ -11,7 +12,7 @@ from linkedin.mobster.mobsterconfig import config -class RemoteWebKitCommunicator(WebSocketClient): +class WebKitCommunicator(WebSocketClient): """ Asynchronous interface for communicating with a remote WebKit-based browser via remote debugging protocol. Currently tested only on desktop and Android @@ -27,14 +28,22 @@ class RemoteWebKitCommunicator(WebSocketClient): """ def __init__(self, page_num = 0): + self._page_num = page_num self._counter = 0 self._response_callbacks = {} self._domain_callbacks = defaultdict(lambda: {}) self._stopped = False self._command_queue = Queue() + debug_ws_url = self.get_ws_debug_url() + super(WebKitCommunicator, self).__init__(debug_ws_url) + self.start() + + def get_ws_debug_url(self): + """ + Returns the WebSocket debugging URL used by the browser for the specified + page number. + """ - # Access list of open browser pages and pick the page with the specified - # index url = 'http://localhost:{0}/json'.format(config["WS_DEBUG_PORT"]) try: response = urllib2.urlopen(url).read() @@ -44,15 +53,12 @@ def __init__(self, page_num = 0): sys.exit() page_info = json.loads(response) - page = page_info[page_num] - debug_ws_url = page['webSocketDebuggerUrl'] + return page_info[self._page_num]['webSocketDebuggerUrl'] - super(RemoteWebKitCommunicator, self).__init__(debug_ws_url) - self.start() def opened(self): pass def closed(self, code, reason=None): pass - + def received_message(self, messageData): """Called whenever the WebSocket receives a message""" response = json.loads(str(messageData)) diff --git a/src/linkedin/mobster/har/flowprofiler.py b/src/linkedin/mobster/har/flowprofiler.py index c643bc2..5a2a412 100644 --- a/src/linkedin/mobster/har/flowprofiler.py +++ b/src/linkedin/mobster/har/flowprofiler.py @@ -1,19 +1,21 @@ import json +import logging +import time +from linkedin.mobster.comm.webkitcommunicator import WebKitCommunicator from linkedin.mobster.har.css import CSSProfileParser from linkedin.mobster.har.network import NetworkEventHandler from linkedin.mobster.har.page import PageEventHandler, PageLoadNotifier from linkedin.mobster.har.timeline import TimelineEventHandler from linkedin.mobster.utils import wait_until, format_time from linkedin.mobster.webkitclient import RemoteWebKitClient -from linkedin.mobster.webkitcommunicator import RemoteWebKitCommunicator class FlowProfiler(RemoteWebKitClient): """ Runs a flow and records profiling information for each navigation. Generates HAR file. """ def __init__(self, test_file, iterations=1): - super(FlowProfiler, self).__init__(RemoteWebKitCommunicator()) + super(FlowProfiler, self).__init__(WebKitCommunicator()) assert iterations > 0, "iterations must be a positive integer" self._iterations = iterations self._page_event_handler = None @@ -39,7 +41,10 @@ def profile(self): for x in range(0, self._iterations): hars = [] - self.clear_http_cache() + if self.can_clear_http_cache(): + self.clear_http_cache() + else: + logging.info('Not clearing browser cache') self.clear_cookies() # Navigate to about:blank, to reset memory, etc. @@ -51,7 +56,7 @@ def profile(self): for navigation in self._test['navigations']: assert len(navigation) > 0, 'Each navigation must have at least one action' - + self._start_time = time.time() # do all the actions except the last one, because the last action causes the actual page navigation for i in range(0, len(navigation) - 1): self.process_action(navigation[i]) @@ -150,10 +155,23 @@ def get_page_timings(self): Returns a dictionary containing onContentLoad and onLoad times, both in (whole) milliseconds from the start of the request """ - browser_timings = self.get_window_performance() + #FIXME the two different methods of calculating timings need to be merged first_request_start = self._network_event_handler.get_first_request_time() - return { - 'onContentLoad': max(int(browser_timings['domContentLoadedEventEnd'] - (first_request_start * 1000)), -1), - 'onLoad' : max(int(browser_timings['loadEventEnd'] - (first_request_start * 1000)), -1) - } + if self.get_os_name() == 'iOS': + onload = int(1000 * \ + (self._page_event_handler.dom_content_load - self._start_time)) + domcontent = int(1000 * \ + (self._page_event_handler.on_load - self._start_time)) + + return { + 'onContentLoad': max(domcontent, -1), + 'onLoad' : max(onload, -1) + } + else: + browser_timings = self.get_window_performance() + return { + 'onContentLoad': max(int(browser_timings['domContentLoadedEventEnd'] - (first_request_start * 1000)), -1), + 'onLoad' : max(int(browser_timings['loadEventEnd'] - (first_request_start * 1000)), -1) + } + diff --git a/src/linkedin/mobster/har/network.py b/src/linkedin/mobster/har/network.py index 164d612..f482cd3 100644 --- a/src/linkedin/mobster/har/network.py +++ b/src/linkedin/mobster/har/network.py @@ -169,7 +169,7 @@ def process_response_received(self, message): # timings are not included for about: url's if not self._requests[request_id]['url'].startswith('about:'): if not 'timing' in params['response']: - log.info('No timing information in message') + logging.info('No timing information in message') return provided_timings = params['response']['timing'] diff --git a/src/linkedin/mobster/har/page.py b/src/linkedin/mobster/har/page.py index 41b4c8e..709489d 100644 --- a/src/linkedin/mobster/har/page.py +++ b/src/linkedin/mobster/har/page.py @@ -10,6 +10,10 @@ def process_event(self, message): if message['method'] == 'Page.loadEventFired': logging.info('Page.loadEventFired recorded') self.page_loaded = True + self.on_load = message['params']['timestamp'] + elif message['method'] == 'Page.domContentEventFired': + logging.info('Page.domContentEventFired recorded') + self.dom_content_load = message['params']['timestamp'] class PageLoadNotifier(object): @@ -30,7 +34,6 @@ def process_page_event(self, message): logging.info('Page.loadEventFired recorded') self._received_page_load_event = True - def process_network_event(self, message): self._last_network_event_time = time.time() diff --git a/src/linkedin/mobster/webkitclient.py b/src/linkedin/mobster/webkitclient.py index d8cd626..f614386 100644 --- a/src/linkedin/mobster/webkitclient.py +++ b/src/linkedin/mobster/webkitclient.py @@ -72,8 +72,11 @@ def response_handler(response): elif 'result' in response and 'result' in response['result']: if 'wasThrown' in response['result']['result']: logging.error('Received error after running JS: {0}\n{1}'.format(js, response['result'])) - - self._js_result = response['result']['result']['value'] + + try: + self._js_result = response['result']['result']['value'] + except KeyError: + logging.error('Javascript failed: {0}'.format(js)) self._got_js_result = True self._communicator.send_cmd('Runtime.evaluate', @@ -474,8 +477,9 @@ def get_os_version(self): if self._device_is_android(): return re.search('Android\s+([\d\.]+)',self.get_user_agent()).groups()[0] elif self._device_is_ios(): - start = self.get_user_agent().index('OS') - return self.get_user_agent()[start + 3, start + 6].replace('_', '.') + user_agent = self.get_user_agent() + start = user_agent.index('OS') + return user_agent[start + 3:start + 8].replace('_', '.') else: return '' From 56f9ab696262d55f3412e2d2e0a51c96eb854296 Mon Sep 17 00:00:00 2001 From: Arman Boehm Date: Wed, 27 Mar 2013 21:44:34 -0700 Subject: [PATCH 2/3] updated readme --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 8962bbc..bcf61c0 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,11 @@ Mobster provides a simple way for developers to record crucial web performance d 4. In the Chrome settings, enable USB Debugging (Advanced -> Developer Tools -> Enable USB Web Debugging) +#### iOS-Specific Prerequisites #### +1. Install the [iOS WebKit Debugging Proxy](https://github.com/google/ios-webkit-debug-proxy) on your computer. + +2. In the Safari settings, enable Web Inspector (Safari Settings -> Advanced -> Web Inspector) + ### Run Mobster with an Android Device ### 1. Execute the command below under the Android SDK folder: @@ -87,6 +92,16 @@ Mobster provides a simple way for developers to record crucial web performance d **Important Note:** If you use Chrome as your web browser normally, it will be annoying to run Mobster with Chrome because Mobster by default uses one of the currently open tab(s) for testing and also clears cookies, etc. This means that, at the end of a test, one of your open tab(s) will be showing the final web page from your test and you will be logged out of all websites. **An easy way to avoid this problem is to run Mobster with [Chromium](http://www.chromium.org/Home) or [Chrome Canary](https://www.google.com/intl/en/chrome/browser/canary.html) so your normal browsing is not affected.** Chrome, Chromium, and Chrome Canary can all be installed side-by-side. +## Run Mobster with an iOS Device ## + + +1. Open Safari on the device. Note that **Mobster will clear cookies** and other browsing data. The tab currently being viewed will be used by Mobster to navigate to webpages. Currently, Safari must be visible on the screen for Mobster to work. + +2. Run the main Mobster script with a sample flow in your Mobster home directory. Note the --ios argument, which tells Mobster to start the iOS webkit debugging proxy (which **must** be installed). + + 
./bin/mobster.py -t bin/sampleinput/sample.json -p -b --ios
+ + Mobster reports will be generated in the MOBSTER_HOME/report folder if no folder is specified. Run mobster.py with the "-h" option to learn more about command-line options. To learn how to make your own flows, look at the scripts in the bin/sampleinput/ directory. ## Contribution ## From c159dc489e99a946f4d667e930de6e2841ce0c48 Mon Sep 17 00:00:00 2001 From: Arman Boehm Date: Wed, 27 Mar 2013 21:46:15 -0700 Subject: [PATCH 3/3] readme fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bcf61c0..2992dc9 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Mobster provides a simple way for developers to record crucial web performance d **Important Note:** If you use Chrome as your web browser normally, it will be annoying to run Mobster with Chrome because Mobster by default uses one of the currently open tab(s) for testing and also clears cookies, etc. This means that, at the end of a test, one of your open tab(s) will be showing the final web page from your test and you will be logged out of all websites. **An easy way to avoid this problem is to run Mobster with [Chromium](http://www.chromium.org/Home) or [Chrome Canary](https://www.google.com/intl/en/chrome/browser/canary.html) so your normal browsing is not affected.** Chrome, Chromium, and Chrome Canary can all be installed side-by-side. -## Run Mobster with an iOS Device ## +### Run Mobster with an iOS Device ### 1. Open Safari on the device. Note that **Mobster will clear cookies** and other browsing data. The tab currently being viewed will be used by Mobster to navigate to webpages. Currently, Safari must be visible on the screen for Mobster to work.