v. 1.3.0 release

(PLDI 2017 -- "Control-Flow Recovery from Partial Failure Reports")

.gitignore

@@ -4,7 +4,8 @@
 .sconsign.dblite
 Release
 config.log
-csi-*.*.*
-csi-*.*.*.tar.gz
+/csi-*.*.*
+/csi-*.*.*.tar.gz
+stamp
 
 **/*.pyc

COPYING

@@ -3,7 +3,7 @@ CSI Release License
 -------------------
 Apache License, Version 2.0
 
-Copyright (c) 2016 Peter J. Ohmann and Benjamin R. Liblit
+Copyright (c) 2017 Peter J. Ohmann and Benjamin R. Liblit
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

NEWS

@@ -1,3 +1,8 @@
+* csi-1.3.0
+
+** Build used for PLDI 2017 conference paper on program analysis.
+** Support for OSX 10.11+
+
 * csi-1.2.0
 
 ** Build used for 2016 ASE conference paper on coverage optimization.

README.md

@@ -11,7 +11,7 @@ Preprint: http://pages.cs.wisc.edu/~liblit/ase-journal-2016/ase-journal-2016.pdf
 
 ## Current Release
 
-[`csi-cc` v1.2.0](../../releases/tag/v1.2.0)
+[`csi-cc` v1.3.0](../../releases/tag/v1.3.0)
 
 ## Documentation
 
@@ -20,7 +20,16 @@ Documentation is included in the source download, or you can view the
 
 ## Changelog
 
-###[v. 1.2.0](../../releases/tag/v1.2.0)
+### [v. 1.3.0](../../releases/tag/v1.3.0)
+
+Release associated with the PLDI 2017 paper "Control-Flow Recovery from
+Partial Failure Reports."  Major changes include:
+
+- Support for more recent LLVM releases (through 4.0)
+- Support for OSX 10.11+
+- Tools for extracting annotated control-flow graphs for downstream analyses
+
+### [v. 1.2.0](../../releases/tag/v1.2.0)
 
 Release associated with the ASE 2016 paper "Optimizing Customized Program
 Coverage."  Major changes include:
@@ -29,7 +38,7 @@ Coverage."  Major changes include:
 coverage instrumentation (see the
 [online documentation](https://rawgit.com/liblit/csi-cc/master/doc/running_optimization.html))
 
-###[v. 1.1.0](../../releases/tag/v1.1.0)
+### [v. 1.1.0](../../releases/tag/v1.1.0)
 
 Release associated with the article from the Automated Software Engineering
 journal (2016).  Major changes include:
@@ -38,6 +47,6 @@ journal (2016).  Major changes include:
 - Support for more customizable tracing (see the
 [online documentation](https://rawgit.com/liblit/csi-cc/master/doc/running_schemes.html))
 
-###[v. 1.0.0](../../releases/tag/v1.0.0)
+### [v. 1.0.0](../../releases/tag/v1.0.0)
 
 Initial release.

SConstruct

@@ -36,8 +36,8 @@ opts.AddVariables(
 )
 env = Environment(
     options=opts,
-    tools=('default', 'llvm', 'textfile'),
-    toolpath=('instrumentor',),
+    tools=('default', 'llvm', 'textfile', 'unix'),
+    toolpath=('instrumentor', 'scons-tools',),
 )
 opts.Save('.scons-config', env)
 
@@ -83,9 +83,12 @@ SConscript(dirs=[
     'instrumentor',
     'driver',
     'tests',
+    'Tools',
 ])
 
 Default('Release')
+Depends('Release', 'Tools')
+Depends('test', 'Tools')
 
 
 ########################################################################

Tools/.gitignore

@@ -0,0 +1,2 @@
+extract-cfg
+extract_section.py

Tools/SConscript

@@ -0,0 +1,14 @@
+Import('env')
+
+extractor = env.Substfile('extract-cfg.in', \
+                          SUBST_DICT={'@LLVM_BINDIR@': env.subst('$LLVM_bindir'),
+                                      '@SHLIB_PREFIX@': env.subst('$SHLIBPREFIX'),
+                                      '@SHLIB_SUFFIX@': env.subst('$SHLIBSUFFIX')})
+AddPostAction(extractor, Chmod('$TARGET', 0755))
+Default(extractor)
+
+extractor = env.Substfile('extract_section.py.in', \
+                          SUBST_DICT={'@OBJDUMP_EXE@': env.subst('$OBJDUMPEXE')})
+
+AddPostAction(extractor, Chmod('$TARGET', 0755))
+Default(extractor)

Tools/extract-cfg.in

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+__pychecker__ = 'no-shadowbuiltin'
+
+import os.path
+import platform
+from shutil import rmtree
+from subprocess import check_call, STDOUT
+from sys import argv, exit, path, stderr
+from tempfile import mkdtemp
+
+
+PATH_TO_CSI = os.path.dirname(os.path.dirname(os.path.realpath(os.path.abspath(argv[0]))))
+PATH_TO_CSI_RELEASE = os.path.join(PATH_TO_CSI, "Release")
+PATH_TO_CSI_TOOLS = os.path.join(PATH_TO_CSI, "Tools")
+
+path.insert(1, PATH_TO_CSI_TOOLS)
+from extract_section import getSectionContents
+
+
+def __llvmBin(command):
+    return os.path.join('@LLVM_BINDIR@', command)
+
+def __isOSX():
+    return 'darwin' in platform.system().lower()
+
+def main():
+    if len(argv) != 3:
+        print >>stderr, 'Usage: %s <executable | object-file> <output-file>' % argv[0]
+        exit(1)
+
+    inFile = argv[1]
+    outFile = argv[2]
+
+    sectionName = ('__CSI' if __isOSX() else '') + '.debug_CSI_BC'
+    bc = getSectionContents(inFile, sectionName, True)
+    if bc == None:
+        print >>stderr, 'File %s contains no CSI bitcode section' % inFile
+        exit(1)
+    bcModules = filter(None, bc.split(';; CSI BC SEPARATOR ;;'))
+    if len(bcModules) < 1:
+        print >>stderr, 'File %s contains no CSI bitcode' % inFile
+        exit(1)
+
+    # use a try-finally here (rather than a context manager) to support python2
+    try:
+        scratchDir = mkdtemp()
+
+        moduleFiles = []
+        for (idx, module) in enumerate(bcModules):
+            moduleFiles.append(os.path.join(scratchDir, 'bc_'+str(idx)+'.bc'))
+            with open(moduleFiles[-1], 'wb') as f:
+                f.write(module)
+
+        linkedBc = os.path.join(scratchDir, 'linked_bc.bc')
+        check_call([__llvmBin('llvm-link'), '-o', linkedBc] + moduleFiles)
+
+        optArgs = ['-load']
+        optArgs.append(os.path.join(PATH_TO_CSI_RELEASE, "@SHLIB_PREFIX@" + "CSI" + "@SHLIB_SUFFIX@"))
+        optArgs.append('-cfg-write')
+        optArgs.append('-cfg-file')
+        optArgs.append(outFile)
+        optArgs.append(linkedBc)
+        check_call([__llvmBin('opt')] + optArgs + ['-o', '/dev/null'])
+    finally:
+        rmtree(scratchDir, ignore_errors=True)
+
+
+if __name__ == '__main__':
+    main()

Tools/extract-section

@@ -0,0 +1 @@
+./extract_section.py

Tools/extract_section.py → Tools/extract_section.py.in

@@ -1,4 +1,4 @@
-#!/usr/bin/python -O
+#!/usr/bin/env python
 
 __pychecker__ = 'no-shadowbuiltin'
 
@@ -12,7 +12,7 @@
 
 
 def getSectionContents(filename, desiredSection, require=False):
-    objdump = Popen(['objdump', '--section-headers', '--wide', filename], stdout=PIPE)
+    objdump = Popen(['@OBJDUMP_EXE@', '--section-headers', '--wide', filename], stdout=PIPE)
 
     size = offset = None
     for line in objdump.stdout:

doc/building_comments.html

@@ -54,6 +54,13 @@ <h3>Optional Build Configuration, Additional Comments, and Suggestions</h3>
 the <kbd>gams</kbd> executable, as well as a subdirectory called
 <kbd>apifiles/</kbd>.</p>
 
+<p>If building on OSX, note that many pre-packaged distributions of LLVM do not
+include a full LLVM shared object for linking with CSI.  If you see linker
+errors when building with pre-installed LLVM, please try building LLVM from
+source.  The LLVM project provides guides for installing
+<a href="http://llvm.org/docs/GettingStarted.html">LLVM</a> and
+<a href="http://clang.llvm.org/get_started.html">Clang</a> from source.</p>
+
 <hr/>
 <table class="toptable"><tr>
 <td class="topprev"><a href="building.html">&larr; Prev</a></td>

doc/metadata.html

@@ -15,14 +15,27 @@
 <h1>CSI Output: Metadata</h1>
 <hr class="half"/>
 <h3>General</h3>
-<p>To extract static instrumentation metadata, use the <kbd>extract-section</kbd> tool in the
-<samp>Tools</samp> directory of this release.  This tool requires the
-following non-standard packages:</p>
+<p>To extract static instrumentation metadata, use the <kbd>extract-cfg</kbd>
+and <kbd>extract-section</kbd> tools in the <samp>Tools</samp> directory of this
+release.  The tools requires the following non-standard packages:</p>
 <ul><li><a href="http://www.python.org/">Python</a> 2.6+</li></ul>
 
-<p>It also requires the application <kbd>objdump</kbd>, which is
+<p>They also require the application <kbd>objdump</kbd>, which is
 standard on most Unix-based operating systems.</p>
 
+<h3>Control-flow Graph</h3>
+<p>CSI embeds the LLVM bitcode necessary to extract an annotated representation
+of the program's control-flow graph from generated object files and executables.
+Note that in the case of multiple object files linked into a single
+executable, the executable contains the complete bitcode necessary to build the
+whole-program control-flow graph.  To extract the graph, use a command similar
+to<br/>
+<kbd class="indent">Tools/extract-cfg <var>myexe</var> <var>output-file</var></kbd></p>
+
+<p>The graph is extracted in
+<a href="http://graphml.graphdrawing.org">GraphML</a> format, with appropriate
+node and edge annotations for running CSI analysis tools.</p>
+
 <h3>Path Tracing</h3>
 <p>Path Tracing metadata is stored as text in the debug
 section <samp>.debug_PT</samp> of the object file or executable.  Note
@@ -54,6 +67,17 @@ <h3>Program Coverage</h3>
 <p>For details about the format of the Program Coverage metadata, please see the
 <a href="metadata_cc.html">Program Coverage</a> page.</p>
 
+<h3>OSX</h3>
+<p>Section names in Mach-O executables are named slightly differently than those
+in ELF executables.  Specifically, all section names should be preceded by
+<samp>__CSI</samp>.  Thus, the metadata sections above are named as follows:
+<ul>
+<li><samp>__CSI.debug_PT</samp> (for path tracing)</li>
+<li><samp>__CSI.debug_FC</samp> (for function coverage)</li>
+<li><samp>__CSI.debug_CC</samp> (for call coverage)</li>
+<li><samp>__CSI.debug_BBC</samp> (for statement coverage)</li>
+</ul>
+
 <hr/>
 <table class="toptable"><tr>
 <td class="topprev"><a href="output.html">&larr; Prev</a></td>

driver/SConscript

@@ -6,5 +6,9 @@ Default(helper)
 driver = env.Substfile('#Release/csi-cc',
                        'csi-driver.py.in',
                        SUBST_DICT={"@USE_GAMS@": str("GAMSDIR" in env),
-                                   "@GAMS_INSTALL_DIR@": env.get("GAMSDIR", "")})
+                                   "@GAMS_INSTALL_DIR@": env.get("GAMSDIR", ""),
+                                   "@SHLIB_PREFIX@": env.subst("$SHLIBPREFIX"),
+                                   "@SHLIB_SUFFIX@": env.subst("$SHLIBSUFFIX"),
+                                   "@OBJCOPY_EXE@": env.subst("$OBJCOPYEXE"),
+                                   "@LLVM_ASSERTIONS@": env.get("LLVM_assertions", False)})
 AddPostAction(driver, Chmod('$TARGET', 0755))