Convolution with minibatch size 32 or 64 is now about twice as fast. …

…Added --conserve-mem parameter.

convnet.py

@@ -181,7 +181,8 @@ def get_options_parser(cls):
         op.add_option("logreg-name", "logreg_name", StringOptionParser, "Cropped DP: logreg layer name (for --multiview-test)", default="")
         op.add_option("conv-to-local", "conv_to_local", ListOptionParser(StringOptionParser), "Convert given conv layers to unshared local", default=[])
         op.add_option("unshare-weights", "unshare_weights", ListOptionParser(StringOptionParser), "Unshare weight matrices in given layers", default=[])
-
+        op.add_option("conserve-mem", "conserve_mem", BooleanOptionParser, "Conserve GPU memory (slower)?", default=0)
+
         op.delete_option('max_test_err')
         op.options["max_filesize_mb"].default = 0
         op.options["testing_freq"].default = 50

gpumodel.py

@@ -97,6 +97,7 @@ def fill_excused_options(self):
         pass
 
     def init_data_providers(self):
+        self.dp_params['convnet'] = self
         try:
             self.test_data_provider = DataProvider.get_instance(self.data_path, self.test_batch_range,
                                                                 type=self.dp_type, dp_params=self.dp_params, test=True)
@@ -231,7 +232,18 @@ def get_test_error(self):
                 test_outputs += [self.finish_batch()]
                 break            
         return self.aggregate_test_outputs(test_outputs)
-
+
+    def set_var(self, var_name, var_val):
+        setattr(self, var_name, var_val)
+        self.model_state[var_name] = var_val
+        return var_val
+
+    def get_var(self, var_name):
+        return self.model_state[var_name]
+
+    def has_var(self, var_name):
+        return var_name in self.model_state
+
     def save_state(self):
         for att in self.model_state:
             if hasattr(self, att):

include/layer.cuh

@@ -56,6 +56,7 @@ protected:
     NVMatrix *_outputs; // TODO: make this a pointer so you can reuse previous layers' matrices
     NVMatrix *_actsGrad; // Layer activity gradients
     bool _gradConsumer, _foundGradConsumers, _trans;
+    bool _conserveMem;
     int _numGradProducersNext;
     int _actsTarget, _actsGradTarget;
     std::string _name, _type;
@@ -218,6 +219,7 @@ public:
 
 class LocalUnsharedLayer : public LocalLayer {
 protected:
+    NVMatrix _sexMask;
     void fpropActs(int inpIdx, float scaleTargets, PASS_TYPE passType);
     void bpropActs(NVMatrix& v, int inpIdx, float scaleTargets, PASS_TYPE passType);
     void bpropBiases(NVMatrix& v, PASS_TYPE passType);

layer.py

@@ -190,14 +190,14 @@ def set_defaults(self):
         self.dic['forceOwnActs'] = True
 
         # Does this layer need the gradient at all?
-        # Should only be true for layers with parameters (weights)
-        # or layers which have layers with parameters below them.
+        # Should only be true for layers with parameters (weights).
         self.dic['gradConsumer'] = False
 
     def parse(self, name, mcp, prev_layers, model=None):
         self.prev_layers = prev_layers
         self.dic['name'] = name
         self.dic['type'] = mcp.safe_get(name, 'type')
+        self.dic['conserveMem'] = model.op.get_value('conserve_mem') if model is not None else 0
 
         return self.dic  
 
@@ -640,7 +640,7 @@ def unshare(layer, layers, indices):
             unshare(layer, layers, range(len(layer['inputs'])) if matrix_idx is None else [matrix_idx])
 
     # Load weight/biases initialization module
-    def call_init_func(self, param_name, shapes):
+    def call_init_func(self, param_name, shapes, input_idx=-1):
         dic = self.dic
         func_pat = re.compile('^([^\.]+)\.([^\(\)]+)\s*(?:\(([^,]+(?:,[^,]+)*)\))?$')
         m = func_pat.match(dic[param_name])
@@ -650,7 +650,7 @@ def call_init_func(self, param_name, shapes):
         params = m.group(3).split(',') if m.group(3) is not None else []
         try:
             mod = __import__(module)
-            return getattr(mod, func)(dic['name'], shapes, params=params)
+            return getattr(mod, func)(dic['name'], input_idx, shapes, params=params) if input_idx >= 0 else getattr(mod, func)(dic['name'], shapes, params=params)
         except (ImportError, AttributeError, TypeError), e:
             raise LayerParsingError("Layer '%s': %s." % (dic['name'], e))
 
@@ -660,22 +660,19 @@ def make_weights(self, initW, rows, cols, order='C'):
         if dic['initWFunc']: # Initialize weights from user-supplied python function
             # Initialization function is supplied in the format
             # module.func
-            dic['weights'] = self.call_init_func('initWFunc', zip(rows, cols))
-            if type(dic['weights']) != list:
-                raise LayerParsingError("Layer '%s': weight initialization function %s must return list of weight matrices. Got: %s." % (dic['name'], dic['initWFunc'], type(dic['weights'])))
-            if len(dic['weights']) != len(rows):
-                raise LayerParsingError("Layer '%s': weight initialization function %s returned %d weight matrices; should be %d." % (dic['name'], dic['initWFunc'], len(dic['weights']), len(rows)))
-            for i in xrange(len(dic['weights'])):
+            for i in xrange(len(dic['inputs'])):
+                dic['weights'] += [self.call_init_func('initWFunc', (rows[i], cols[i]), input_idx=i)]
+
                 if type(dic['weights'][i]) != n.ndarray:
-                    raise LayerParsingError("Layer '%s': weight initialization function %s must return list of weight matrices as numpy.ndarray objects. Got: %s." % (dic['name'], dic['initWFunc'], type(dic['weights'][i])))
+                    raise LayerParsingError("Layer '%s[%d]': weight initialization function %s must return numpy.ndarray object. Got: %s." % (dic['name'], i, dic['initWFunc'], type(dic['weights'][i])))
                 if dic['weights'][i].dtype != n.float32:
-                    raise LayerParsingError("Layer '%s': weight initialization function %s must return list of weight matrices consisting of single-precision floats. Got: %s." % (dic['name'], dic['initWFunc'], dic['weights'][i].dtype))
+                    raise LayerParsingError("Layer '%s[%d]': weight initialization function %s must weight matrices consisting of single-precision floats. Got: %s." % (dic['name'], i, dic['initWFunc'], dic['weights'][i].dtype))
                 if dic['weights'][i].shape != (rows[i], cols[i]):
-                    raise LayerParsingError("Layer '%s': weight matrix %d returned by weight initialization function %s has wrong shape. Should be: %s; got: %s." % (dic['name'], i, dic['initWFunc'], (rows[i], cols[i]), dic['weights'][i].shape))
-                # Convert to desired order
-                dic['weights'][i] = n.require(dic['weights'][i], requirements=order)
-            dic['weightsInc'] = [n.zeros_like(w) for w in dic['weights']]
-            print "Layer '%s' initialized weight matrices from function %s" % (dic['name'], dic['initWFunc'])
+                    raise LayerParsingError("Layer '%s[%d]': weight matrix returned by weight initialization function %s has wrong shape. Should be: %s; got: %s." % (dic['name'], i, dic['initWFunc'], (rows[i], cols[i]), dic['weights'][i].shape))
+                    # Convert to desired order
+                    dic['weights'][i] = n.require(dic['weights'][i], requirements=order)
+                dic['weightsInc'] += [n.zeros_like(dic['weights'][i])]
+                print "Layer '%s[%d]' initialized weight matrices from function %s" % (dic['name'], i, dic['initWFunc'])
         else:
             for i in xrange(len(dic['inputs'])):
                 if dic['weightSourceLayerIndices'][i] >= 0: # Shared weight matrix
@@ -802,9 +799,30 @@ def conv_to_local(layers, idx):
 
     # Returns (groups, filterChannels) array that represents the set
     # of image channels to which each group is connected
-    def gen_rand_conns(self, groups, channels, filterChannels):
+    def gen_rand_conns(self, groups, channels, filterChannels, inputIdx):
+        dic = self.dic
         overSample = groups * filterChannels / channels
-        return [x for i in xrange(overSample) for x in nr.permutation(range(channels))]
+        filterConns = [x for i in xrange(overSample) for x in nr.permutation(range(channels))]
+
+        if dic['initCFunc']: # Initialize connectivity from outside source
+            filterConns = self.call_init_func('initCFunc', (groups, channels, filterChannels), input_idx=inputIdx)
+            if len(filterConns) != overSample * channels:
+                raise LayerParsingError("Layer '%s[%d]': random connectivity initialization function %s must return list of length <groups> * <filterChannels> = %d; got: %d" % (dic['name'], inputIdx, dic['initCFunc'], len(filterConns)))
+            if any(c not in range(channels) for c in filterConns):
+                raise LayerParsingError("Layer '%s[%d]': random connectivity initialization function %s must return list of channel indices in the range 0-<channels-1> = 0-%d." % (dic['name'], inputIdx, dic['initCFunc'], channels-1))
+            # Every "channels" sub-slice should be a permutation of range(channels)
+            if any(len(set(c)) != len(c) for c in [filterConns[o*channels:(o+1)*channels] for o in xrange(overSample)]):
+                raise LayerParsingError("Layer '%s[%d]': random connectivity initialization function %s must return list of channel indices such that every non-overlapping sub-list of <channels> = %d elements is a permutation of the integers 0-<channels-1> = 0-%d." % (dic['name'], inputIdx, dic['initCFunc'], channels, channels-1))
+
+        elif dic['weightSourceLayerIndices'][inputIdx] >= 0: # Shared weight matrix
+            src_layer = self.prev_layers[dic['weightSourceLayerIndices'][inputIdx]] if dic['weightSourceLayerIndices'][inputIdx] < len(self.prev_layers) else dic
+            src_inp = dic['weightSourceMatrixIndices'][inputIdx]
+            if 'randSparse' not in src_layer or not src_layer['randSparse']:
+                raise LayerParsingError("Layer '%s[%d]': randSparse is true in this layer but false in weight sharing source layer '%s[%d]'." % (dic['name'], inputIdx, src_layer['name'], src_inp))
+            if (groups, channels, filterChannels) != (src_layer['groups'][src_inp], src_layer['channels'][src_inp], src_layer['filterChannels'][src_inp]):
+                raise LayerParsingError("Layer '%s[%d]': groups, channels, filterChannels set to %d, %d, %d, respectively. Does not match setting in weight sharing source layer '%s[%d]': %d, %d, %d." % (dic['name'], inputIdx, groups, channels, filterChannels, src_layer['name'], src_inp, src_layer['groups'][src_inp], src_layer['channels'][src_inp], src_layer['filterChannels'][src_inp]))
+            filterConns = src_layer['filterConns'][src_inp]
+        return filterConns
 
     def parse(self, name, mcp, prev_layers, model):
         dic = WeightLayerParser.parse(self, name, mcp, prev_layers, model)
@@ -819,12 +837,13 @@ def parse(self, name, mcp, prev_layers, model):
         dic['groups'] = mcp.safe_get_int_list(name, 'groups', default=[1]*len(dic['inputs']))
         dic['randSparse'] = mcp.safe_get_bool_list(name, 'randSparse', default=[False]*len(dic['inputs']))
         dic['initW'] = mcp.safe_get_float_list(name, 'initW')
+        dic['initCFunc'] = mcp.safe_get(name, 'initCFunc', default='')
 
         self.verify_num_params(['channels', 'padding', 'stride', 'filterSize', \
                                                      'filters', 'groups', 'randSparse', 'initW'])
 
         self.verify_num_range(dic['stride'], 'stride', 1, None)
-        self.verify_num_range(dic['filterSize'],'filterSize', 1, None)
+        self.verify_num_range(dic['filterSize'],'filterSize', 1, None)  
         self.verify_num_range(dic['padding'], 'padding', 0, None)
         self.verify_num_range(dic['channels'], 'channels', 1, None)
         self.verify_num_range(dic['groups'], 'groups', 1, None)
@@ -864,7 +883,7 @@ def parse(self, name, mcp, prev_layers, model):
                 self.verify_divisible(dic['channels'][i], dic['filterChannels'][i], 'channels', 'filterChannels', input_idx=i)
                 self.verify_divisible(dic['filterChannels'][i], 4, 'filterChannels', input_idx=i)
                 self.verify_divisible( dic['groups'][i]*dic['filterChannels'][i], dic['channels'][i], 'groups * filterChannels', 'channels', input_idx=i)
-                dic['filterConns'][i] = self.gen_rand_conns(dic['groups'][i], dic['channels'][i], dic['filterChannels'][i])
+                dic['filterConns'][i] = self.gen_rand_conns(dic['groups'][i], dic['channels'][i], dic['filterChannels'][i], i)
             else:
                 if dic['groups'][i] > 1:
                     self.verify_divisible(dic['channels'][i], 4*dic['groups'][i], 'channels', '4 * groups', input_idx=i)

src/convnet.cu

@@ -137,11 +137,6 @@ void ConvNet::initCuda() {
     cudaDeviceSetCacheConfig(cudaFuncCachePreferShared);
     cublasInit();
     NVMatrix::initRandom(time(0));
-
-    // Uncomment these lines to save memory
-//    Layer::_saveActsGrad = false;
-//    Layer::_saveActs = false;
-
     copyToGPU();
 }