diff --git a/src/gnnwr/datasets.py b/src/gnnwr/datasets.py index 4ce7853..25dd15a 100644 --- a/src/gnnwr/datasets.py +++ b/src/gnnwr/datasets.py @@ -619,8 +619,10 @@ def init_dataset(data, test_ratio, valid_ratio, x_column, y_column, spatial_colu if max_test_size < 0: max_test_size = len(test_dataset) if process_fn == "minmax_scale": distance_scale = MinMaxScaler() + temporal_scale = MinMaxScaler() else: distance_scale = StandardScaler() + temporal_scale = StandardScaler() # scale distance matrix train_distance_len = len(train_dataset.distances) val_distance_len = len(val_dataset.distances) @@ -634,6 +636,17 @@ def init_dataset(data, test_ratio, valid_ratio, x_column, y_column, spatial_colu val_dataset.distances = distances[train_distance_len:train_distance_len + val_distance_len] test_dataset.distances = distances[train_distance_len + val_distance_len:] train_dataset.distances_scale_param = val_dataset.distances_scale_param = test_dataset.distances_scale_param = distance_scale_param + '''temporal = np.concatenate((train_dataset.temporal, val_dataset.temporal, test_dataset.temporal), axis=0) + temporal = temporal_scale.fit_transform(temporal.reshape(-1, temporal.shape[-1])).reshape(temporal.shape) + if process_fn == "minmax_scale": + temporal_scale_param = {"min": temporal_scale.data_min_, "max": temporal_scale.data_max_} + else: + temporal_scale_param = {"mean": temporal_scale.mean_, "var": temporal_scale.var_} + train_dataset.temporal = temporal[:train_distance_len] + val_dataset.temporal = temporal[train_distance_len:train_distance_len + val_distance_len] + test_dataset.temporal = temporal[train_distance_len + val_distance_len:] + train_dataset.temporal_scale_param = val_dataset.temporal_scale_param = test_dataset.temporal_scale_param = temporal_scale_param + ''' train_dataset.dataloader = DataLoader( train_dataset, batch_size=batch_size, shuffle=shuffle) val_dataset.dataloader = DataLoader( diff --git a/src/gnnwr/models.py b/src/gnnwr/models.py index af9c8e1..76bfe1c 100644 --- a/src/gnnwr/models.py +++ b/src/gnnwr/models.py @@ -109,6 +109,9 @@ def __init__( os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(map(str, devices)) else: self._use_gpu = False + self._optimizer = None + self._scheduler = None + self._optimizer_name = None self.init_optimizer(optimizer, optimizer_params) # initialize the optimizer def init_optimizer(self, optimizer, optimizer_params=None): @@ -121,10 +124,10 @@ def init_optimizer(self, optimizer, optimizer_params=None): # initialize the optimizer if optimizer == "SGD": self._optimizer = optim.SGD( - self._model.parameters(), lr=1) + self._model.parameters(), lr=1, momentum=0.9, weight_decay=1e-3) elif optimizer == "Adam": self._optimizer = optim.Adam( - self._model.parameters(), lr=self._start_lr) + self._model.parameters(), lr=self._start_lr, weight_decay=1e-3) elif optimizer == "RMSprop": self._optimizer = optim.RMSprop( self._model.parameters(), lr=self._start_lr) @@ -145,10 +148,10 @@ def init_optimizer(self, optimizer, optimizer_params=None): maxlr = optimizer_params.get("maxlr", 0.1) minlr = optimizer_params.get("minlr", 0.01) upepoch = optimizer_params.get("upepoch", 100) - uprate = (maxlr - minlr) / upepoch * 1000 + uprate = (maxlr - minlr) / upepoch * (upepoch // 20) decayepoch = optimizer_params.get("decayepoch", 200) decayrate = optimizer_params.get("decayrate", 0.1) - lamda_lr = lambda epoch: (epoch // 1000) * uprate + minlr if epoch < upepoch else ( + lamda_lr = lambda epoch: (epoch // (upepoch // 20)) * uprate + minlr if epoch < upepoch else ( maxlr if epoch < decayepoch else maxlr * (decayrate ** (epoch - decayepoch))) self._scheduler = optim.lr_scheduler.LambdaLR( self._optimizer, lr_lambda=lamda_lr) @@ -182,12 +185,11 @@ def __train(self): self._model.train() # set the model to train mode train_loss = 0 # initialize the loss data_loader = self._train_dataset.dataloader # get the data loader - maxindex = len(data_loader) # get the number of batches weight_all = torch.tensor([]).to(torch.float32) x_true = torch.tensor([]).to(torch.float32) y_true = torch.tensor([]).to(torch.float32) y_pred = torch.tensor([]).to(torch.float32) - for index, (data, coef, label, id) in enumerate(data_loader): + for index, (data, coef, label, data_index) in enumerate(data_loader): # move the data to gpu device = torch.device('cuda') if self._use_gpu else torch.device('cpu') data, coef, label = data.to(device), coef.to(device), label.to(device) @@ -231,7 +233,7 @@ def __valid(self): data_loader = self._valid_dataset.dataloader # get the data loader with torch.no_grad(): # disable gradient calculation - for data, coef, label, id in data_loader: + for data, coef, label, data_index in data_loader: device = torch.device('cuda') if self._use_gpu else torch.device('cpu') data, coef, label = data.to(device), coef.to(device), label.to(device) # weight = self._model(data) @@ -254,7 +256,8 @@ def __valid(self): print(label_list) print(out_list) self._valid_r2 = r2 - if r2 > self._bestr2: # if the R square is better than the best R square,record the R square and save the model + if r2 > self._bestr2: + # if the R square is better than the best R square,record the R square and save the model self._bestr2 = r2 self._besttrainr2 = self._train_diagnosis.R2().data self._noUpdateEpoch = 0 @@ -278,7 +281,7 @@ def __test(self): y_pred = torch.tensor([]).to(torch.float32) weight_all = torch.tensor([]).to(torch.float32) with torch.no_grad(): - for data, coef, label, id in data_loader: + for data, coef, label, data_index in data_loader: device = torch.device('cuda') if self._use_gpu else torch.device('cpu') data, coef, label = data.to(device), coef.to(device), label.to(device) x_data, y_data, y_pred, weight_all = x_data.to(device), y_data.to(device), y_pred.to( @@ -472,7 +475,7 @@ def add_graph(self): """ add graph to tensorboard """ - for data, coef, label, id in self._train_dataset.dataloader: + for data, coef, label, data_index in self._train_dataset.dataloader: if self._use_gpu: data = data.cuda() self._model = self._model.cuda() @@ -489,6 +492,7 @@ def result(self, path=None, use_dict=False, map_location=None): :param path: the path of the model :param use_dict: whether use dict to load the model + :param map_location: """ # load model if not self.__istrained: @@ -557,23 +561,23 @@ def reg_result(self, filename=None, model_path=None, use_dict=False, only_return device = torch.device('cuda') if self._use_gpu else torch.device('cpu') result = torch.tensor([]).to(torch.float32).to(device) with torch.no_grad(): - for data, coef, label, id in self._train_dataset.dataloader: - data, coef, label, id = data.to(device), coef.to(device), label.to(device), id.to(device) + for data, coef, label, data_index in self._train_dataset.dataloader: + data, coef, label, data_index = data.to(device), coef.to(device), label.to(device), data_index.to(device) output = self._out(self._model(data).mul(coef.to(torch.float32))) weight = self._model(data).mul(torch.tensor(self._weight).to(torch.float32).to(device)) - output = torch.cat((weight, output, id), dim=1) + output = torch.cat((weight, output, data_index), dim=1) result = torch.cat((result, output), 0) - for data, coef, label, id in self._valid_dataset.dataloader: - data, coef, label, id = data.to(device), coef.to(device), label.to(device), id.to(device) + for data, coef, label, data_index in self._valid_dataset.dataloader: + data, coef, label, data_index = data.to(device), coef.to(device), label.to(device), data_index.to(device) output = self._out(self._model(data).mul(coef.to(torch.float32))) weight = self._model(data).mul(torch.tensor(self._weight).to(torch.float32).to(device)) - output = torch.cat((weight, output, id), dim=1) + output = torch.cat((weight, output, data_index), dim=1) result = torch.cat((result, output), 0) - for data, coef, label, id in self._test_dataset.dataloader: - data, coef, label, id = data.to(device), coef.to(device), label.to(device), id.to(device) + for data, coef, label, data_index in self._test_dataset.dataloader: + data, coef, label, data_index = data.to(device), coef.to(device), label.to(device), data_index.to(device) output = self._out(self._model(data).mul(coef.to(torch.float32))) weight = self._model(data).mul(torch.tensor(self._weight).to(torch.float32).to(device)) - output = torch.cat((weight, output, id), dim=1) + output = torch.cat((weight, output, data_index), dim=1) result = torch.cat((result, output), 0) result = result.cpu().detach().numpy() columns = list(self._train_dataset.x) @@ -583,7 +587,8 @@ def reg_result(self, filename=None, model_path=None, use_dict=False, only_return columns = columns + ["Pred_" + self._train_dataset.y[0]] + self._train_dataset.id result = pd.DataFrame(result, columns=columns) result[self._train_dataset.id] = result[self._train_dataset.id].astype(np.int32) - if only_return: return result + if only_return: + return result if filename is not None: result.to_csv(filename, index=False) else: @@ -658,7 +663,7 @@ def __init__(self, ): if optimizer_params is None: - optimizer_params = {} + optimizer_params = {'scheduler': 'MultiStepLR', 'scheduler_milestones': [100, 300, 500]} if dense_layers is None: dense_layers = [[], []] super(GTNNWR, self).__init__(train_dataset, valid_dataset, test_dataset, dense_layers[1], start_lr, optimizer, diff --git a/src/gnnwr/networks.py b/src/gnnwr/networks.py index 3ce3aa1..6ef1877 100644 --- a/src/gnnwr/networks.py +++ b/src/gnnwr/networks.py @@ -70,7 +70,7 @@ def __init__(self, dense_layer=None, insize=-1, outsize=-1, drop_out=0.2, activa m.bias.data.fill_(0) def forward(self, x): - x.to(torch.float32) + x.to(torch.float64) x = self.fc(x) return x @@ -122,7 +122,7 @@ def __init__(self, dense_layer, insize, outsize, drop_out=0.2, activate_func=nn. def forward(self, x): # STPNN - x.to(torch.float32) + x.to(torch.float64) batch = x.shape[0] height = x.shape[1] x = torch.reshape(x, shape=(batch * height, x.shape[2])) @@ -172,7 +172,7 @@ def weight_share(model, x, output_size=1): :param output_size: output size of neural network :return: output of neural network """ - x.to(torch.float32) + x.to(torch.float64) batch = x.shape[0] height = x.shape[1] x = torch.reshape(x, shape=(batch * height, x.shape[2]))