From e2b5220ae4a78275cb433c9003b4cc46d1345faa Mon Sep 17 00:00:00 2001 From: Hadi Date: Fri, 15 Mar 2024 12:27:16 +0100 Subject: [PATCH 1/4] fixed bugs causing code to crash --- examples/miceprotein.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/examples/miceprotein.py b/examples/miceprotein.py index 054e248..64d326a 100755 --- a/examples/miceprotein.py +++ b/examples/miceprotein.py @@ -36,64 +36,63 @@ print("Lambda =", model.best_lambda_) plot_cv(model, X_test, y_test) plt.savefig("miceprotein-cv.png") -1 / 0 model = LassoNetClassifier() -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein.png") model = LassoNetClassifier(dropout=0.5) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_dropout.png") model = LassoNetClassifier(hidden_dims=(100, 100)) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_deep.png") model = LassoNetClassifier(hidden_dims=(100, 100), gamma=0.01) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_deep_l2_weak.png") model = LassoNetClassifier(hidden_dims=(100, 100), gamma=0.1) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_deep_l2_strong.png") model = LassoNetClassifier(hidden_dims=(100, 100), gamma=1) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_deep_l2_super_strong.png") model = LassoNetClassifier(hidden_dims=(100, 100), dropout=0.5) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_deep_dropout.png") model = LassoNetClassifier(hidden_dims=(100, 100), backtrack=True, dropout=0.5) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_deep_dropout_backtrack.png") model = LassoNetClassifier(batch_size=64) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_64.png") model = LassoNetClassifier(backtrack=True) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_backtrack.png") model = LassoNetClassifier(batch_size=64, backtrack=True) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_backtrack_64.png") model = LassoNetClassifier(class_weight=[0.1, 0.2, 0.3, 0.1, 0.3, 0, 0, 0]) -path = model.path(X_train, y_train) +path = model.path(X_train, y_train, return_state_dicts=True) plot_path(model, path, X_test, y_test) plt.savefig("miceprotein_weighted.png") From 959642400877595b84194f4847a1522bfaf97981 Mon Sep 17 00:00:00 2001 From: Hadi Date: Fri, 15 Mar 2024 12:32:50 +0100 Subject: [PATCH 2/4] fixed split ratio and added a helper function --- experiments/data_utils.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/experiments/data_utils.py b/experiments/data_utils.py index f997e9b..b48c863 100755 --- a/experiments/data_utils.py +++ b/experiments/data_utils.py @@ -67,21 +67,6 @@ def load_isolet(): return (x_train, y_train - 1), (x_test, y_test - 1) -def load_activity(): - x_train = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_X_train.txt'), delimiter = ',', encoding = 'UTF-8') - x_test = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_X_test.txt'), delimiter = ',', encoding = 'UTF-8') - y_train = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_y_train.txt'), delimiter = ',', encoding = 'UTF-8') - y_test = np.loadtxt(os.path.join('/home/lemisma/datasets/dataset_uci', 'final_y_test.txt'), delimiter = ',', encoding = 'UTF-8') - - X = MinMaxScaler(feature_range=(0,1)).fit_transform(np.concatenate((x_train, x_test))) - x_train = X[: len(y_train)] - x_test = X[len(y_train):] - - print(x_train.shape, y_train.shape) - print(x_test.shape, y_test.shape) - return (x_train, y_train), (x_test, y_test) - - import numpy as np def load_epileptic(): filling_value = -100000 @@ -165,13 +150,13 @@ def load_data(fashion = False, digit = None, normalize = False): def load_mnist(): train, test = load_data(fashion = False, normalize = True) - x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.6) + x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.2) return (x_train, y_train), (x_test, y_test) def load_fashion(): train, test = load_data(fashion = True, normalize = True) - x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.6) + x_train, x_test, y_train, y_test = train_test_split(test[0], test[1], test_size = 0.2) return (x_train, y_train), (x_test, y_test) def load_mnist_two_digits(digit1, digit2): @@ -213,4 +198,21 @@ def load_activity(): print(x_train.shape, y_train.shape) print(x_test.shape, y_test.shape) - return (x_train, y_train), (x_test, y_test) \ No newline at end of file + return (x_train, y_train), (x_test, y_test) + +def load_dataset(dataset): + if dataset == 'MNIST': + return load_mnist() + elif dataset == 'MNIST-Fashion': + return load_fashion() + if dataset == 'MICE': + return load_mice() + elif dataset == 'COIL': + return load_coil() + elif dataset == 'ISOLET': + return load_isolet() + elif dataset == 'Activity': + return load_activity() + else: + print('Please specify a valid dataset') + return None \ No newline at end of file From 3727b2a633842df996b2ec142a4b076e64c34e10 Mon Sep 17 00:00:00 2001 From: Hadi Date: Fri, 15 Mar 2024 12:33:51 +0100 Subject: [PATCH 3/4] README --- experiments/README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/README.MD b/experiments/README.MD index c25fe5f..ee4bbb4 100755 --- a/experiments/README.MD +++ b/experiments/README.MD @@ -1,5 +1,5 @@ - The data to reproduce Table 1 are available in [this Google Drive repository](https://drive.google.com/open?id=1quiURu7w0nU3Pxcc448xRgfeI80okLsS). Alternatively, you can download all the data sets (except MNIST and MNIST-Fashion) directly from the [UCI Repository](https://archive.ics.uci.edu/ml/datasets.php). - `data-utils.py` contains starter code to load the 6 datasets in Table 1 of [the paper](https://arxiv.org/abs/1907.12207). - You will need to download the files, unzip them and modify the `/home/lemisma/datasets` path in `data-utils.py` to point to your local copy. +- `run.py` contains the necessary code to reproduce the results in Table 1. This script allows you to specify the dataset and other parameters. To run the script, navigate to the directory containing `run.py` and use the following command: `python run.py`. -We may release additional software to reproduce the experiments, if there is strong user demand. From bbe4078127880cb75ca38c2a4717d722a9abbc90 Mon Sep 17 00:00:00 2001 From: Hadi Date: Fri, 15 Mar 2024 12:34:13 +0100 Subject: [PATCH 4/4] Code to reproduce experiments --- experiments/run.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 experiments/run.py diff --git a/experiments/run.py b/experiments/run.py new file mode 100644 index 0000000..8451958 --- /dev/null +++ b/experiments/run.py @@ -0,0 +1,47 @@ +from sklearn.model_selection import train_test_split +from lassonet import LassoNetClassifier +from lassonet.utils import eval_on_path +from data_utils import load_dataset +import torch +import pickle + +seed = None +device = 'cuda' if torch.cuda.is_available() else 'cpu' +batch_size = 256 +K = 50 # Number of features to select +n_epochs = 1000 +dataset = 'ISOLET' + +# Load dataset and split the data +(X_train_valid, y_train_valid), (X_test, y_test) = load_dataset(dataset) +X_train, X_val, y_train, y_val = train_test_split(X_train_valid, y_train_valid, test_size=0.125, random_state=seed) + +# Set the dimensions of the hidden layers +data_dim = X_test.shape[1] +hidden_dim = (data_dim//3,) + +# Initialize the LassoNetClassifier model and compute the path +lasso_model = LassoNetClassifier(M=10, hidden_dims=hidden_dim, verbose=1, torch_seed=seed, random_state=seed, device=device, n_iters=n_epochs, batch_size=batch_size) +path = lasso_model.path(X_train, y_train, X_val=X_val, y_val=y_val) + +# Select the features +desired_save = next(save for save in path if save.selected.sum().item() <= K) +SELECTED_FEATURES = desired_save.selected +print("Number of selected features:", SELECTED_FEATURES.sum().item()) + +# Select the features from the training, validation, and test data +X_train_selected = X_train[:, SELECTED_FEATURES] +X_val_selected = X_val[:, SELECTED_FEATURES] +X_test_selected = X_test[:, SELECTED_FEATURES] + +# Initialize another LassoNetClassifier for retraining with the selected features +lasso_sparse = LassoNetClassifier(M=10, hidden_dims=hidden_dim, verbose=1, torch_seed=seed, random_state=seed, device=device, n_iters=n_epochs) +path_sparse = lasso_sparse.path(X_train_selected, y_train, X_val=X_val_selected, y_val=y_val, lambda_seq=[0], return_state_dicts=True)[:1] + +# Evaluate the model on the test data +score = eval_on_path(lasso_sparse, path_sparse, X_test_selected, y_test) +print("Test accuracy:", score) + +# Save the path +with open(f'{dataset}_path.pkl', 'wb') as f: + pickle.dump(path_sparse, f) \ No newline at end of file