Merge branch 'dev' of https://github.com/crossminer/scava into dev

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/IImporter.java

@@ -21,7 +21,7 @@
  *
  */
 public interface IImporter {
-	Artifact importProject(String artId) 
+	Artifact importProject(String artId, String access_token) 
 			throws IOException;
 	void importAll();
 	void storeGithubUserCommitter(List<GithubUser> committers, String repoName);

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/ISimilarityManager.java

@@ -40,4 +40,5 @@ public interface ISimilarityManager {
 	void storeDistanceMatrix(Table<String, String, Double> distanceMatrix, ISimilarityCalculator similarityCalculator);
 	void deleteRelations(ISimilarityCalculator simCalculator);
 	List<Relation> getRelations(ISimilarityCalculator simCalculator);
+	void storeAllSimilarityDistances();
 }

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/SchedulerManager.java

@@ -9,55 +9,32 @@
  ******************************************************************************/
 package org.eclipse.scava.business;
 
+import java.time.LocalDateTime;
+
+import org.eclipse.scava.business.impl.SimilarityManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Component;
 
 @Component
 public class SchedulerManager {
 
 	private static final Logger logger = LoggerFactory.getLogger(SchedulerManager.class);
-//	@Autowired
-//	private List<ISimilarityCalculator> simCalcs;
-//
-//	@Autowired
-//	private SimilarityManager simManager;
-//
-//	@Autowired
-//	private ClusterManager clusterManager;
-//
-//	@Autowired
-//	@Qualifier("Ossmeter")
-//	private IImporter importer;
-//		
-//	@Autowired
-//	private CROSSRecSimilarityCalculator crossRecSimilarityCalculator;
 
-	// TODO DE-COMMENT ROW BELOW TO ENABLE TASK EXECUTION
-	// @Scheduled(cron = "0 53 12 * * *")
+	@Autowired
+	private SimilarityManager simManager;
+
+	@Scheduled(cron = "0 30 20 ? * SUN")
+//	@Scheduled(cron = "0 0/5 * * * ?")
 	public void scheduler() {
-//		try {
-//			logger.info("Importing projects from scava platform");
-//			importer.importAll();
-//			crossRecSimilarityCalculator.createCROSSRecGraph();
-//			logger.info("Imported projects from scava platform");
-//		} catch (Exception e) {
-//		}
-//
-//		for (ISimilarityCalculator iSimilarityCalculator : simCalcs)
-//			try {
-//				logger.info("Generating: " + iSimilarityCalculator.getSimilarityName() + " distance matrix");
-//				simManager.deleteRelations(iSimilarityCalculator);
-//				simManager.createAndStoreDistanceMatrix(iSimilarityCalculator);
-//				logger.info("Generated: " + iSimilarityCalculator.getSimilarityName() + " distance matrix");
-//			} catch (Exception e) {
-//			}
-//		for (ISimilarityCalculator iSimilarityCalculator : simCalcs)
-//			try {
-//				logger.info("Generating: " + iSimilarityCalculator.getSimilarityName() + " clusterization");
-//				clusterManager.calculateAndStoreClusterization(iSimilarityCalculator);
-//				logger.info("Generated: " + iSimilarityCalculator.getSimilarityName() + " clusterization");
-//			} catch (Exception e) {
-//			}
+		try {
+			logger.info("Offline computating at: {}", LocalDateTime.now());
+			simManager.storeAllSimilarityDistances();
+			logger.info("Offline computated at: {}", LocalDateTime.now());
+		} catch (Exception e) {
+			logger.error("error in computing distances: {}", e.getMessage());
+		}
 	}
 }

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/impl/CROSSRecSimilarityCalculator.java

@@ -43,6 +43,8 @@ public class CROSSRecSimilarityCalculator implements IAggregatedSimilarityCalcul
 
 	@Autowired
 	private CROSSRecGraphRepository crossRecGraphRepository;
+	@Autowired
+	private SimilarityManager simManager;
 
 	@Override
 	public boolean appliesTo(Artifact art) {
@@ -84,12 +86,7 @@ public Table<String, String, Double> calculateAggregatedSimilarityValues(List<Ar
 	 * 
 	 */
 	public Map<String, Double> computeWeightCosineSimilarity(List<Dependency> dependencies) throws Exception{
-		CROSSRecGraph bigGraph;
-		List<CROSSRecGraph> tempGraphs = crossRecGraphRepository.findAll();
-		if(tempGraphs.size() == 0) {
-			bigGraph = createCROSSRecGraph();
-		}
-		else bigGraph = tempGraphs.get(0);
+		CROSSRecGraph bigGraph = createCROSSRecGraph();
 
 		Set<String> allLibs = extractDepsfromGraph(bigGraph);
 		/*add all libraries from the training set*/
@@ -161,7 +158,7 @@ public Map<String, Double> computeWeightCosineSimilarity(List<Dependency> depend
 	}
 
 	public CROSSRecGraph createCROSSRecGraph() {
-		List<Artifact> arts = artifactRepository.findAll();
+		List<Artifact> arts = simManager.appliableProjects(this);
 		CROSSRecGraph graph = null;
 		for (Artifact artifact : arts) {
 			CROSSRecGraph graph1 = createGraphFromArtifact(artifact);

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/impl/GithubImporter.java

@@ -76,7 +76,6 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 /**
@@ -87,7 +86,7 @@
 @Qualifier("Github")
 public class GithubImporter implements IImporter {
 
-	@Value("${egit.github.token}")
+//	@Value("${egit.github.token}")
 	private String token;
 	private static final String UTF8 = "UTF-8";
 	@Autowired
@@ -99,7 +98,8 @@ public class GithubImporter implements IImporter {
 	private static final Logger logger = LoggerFactory.getLogger(GithubImporter.class);
 
 	@Override
-	public Artifact importProject(String artId) throws IOException {
+	public Artifact importProject(String artId, String access_token) throws IOException {
+		token = access_token;
 		Artifact checkRepo = projectRepository.findOneByFullName(artId);
 		if (checkRepo != null) {
 			logger.info("\t" + artId + " already in DB");
@@ -124,8 +124,14 @@ public Artifact importProject(String artId) throws IOException {
 			p.setMaster_branch(rep.getDefaultBranch());
 			p.setHomePage(rep.getHomepage());
 			p.setName(rep.getName());
-			p.setShortName(p.getShortName());
-
+			p.setShortName(artId.split("/")[1]);
+			try {
+				String[] license = getLicense(rep);
+				p.setLicenseName(license[0]);
+				p.setLicenseUrl(license[1]);
+			} catch (Exception e) {
+				logger.info("License not found");
+			}
 			try {
 				p.setCommitteers(getCommitters(rep));
 			} catch (MalformedURLException e) {
@@ -152,17 +158,13 @@ public Artifact importProject(String artId) throws IOException {
 				logger.error("Error getting stars" + e.getMessage());
 			}
 			p.setTags(getTags(artId.split("/")[0], artId.split("/")[1]));
-
-//			if(p.getDependencies() != null && p.getDependencies().size()>8) {
 			storeGithubUserCommitter(p.getCommitteers(), p.getFullName());
 			storeGithubUser(p.getStarred(), p.getFullName());
 			projectRepository.save(p);
 			logger.debug("Imported project: " + artId);
-//			}
-
 			return p;
 		} catch (Exception e) {
-			logger.error(e.getMessage());
+			logger.error("Project {} is not imported: {}", artId, e.getMessage());
 			throw e;
 		}
 	}
@@ -214,6 +216,28 @@ public List<Tag> getTags(String owner, String repo) throws IOException {
 		return results;
 	}
 
+	private String[] getLicense(Repository rep) throws IOException {
+		String[] results = new String[2];
+		if (getRemainingResource("core") == 0)
+			waitApiCoreRate();
+		URL url;
+		url = new URL("https://api.github.com/repos/" + rep.getOwner().getLogin() + "/" + rep.getName()
+				+ "?access_token=" + token);
+		HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
+		connection.setRequestProperty("Accept", "application/vnd.github.v3.star+json");
+		connection.connect();
+		InputStream is = connection.getInputStream();
+		BufferedReader bufferReader = new BufferedReader(new InputStreamReader(is, Charset.forName(UTF8)));
+		String jsonText = readAll(bufferReader);
+		JSONObject obj = (JSONObject) JSONValue.parse(jsonText);
+		JSONObject license = (JSONObject) obj.get("license");
+		results[0] = (String) license.get("name");
+		results[1] = (String) license.get("url");
+
+		return results;
+
+	}
+
 	private List<Stargazers> getStargazers(Repository rep) throws IOException {
 		List<Stargazers> results = new ArrayList<>();
 		int page = 1;
@@ -330,10 +354,10 @@ public void importAll() {
 						String fullname = entry.get("full_name").toString();
 						try {
 							RepositoryId repo = new RepositoryId(fullname.split("/")[0], fullname.split("/")[1]);
-							importProject(fullname);// (repo,
-													// pomFiles,
-													// pomPath,
-													// users);
+							importProject(fullname, token);// (repo,
+							// pomFiles,
+							// pomPath,
+							// users);
 							Files.write(Paths.get("repo2.txt"),
 									(repo.getOwner() + "/" + repo.getName() + "\n").getBytes(),
 									StandardOpenOption.APPEND);

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/impl/OssmeterImporter.java

@@ -76,7 +76,7 @@ private static String readAll(Reader rd) throws IOException {
 	}
 
 	@Override
-	public Artifact importProject(String projectName) throws IOException {
+	public Artifact importProject(String projectName, String access_token) throws IOException {
 
 		Artifact result = projectRepository.findOneByName(projectName);
 		if (result == null)
@@ -173,7 +173,7 @@ public void importAll() {
 					guard = false;
 				else
 					for (Object object : projects)
-						importProject((String) ((JSONObject) object).get("name"));
+						importProject((String) ((JSONObject) object).get("name"), null);
 				page++;
 			} catch (IOException e) {
 				guard = false;

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/impl/RecommenderManager.java

@@ -39,6 +39,7 @@
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.domain.Sort.Direction;
 import org.springframework.data.mongodb.core.MongoTemplate;
+import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.TextCriteria;
 import org.springframework.data.mongodb.core.query.TextQuery;
 import org.springframework.stereotype.Service;
@@ -222,7 +223,7 @@ public List<Artifact> getArtifactsByQuery(String projectQuery, Pageable page) {
 		TextCriteria criteria = TextCriteria.forDefaultLanguage().matchingPhrase(projectQuery);
 		org.springframework.data.mongodb.core.query.Query query = TextQuery.queryText(criteria).sortByScore()
 				.with(page);
-
+		query.addCriteria(Criteria.where("type.name").ne("FOCUS"));
 		List<Artifact> recipes = template.find(query, Artifact.class);
 		if (page.getSort().getOrderFor("temp").getDirection() == Direction.ASC)
 			return Lists.reverse(recipes);

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/impl/SimilarityManager.java

@@ -60,7 +60,24 @@ public class SimilarityManager implements ISimilarityManager {
 
 	@Autowired
 	MongoOperations mongoOperations;
-
+
+	@Autowired
+	List<ISimilarityCalculator> simCalcs;
+
+	@Override
+	public void storeAllSimilarityDistances() {
+		for (ISimilarityCalculator similarityCalculator : simCalcs) {
+			try {
+				logger.info("Computing distance matrix for: {}", similarityCalculator.getSimilarityName());
+				createAndStoreDistanceMatrix(similarityCalculator);
+				logger.info("Computed distance matrix for: {}", similarityCalculator.getSimilarityName());
+			}
+			catch (Exception e) {
+				logger.error("error when computing {} distances: {}", similarityCalculator.getSimilarityName(), e.getMessage());
+			}
+		}
+	}
+
 	@Override
 	public Set<Relation> getSimilarProjectsRelations(Artifact prj1, ISimilarityCalculator similarityCalculator) {
 		Query q1 = new Query(Criteria.where("type.name").is(similarityCalculator.getSimilarityName()).orOperator(
@@ -109,40 +126,10 @@ public List<Artifact> getSimilarProjects(Artifact prj1, double threshold) {
 	 */
 	@Override
 	public void createAndStoreDistanceMatrix(ISimilarityCalculator simCalculator) {
-		List<Artifact> artifacts = appliableProjects(simCalculator);
-		if (simCalculator instanceof ISingleSimilarityCalculator) {
-			ISingleSimilarityCalculator singleCalculator = (ISingleSimilarityCalculator) simCalculator;
-			Artifact[] artifactsArray = new Artifact[artifacts.size()];
-			artifactsArray = artifacts.toArray(artifactsArray);
-			for (int i = 0; i < artifactsArray.length - 1; i++) {
-				for (int j = i + 1; j < artifactsArray.length; j++) {
-					double similarity = 0;
-					try {
-						similarity = singleCalculator.calculateSimilarity(artifactsArray[i], artifactsArray[j]);
-					} catch (Exception e) {
-						logger.error(e.getMessage());
-					}
-					RelationType relType = getRelationType(simCalculator.getSimilarityName());
-					Relation rel = new Relation();
-					rel.setType(relType);
-					rel.setFromProject(artifactsArray[i]);
-					rel.setToProject(artifactsArray[j]);
-					rel.setValue(similarity);
-					relationRepository.save(rel);
-				}
-			}
-		} else {
-			IAggregatedSimilarityCalculator aggregateSimilarityCalculator = (IAggregatedSimilarityCalculator) simCalculator;
-			Map<String, String> map = new HashMap<>();
-			map.put("committers", "true");
-			map.put("deps", "true");
-			map.put("stargazers", "true");
-			map.put("freqDeps", "129");
-			Table<String, String, Double> distanceMatrix = aggregateSimilarityCalculator
-					.calculateAggregatedSimilarityValues(artifacts, map);
-			storeDistanceMatrix(distanceMatrix, aggregateSimilarityCalculator);
-
-		}
+		List<Relation> dbd = relationRepository.findAllByTypeName(simCalculator.getSimilarityName());
+		logger.info("#Reletaion to be deleted {}", dbd.size());
+		relationRepository.delete(dbd);
+		storeDistanceMatrix(createDistanceMatrix(simCalculator), simCalculator);
 	}
 
 	public Map<String, Double> getSimilarProjectsOnLine(ISimilarityCalculator simCalculator, Artifact art) {
@@ -324,7 +311,8 @@ public List<Artifact> getSimilarProjects(Artifact prj1, int numResult) {
 
 	@Override
 	public List<Artifact> appliableProjects(ISimilarityCalculator simCalculator) {
-		return artifactRepository.findAll().stream().filter(z -> simCalculator.appliesTo(z)).collect(Collectors.toList());
+		return artifactRepository.findAll().stream().filter(z -> simCalculator.appliesTo(z))
+				.collect(Collectors.toList());
 	}
 
 }

knowledge-base/org.eclipse.scava.knowledgebase/src/main/java/org/eclipse/scava/business/model/Artifact.java

@@ -46,6 +46,8 @@ public class Artifact {
 	@DBRef
 	private List<Artifact> subArtifacts;
 	private ArtifactType type;
+	private String licenseName;
+	private String licenseUrl;
 	private Boolean private_;
 	private Boolean fork;
 	private String html_url;
@@ -307,6 +309,26 @@ public List<MethodDeclaration> getMethodDeclarations() {
 	public void setMethodDeclarations(List<MethodDeclaration> methodDeclarations) {
 		this.methodDeclarations = methodDeclarations;
 	}
+
+
+	public String getLicenseName() {
+		return licenseName;
+	}
+
+
+	public void setLicenseName(String licenseName) {
+		this.licenseName = licenseName;
+	}
+
+
+	public String getLicenseUrl() {
+		return licenseUrl;
+	}
+
+
+	public void setLicenseUrl(String licenseUrl) {
+		this.licenseUrl = licenseUrl;
+	}