restart

src/main/java/focusedCrawler/rest/RestServer.java

@@ -102,7 +102,8 @@ public void start() {
         server.post("/crawls/:crawler_id/labels",     json(labelsResource.addLabels));
         server.get( "/crawls/:crawler_id/_search",    elasticsearchProxyResource.searchApi);
         server.post("/crawls/:crawler_id/_search",    elasticsearchProxyResource.searchApi);
-
+        server.post("/crawls/:crawler_id/link_filter",   json(crawlerResource.addurl));
+        server.post("/crawls/:crawler_id/restartCrawl",  json(crawlerResource.restartCrawl));
         /*
          * Thread management routes
          */

src/main/java/focusedCrawler/rest/resources/CrawlerResource.java

@@ -6,13 +6,16 @@
 import java.util.Optional;
 
 import javax.servlet.http.HttpServletResponse;
+import com.fasterxml.jackson.core.JsonParseException;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableMap;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLMapper;
 
 import focusedCrawler.Main;
 import focusedCrawler.crawler.CrawlersManager;
@@ -23,8 +26,22 @@
 import focusedCrawler.util.MetricsManager;
 import spark.Request;
 import spark.Route;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Collection;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.annotation.concurrent.Immutable;
+import org.apache.commons.io.FileUtils;
+import java.util.concurrent.TimeUnit ;
 
 public class CrawlerResource {
+    private Map<String, CrawlContext> crawlers = new HashMap<>();
 
     public static final String VERSION = Main.class.getPackage().getImplementationVersion();
 
@@ -54,7 +71,7 @@ public CrawlerResource(CrawlersManager crawlManager) {
         Map<String, CrawlContext> crawlers = crawlersManager.getCrawls();
         return ImmutableMap.of("crawlers", crawlers.values());
     };
-
+   
     public Route metricsResource = (request, response) -> {
         String crawlerId = request.params(":crawler_id");
 
@@ -68,10 +85,50 @@ public CrawlerResource(CrawlersManager crawlManager) {
         return metricsManager != null ? metricsManager.getMetricsRegistry() : null;
     };
 
+    public Route addurl = (request, response) -> {
+          try {
+            String crawlerId = request.params(":crawler_id");
+            CrawlContext context = crawlersManager.getCrawl(crawlerId);  
+            System.out.println(context);        
+            System.out.print(request.body());
+            ObjectMapper mapper = new ObjectMapper();
+            JsonNode jsonNode = mapper.readTree(request.body());
+            String yaml = new YAMLMapper().writeValueAsString(jsonNode);
+            String path;
+            String directory;
+            if (context == null) {
+                response.status(HttpServletResponse.SC_NOT_FOUND);
+            return ImmutableMap.of("message", "Crawler not found for crawler_id " + crawlerId);
+            }
+               path = context.seedPath;
+               File f = new File(path);
+               directory = f.getParent();
+               String link_filters_path = directory + File.separator + "link_filters.yaml";
+               System.out.println(link_filters_path);
+               File link_filter_file = new File(link_filters_path);
+               link_filter_file.delete();
+           //create new file with content 
+               link_filter_file.createNewFile();
+
+            FileWriter myWriter = new FileWriter(link_filter_file);
+            //FileWriter myWriter = new FileWriter(link_filters_path);
+            myWriter.write(yaml);
+            myWriter.close();
+            System.out.println("Successfully wrote to the file.");
+               return ImmutableMap.of("output", yaml);
+
+            } catch (Exception e) {
+                logger.error("failed.", e);
+                response.status(HttpServletResponse.SC_NOT_ACCEPTABLE);
+                System.out.println("hello");
+                return ImmutableMap.of("message", false);
+
+            }
+          };
+
     public Route startCrawl = (request, response) -> {
         try {
             String crawlerId = request.params(":crawler_id");
-
             StartCrawlParams params = json.readValue(request.body(), StartCrawlParams.class);
 
             crawlersManager.createCrawler(crawlerId, params);
@@ -89,6 +146,49 @@ public CrawlerResource(CrawlersManager crawlManager) {
                 "crawlerStarted", false);
         }
     };
+    public Route restartCrawl = (request, response) -> {
+        try {
+            String crawlerId = request.params(":crawler_id");
+            StartCrawlParams params = json.readValue(request.body(), StartCrawlParams.class);
+            CrawlContext context = crawlersManager.getCrawl(crawlerId);          
+            System.out.println(context);
+            String path;
+            String directory;
+            if (context == null) {
+                response.status(HttpServletResponse.SC_NOT_FOUND);
+            return ImmutableMap.of("message", "Crawler not found for crawler_id " + crawlerId);
+            }
+
+            path = context.dataPath;
+            File f = new File(path);
+            directory = f.getParent();
+            String crawler_path = directory + File.separator + crawlerId;
+            File crawler_directory = new File(crawler_path);
+
+            AsyncCrawler crawler = context.getCrawler();
+            crawler.stopAsync();
+                crawler.awaitTerminated();
+                if (crawler_directory.exists()) {
+                FileUtils.cleanDirectory(crawler_directory); 
+                FileUtils.forceDelete(crawler_directory);
+                System.out.println("folder deleted");
+            }
+                crawlersManager.createCrawler(crawlerId, params);
+                crawlersManager.startCrawl(crawlerId);
+                System.out.println("Crawler stopped successfully");
+
+            return ImmutableMap.of(
+                "message", "Crawler restarted successfully.",
+                "crawlerreStarted", true);
+
+        } catch (Exception e) {
+            logger.error("Failed to restart crawler.", e);
+            response.status(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+            return ImmutableMap.of(
+                "message", "Failed to restart crawler.",
+                "crawlerreStarted", false);
+        }
+    };
 
     public Route stopCrawl = (request, response) -> {
         try {

src/main/java/focusedCrawler/target/repository/ElasticSearchRestTargetRepository.java

@@ -193,7 +193,7 @@ public RestClient createRestClient(ElasticSearchConfig config) {
         for (String host : esHosts) {
             try {
                 URL url = new URL(host);
-                hosts.add(new HttpHost(url.getHost(), url.getPort()));
+                hosts.add(new HttpHost(url.getHost(), url.getPort(), url.getProtocol()));
             } catch (MalformedURLException e) {
                 throw new RuntimeException("Failed to initialize Elasticsearch REST client. "
                         + "Invalid host: " + host, e);