Merge pull request #27 from yana-safonova/ig_rep_con_2.0

Ig_repertoire_constructor 2.0 release

.gitignore

@@ -0,0 +1 @@
+*.pyc

Makefile

@@ -1,17 +1,14 @@
 all:
 	$(MAKE) -C build/release all
-	$(MAKE) -C build/debug all
-	g++ src/ig_tools/paired_read_merger/main.cpp -o build/release/bin/paired_read_merger
-	g++ src/ig_tools/fastq_to_fasta/fastq_to_fasta.cpp -o build/release/bin/fastq_to_fasta
-	g++ -std=c++11 src/ig_tools/merged_reads_stats_calculator/main.cpp -o build/release/bin/compute_merged_reads_stats
 
 rig:
 	$(MAKE) -C build/release/ig_repertoire_constructor ig_repertoire_constructor
 
 dig:
 	$(MAKE) -C build/debug/ig_repertoire_constructor ig_repertoire_constructor
 
-igtools:
-	g++ src/ig_tools/paired_read_merger/main.cpp -o build/release/bin/paired_read_merger
-	g++ src/ig_tools/fastq_to_fasta/fastq_to_fasta.cpp -o build/release/bin/fastq_to_fasta
-	g++ -std=c++11 src/ig_tools/merged_reads_stats_calculator/main.cpp -o build/release/bin/compute_merged_reads_stats
+dsf:
+	$(MAKE) -C build/release/dense_sgraph_finder dense_sgraph_finder
+
+metis:
+	$(MAKE) -C build/release/ext_tools/metis-5.1.0/ metis

VERSION

@@ -1 +1 @@
-1.0.0
+2.0.0

cleanup

@@ -1,5 +1,4 @@
 rm *.pyc
-rm -rf ig_data_cleaner_test
-rm -rf ig_repertoire_constructor_test
+rm -rf igrepcon_test
 rm -rf ms_analyzer_test
 rm *~

configs/.gitignore

@@ -1,2 +1,4 @@
 ig_repertoire_constructor/config.info
 ig_repertoire_constructor/log.properties
+dense_sgraph_finder/config.info
+dense_sgraph_finder/log.properties

configs/dense_sgraph_finder/config.info.template

@@ -0,0 +1,46 @@
+; input-output parameters ;
+io {
+	input {
+                graph_filename          test_dataset/test.graph
+	}
+
+	output_base {
+		log_filename 		log.properties
+    		output_dir      	dsf_test
+		decomposition_filename	dense_subgraphs.txt
+	}
+
+	output_nonparallel {
+		graph_copy_filename	graph_copy.graph
+		permutation_filename	graph_copy.graph.iperm
+	}
+
+	output_mthreading {
+		connected_components_dir		connected_components
+		decompositions_dir			dense_subgraphs
+		output_component_decompositions		true
+	}
+}
+
+; run params ;
+rp {
+	developer_mode	true
+    	threads_count   16
+    	max_memory      250
+}
+
+; algorithm parameters ;
+dsf_params {
+	min_graph_size			5
+	primary_edge_fillin		0.3
+	min_fillin_threshold		0.6
+	create_trivial_decomposition	false
+}
+
+; input-output parameters of METIS ;
+metis_io {
+	path_to_metis                   build/release/bin/
+        run_metis                       ./metis
+        trash_output                    metis.output
+}
+

configs/dense_sgraph_finder/log.properties.template

@@ -0,0 +1,7 @@
+default=INFO
+
+#Permutation=TRACE
+#Decomposition=TRACE
+#SimpleDecompositionConstructor=DEBUG
+#GreedyJoiningDecomposition=DEBUG
+#DecompositionStatsCalculator=TRACE

cpcfg

@@ -27,11 +27,18 @@ for arg; do
 case $arg in
   'ig') $cmd configs/ig_repertoire_constructor/config.info.template configs/ig_repertoire_constructor/config.info;;
   'logig') $cmd configs/ig_repertoire_constructor/log.properties.template configs/ig_repertoire_constructor/log.properties;;
+  'dsf') $cmd configs/dense_sgraph_finder/config.info.template configs/dense_sgraph_finder/config.info;;
+  'logdsf') $cmd configs/dense_sgraph_finder/log.properties.template configs/dense_sgraph_finder/log.properties;;
   '-all')
 		echo -----------configs/ig_repertoire_constructor/config.info--------
 		$cmd configs/ig_repertoire_constructor/config.info.template configs/ig_repertoire_constructor/config.info
 		echo ----------configs/ig_repertoire_constructor/log.properties------
-		$cmd configs/ig_repertoire_constructor/log.properties.template configs/ig_repertoire_constructor/log.properties 
+		$cmd configs/ig_repertoire_constructor/log.properties.template configs/ig_repertoire_constructor/log.properties
+                echo -------------configs/dense_sgraph_finder/config.info------------
+                $cmd configs/dense_sgraph_finder/config.info.template configs/dense_sgraph_finder/config.info
+                echo -------------configs/dense_sgraph_finder/log.properties---------
+                $cmd configs/dense_sgraph_finder/log.properties.template configs/dense_sgraph_finder/log.properties
+
 esac
 
 done

dense_subgraph_finder.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python2
+
+import sys
+import os
+import init
+import logging
+import shutil
+
+home_directory = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/'
+spades_src = os.path.join(home_directory, "src/python_pipeline/")
+config_dir = os.path.join(home_directory, "configs/dense_sgraph_finder/")
+ig_binary = os.path.join(home_directory, "build/release/bin/ig_repertoire_constructor")
+
+sys.path.append(spades_src)
+import process_cfg
+import support
+
+def CheckParamsCorrectness(params, log, parser):
+    if not "output" in params or params.output == "":
+        log.info("ERROR: Output directory (-o) was not specified")
+        parser.print_help()
+        sys.exit(-1)
+    if not "graph" in params or params.graph == "":
+        log.info("ERROR: Graph (-g/--graph) was not specified")
+        parser.print_help()
+        sys.exit(-1)
+    if not os.path.exists(params.graph):
+        log.info("ERROR: File with graph " + params.graph + " was not found")
+        parser.print_help()
+        sys.exit(-1)
+    if not os.path.isabs(params.graph):
+        params.graph = os.path.abspath(params.graph)
+
+def PrintParams(params, log):
+    log.info("DSF parameters:")
+    log.info("  Input graph:\t\t\t\t\t" + params.graph)
+    log.info("  Output directory:\t\t\t\t" + params.output)
+    log.info("  Number of threads:\t\t\t\t" + str(params.num_threads))
+    log.info("  Minimum size of processed graphs:\t\t" + str(params.min_graph_size))
+    log.info("  Minimum edge fill-in of dense subgraph:\t" + str(params.min_fillin))
+    if params.create_trivial_decomposition:
+        log.info("  Trivial decomposition will be created")
+    log.info("  Output auxiliary files:\t\t\t" + str(params.save_aux_files))
+    log.info("\n")
+
+def supportInfo(log):
+    log.info("\nIn case you have troubles running DSF, you can write to [email protected].")
+    log.info("Please provide us with dense_sungraph_finder.log file from the output directory.")
+
+def SetOutputParams(params, output_dir):
+    params.output = os.path.abspath(output_dir)
+    params.config_dir = os.path.join(params.output, params.config_dir)
+    params.config_file = os.path.join(params.config_dir, params.config_file)
+
+def PrepareOutputDir(params, log):
+    if os.path.exists(params.output):
+        shutil.rmtree(params.output)
+    if not os.path.isdir(params.output):
+        os.makedirs(params.output)
+
+def CopyConfigs(params, log):
+    if os.path.exists(params.config_dir):
+        shutil.rmtree(params.config_dir)
+    shutil.copytree(config_dir, params.config_dir)
+
+def CreateParamDict(params):
+    param_dict = dict()
+    param_dict['output_dir'] = params.output
+    param_dict['graph_filename'] = params.graph
+    param_dict['threads_count'] = params.num_threads
+    param_dict['min_fillin_threshold'] = params.min_fillin
+    param_dict['min_graph_size'] = params.min_graph_size
+    param_dict['create_trivial_decomposition'] = process_cfg.bool_to_str(params.create_trivial_decomposition)
+    return param_dict
+
+def PrepareConfigs(params, log):
+    CopyConfigs(params, log)
+    param_dict = CreateParamDict(params)
+    if not os.path.exists(params.config_file):
+        log.info("ERROR: config file was not found")
+        sys.exit(1)
+    process_cfg.substitute_params(params.config_file, param_dict, log)
+
+def Cleanup(params, log):
+    params.sgraph_dir = os.path.join(params.output, "connected_components")
+    params.decomposition_dir = os.path.join(params.output, "dense_subgraphs")
+    params.metis_output = os.path.join(params.output, "metis.output")
+    if os.path.exists(params.metis_output):
+        os.remove(params.metis_output)
+    if not "save_aux_files" in params or not params.save_aux_files:
+        if os.path.exists(params.sgraph_dir):
+            shutil.rmtree(params.sgraph_dir)
+        if os.path.exists(params.decomposition_dir):
+            shutil.rmtree(params.decomposition_dir)
+
+def main(argv, external_logger = ""):
+    from src.python_add.argparse_ext import ArgumentHiddenParser
+    parser = ArgumentHiddenParser(description="== DSF: an algorithm for corrupted cliques search ==",
+                                  epilog="""
+                                  In case you have troubles running DSF, you can write to [email protected].
+                                  Please provide us with dense_subgraph_finder.log file from the output directory.
+                                  """,
+                                  add_help=False)
+    req_args = parser.add_argument_group("Input")
+    input_args = req_args.add_mutually_exclusive_group(required=True)
+    input_args.add_argument("-g", "--graph",
+                            type=str,
+                            default="",
+                            dest="graph",
+                            help="Input graph in GRAPH format")
+    input_args.add_argument("--test",
+                            action="store_const",
+                            const="test_dataset/test.graph",
+                            dest="graph",
+                            help="Running test dataset")
+
+    out_args = parser.add_argument_group("Output")
+    out_args.add_argument("-o", "--output",
+                            type=str,
+                            default="dsf_test",
+                            help="Output directory")
+
+    optional_args = parser.add_argument_group("Optional arguments")
+    optional_args.add_argument("-t", "--threads",
+                               type=int,
+                               default=16,
+                               dest="num_threads",
+                               help="Threads number [default: %(default)d]")
+    optional_args.add_argument("-f", '--min-fillin',
+                               type=float,
+                               default=0.6,
+                               dest="min_fillin",
+                               help='Minimum fill-in of dense subgraphs [default: %(default)f]')
+    optional_args.add_argument("-s", "--min-size",
+                               type=int,
+                               default=5,
+                               dest="min_graph_size",
+                               help="Minimum size of graph where dense subgraphs will be computed [default: %(default)d]")
+    optional_args.add_argument("--create-triv-dec",
+                               action="store_const",
+                               const=True,
+                               dest="create_trivial_decomposition",
+                               help='Creating decomposition according to connected components [default: False]')
+    optional_args.add_argument("--save-aux-files",
+                               action="store_const",
+                               const=True,
+                               dest="save_aux_files",
+                               help="Saving auxiliary files: subgraphs in GRAPH format and their decompositions "
+                                    "[default: False]")
+    optional_args.add_argument("-h", "--help",
+                               action="help",
+                               help="Help message and exit")
+
+    parser.set_defaults(config_dir="configs",
+                        config_file="config.info")
+
+    # prepare log
+    log = logging.getLogger('dense_subgraph_finder')
+    log.setLevel(logging.DEBUG)
+    console = logging.StreamHandler(sys.stdout)
+    console.setFormatter(logging.Formatter('%(message)s'))
+    console.setLevel(logging.DEBUG)
+    log.addHandler(console)
+    if external_logger != "":
+        external_log_handler = logging.FileHandler(external_logger, mode = "a")
+        log.addHandler(external_log_handler)
+
+    # parse command line
+    args = [arg for arg in argv if arg != './dense_subgraph_finder.py']
+    params = parser.parse_args(args)
+
+    CheckParamsCorrectness(params, log, parser)
+    SetOutputParams(params, params.output)
+
+    PrepareOutputDir(params, log)
+
+    # log file
+    params.log_filename = os.path.join(params.output, "dense_subgraph_finder.log")
+    if os.path.exists(params.log_filename):
+        os.remove(params.log_filename)
+    log_handler = logging.FileHandler(params.log_filename, mode='a')
+    log.addHandler(log_handler)
+
+    # print command line
+    command_line = "Command_line: "
+    if argv[0] != "dense_subgraph_finder.py":
+        command_line += "dense_subgraph_finder.py "
+    command_line += " ".join(argv)
+    log.info(command_line + "\n")
+    PrintParams(params, log)
+    log.info("Log will be written to " + params.log_filename + "\n")
+
+    PrepareConfigs(params, log)
+
+    # run dense subgraph finder
+    try:
+        dsf_command_line = init.PathToBins.run_dense_sgraph_finder + " " + params.config_file
+        support.sys_call(dsf_command_line, log)
+        Cleanup(params, log)
+        log.info("\nThank you for using Dense Subgraph Finder!\n")
+    except (KeyboardInterrupt):
+        log.info("\nDense subgraph finder was interrupted!")
+    except Exception:
+        exc_type, exc_value, _ = sys.exc_info()
+        if exc_type == SystemExit:
+            sys.exit(exc_value)
+        else:
+            log.exception(exc_value)
+            log.info("\nERROR: Exception caught.")
+            supportInfo(log)
+    except BaseException:
+        exc_type, exc_value, _ = sys.exc_info()
+        if exc_type == SystemExit:
+            sys.exit(exc_value)
+        else:
+            log.exception(exc_value)
+            log.info("\nERROR: Exception caught.")
+            supportInfo(log)
+
+    log.info("Log was written to " + params.log_filename)
+
+
+if __name__ == '__main__':
+    main(sys.argv)