Add ability to limit files considered based on min/max file size

CHANGES

@@ -6,6 +6,13 @@ who contributed the patch or idea appears first, followed by
 those who've otherwise worked on that item. For a list of 
 contributors names and identifiers please see the CONTRIBUTORS file.
 
+Changes from 1.51 to xxx
+- Added -b (min) and -B (max) arguments to allow to filter files 
+  based on the given values
+- Added -e argument to allow for skipping the byte to byte verification
+  when the crc matches
+- Added -x argument to not cross into another device from the given 
+  arguments starting device
 
 Changes from 1.50 to 1.51
 

CONTRIBUTORS

@@ -15,4 +15,4 @@ on their contributions. Names are listed in alphabetical order.
  [LR] Lukas Ruf 		([email protected])
  [PB] Peter Bray 		(Sydney, Australia)
 [SSD] Steven S. Dick 		([email protected])
-
+ [JC] Jason Clara		(Toronto, Canada)

Makefile.inc/VERSION

@@ -2,4 +2,4 @@
 # VERSION determines the program's version number.
 #
 
-VERSION = 1.51
+VERSION = 1.52-dev

README

@@ -20,6 +20,12 @@ Usage: fdupes [options] DIRECTORY...
  -f --omitfirst   	omit the first file in each set of matches
  -1 --sameline    	list each set of matches on a single line
  -S --size        	show size of duplicate files
+ -b --minfilesize	Consider only files larger than N KB
+ -B --maxfilesize	Consider only files smaller than N KB
+ -e --skipverify	Skip final byte to byte verification after 
+                        checksum match
+ -x --xdevice           Do not cross into another device from the given 
+                        arguments starting device
  -q --quiet       	hide progress indicator
  -d --delete      	prompt user for files to preserve and delete all
                   	others; important: under particular circumstances,

fdupes.1

@@ -48,6 +48,18 @@ list each set of matches on a single line
 .B -S --size
 show size of duplicate files
 .TP
+.B -b --minfilesize
+Consider only files larger than N KB
+.TP
+.B -B --maxfilesize
+Consider only files smaller than N KB
+.TP
+.B -e --skipverify
+Skip final byte to byte verification after a checksum match
+.TP
+.B -x --xdevice
+Do not cross into another device from the given arguments starting device
+.TP
 .B -m --summarize
 summarize duplicate files information
 .TP

fdupes.c

@@ -54,6 +54,10 @@
 #define F_SUMMARIZEMATCHES  0x0800
 #define F_EXCLUDEHIDDEN     0x1000
 #define F_PERMISSIONS       0x2000
+#define F_MINFILESIZE       0x4000
+#define F_MAXFILESIZE       0x8000
+#define F_SKIPBYTEVERIFY    0x10000
+#define F_XDEVICE           0x20000
 
 typedef enum {
   ORDER_TIME = 0,
@@ -64,6 +68,10 @@ char *program_name;
 
 unsigned long flags = 0;
 
+unsigned long long int min_file_size = 0;
+unsigned long long int max_file_size = 0;
+dev_t workingdevice;
+
 #define CHUNK_SIZE 8192
 
 #define INPUT_SIZE 256
@@ -237,6 +245,30 @@ int nonoptafter(char *option, int argc, char **oldargv,
   return x;
 }
 
+int skipfile(file_t *file, struct stat info)
+{
+  if(ISFLAG(flags, F_MINFILESIZE) && !S_ISDIR(info.st_mode) && file->size < min_file_size)
+  {
+    //printf("Small: %s %ld - %ld\n", file->d_name, min_file_size, filesize(file->d_name));
+    return 1;
+  }
+
+  if(ISFLAG(flags, F_MAXFILESIZE) && !S_ISDIR(info.st_mode) && file->size > max_file_size)
+  {
+    return 2;
+  }
+
+  return 0;
+}
+
+void getfilestats(file_t *file)
+{
+  file->size = filesize(file->d_name);
+  file->inode = getinode(file->d_name);
+  file->device = getdevice(file->d_name);
+  file->mtime = getmtime(file->d_name);
+}
+
 int grokdir(char *dir, file_t **filelistp)
 {
   DIR *cd;
@@ -249,7 +281,11 @@ int grokdir(char *dir, file_t **filelistp)
   static int progress = 0;
   static char indicator[] = "-\\|/";
   char *fullname, *name;
-
+
+  if (ISFLAG(flags, F_XDEVICE) && getdevice(dir) != workingdevice) {
+  	printf("Skipping out of device item: %s\n", dir);
+	return 0;
+  }
   cd = opendir(dir);
 
   if (!cd) {
@@ -293,6 +329,8 @@ int grokdir(char *dir, file_t **filelistp)
       if (lastchar >= 0 && dir[lastchar] != '/')
 	strcat(newfile->d_name, "/");
       strcat(newfile->d_name, dirinfo->d_name);
+
+      getfilestats(newfile);
 
       if (ISFLAG(flags, F_EXCLUDEHIDDEN)) {
 	fullname = strdup(newfile->d_name);
@@ -305,7 +343,7 @@ int grokdir(char *dir, file_t **filelistp)
 	free(fullname);
       }
 
-      if (filesize(newfile->d_name) == 0 && ISFLAG(flags, F_EXCLUDEEMPTY)) {
+      if (newfile->size == 0 && ISFLAG(flags, F_EXCLUDEEMPTY)) {
 	free(newfile->d_name);
 	free(newfile);
 	continue;
@@ -322,6 +360,13 @@ int grokdir(char *dir, file_t **filelistp)
 	free(newfile);
 	continue;
       }
+
+      if(skipfile(newfile, info))
+      {
+        free(newfile->d_name);
+        free(newfile);
+        continue;
+      }
 
       if (S_ISDIR(info.st_mode)) {
 	if (ISFLAG(flags, F_RECURSE) && (ISFLAG(flags, F_FOLLOWLINKS) || !S_ISLNK(linfo.st_mode)))
@@ -462,17 +507,10 @@ void purgetree(filetree_t *checktree)
   free(checktree);
 }
 
-void getfilestats(file_t *file)
-{
-  file->size = filesize(file->d_name);
-  file->inode = getinode(file->d_name);
-  file->device = getdevice(file->d_name);
-  file->mtime = getmtime(file->d_name);
-}
 
 int registerfile(filetree_t **branch, file_t *file)
 {
-  getfilestats(file);
+  //getfilestats(file);
 
   *branch = (filetree_t*) malloc(sizeof(filetree_t));
   if (*branch == NULL) {
@@ -512,11 +550,11 @@ file_t **checkmatch(filetree_t **root, filetree_t *checktree, file_t *file)
      duplicates unless the user specifies otherwise.
   */    
 
-  if (!ISFLAG(flags, F_CONSIDERHARDLINKS) && (getinode(file->d_name) == 
-      checktree->file->inode) && (getdevice(file->d_name) ==
+  if (!ISFLAG(flags, F_CONSIDERHARDLINKS) && (file->inode == 
+      checktree->file->inode) && (file->device ==
       checktree->file->device)) return NULL; 
 
-  fsize = filesize(file->d_name);
+  fsize = file->size;
 
   if (fsize < checktree->file->size) 
     cmpresult = -1;
@@ -610,7 +648,7 @@ file_t **checkmatch(filetree_t **root, filetree_t *checktree, file_t *file)
     }
   } else 
   {
-    getfilestats(file);
+    //getfilestats(file);
     return &checktree->file;
   }
 }
@@ -995,6 +1033,10 @@ void help_text()
   printf(" -f --omitfirst   \tomit the first file in each set of matches\n");
   printf(" -1 --sameline    \tlist each set of matches on a single line\n");
   printf(" -S --size        \tshow size of duplicate files\n");
+  printf(" -b --minfilesize \tConsider only files larger than N KB\n");
+  printf(" -B --maxfilesize \tConsider only files smaller than N KB\n");
+  printf(" -e --skipverify  \tSkip final byte to byte verification after checksum match\n");
+  printf(" -x --xdevice     \tDo not cross into another device from the given arguments starting device\n");
   printf(" -m --summarize   \tsummarize dupe information\n");
   printf(" -q --quiet       \thide progress indicator\n");
   printf(" -d --delete      \tprompt user for files to preserve and delete all\n"); 
@@ -1018,6 +1060,39 @@ void help_text()
 #endif
 }
 
+unsigned long long int parsesizeinput(char* input){
+	unsigned long long int inputsize;
+	char * endptr = NULL;
+	inputsize = strtoull(input, &endptr,10);
+	switch(*endptr){
+		case '\0':
+			inputsize = inputsize * 1024;
+			break;
+		case 'k':
+		case 'K':
+			inputsize = inputsize * 1024;
+			endptr++;
+			break;
+		case 'm':
+		case 'M':
+			inputsize = inputsize * 1024*1024;
+			endptr++;
+			break;
+		case 'g':
+		case 'G':
+			inputsize = inputsize *1024*1024*1024;
+			endptr++;
+			break;
+		default:
+			break;
+	}
+	if (*endptr != '\0'){
+            fprintf(stderr,"fdupes: provide numeric argument >0 for file size to consider\n");
+		exit(1);
+	}
+	return inputsize;
+}
+
 int main(int argc, char **argv) {
   int x;
   int opt;
@@ -1032,7 +1107,8 @@ int main(int argc, char **argv) {
   char **oldargv;
   int firstrecurse;
   ordertype_t ordertype = ORDER_TIME;
-
+
+
 #ifndef OMIT_GETOPT_LONG
   static struct option long_options[] = 
   {
@@ -1057,6 +1133,10 @@ int main(int argc, char **argv) {
     { "summary", 0, 0, 'm' },
     { "permissions", 0, 0, 'p' },
     { "order", 1, 0, 'o' },
+    { "minfilesize", 1, 0, 'b' },
+    { "maxfilesize", 1, 0, 'B' },
+    { "skipverify", 0, 0, 'e' },
+    { "xdevice", 0, 0, 'x' },
     { 0, 0, 0, 0 }
   };
 #define GETOPT getopt_long
@@ -1068,7 +1148,7 @@ int main(int argc, char **argv) {
 
   oldargv = cloneargs(argc, argv);
 
-  while ((opt = GETOPT(argc, argv, "frRq1SsHlndvhNmpo:"
+  while ((opt = GETOPT(argc, argv, "frRq1SsHlndvhNmpexo:b:B:"
 #ifndef OMIT_GETOPT_LONG
           , long_options, NULL
 #endif
@@ -1118,6 +1198,34 @@ int main(int argc, char **argv) {
       break;
     case 'm':
       SETFLAG(flags, F_SUMMARIZEMATCHES);
+      break;
+    case 'e':
+      SETFLAG(flags, F_SKIPBYTEVERIFY);
+      break;
+    case 'x':
+      SETFLAG(flags, F_XDEVICE);
+      break;
+    case 'b':
+      SETFLAG(flags, F_MINFILESIZE);
+      if (strlen(optarg) == 0) {
+            fprintf(stderr,"fdupes -b: provide numeric argument >0 for minimum file size to consider\n");
+            exit(1);
+      }
+      min_file_size = parsesizeinput(optarg);
+      break;
+    case 'B':
+      SETFLAG(flags, F_MAXFILESIZE);
+      if (strlen(optarg) == 0) {
+            fprintf(stderr,"fdupes -B: provide numeric argument >0 for maximum file size to consider\n");
+            exit(1);
+      }
+      max_file_size = parsesizeinput(optarg);
+
+      if (ISFLAG(flags, F_MAXFILESIZE) && ISFLAG(flags, F_MINFILESIZE) && min_file_size > max_file_size){
+             fprintf(stderr, "fdupes -B: min file size (-b) must be smaller then max file size(-B)\n");
+             exit(1);
+      }
+
       break;
     case 'p':
       SETFLAG(flags, F_PERMISSIONS);
@@ -1167,18 +1275,26 @@ int main(int argc, char **argv) {
 
     /* F_RECURSE is not set for directories before --recurse: */
     for (x = optind; x < firstrecurse; x++)
-      filecount += grokdir(argv[x], &files);
-
+	{
+		workingdevice = getdevice(argv[x]);
+		filecount += grokdir(argv[x], &files);
+	}
     /* Set F_RECURSE for directories after --recurse: */
     SETFLAG(flags, F_RECURSE);
 
     for (x = firstrecurse; x < argc; x++)
-      filecount += grokdir(argv[x], &files);
+	{
+		workingdevice = getdevice(argv[x]);
+	      filecount += grokdir(argv[x], &files);
+	}
   } else {
-    for (x = optind; x < argc; x++)
-      filecount += grokdir(argv[x], &files);
+    for (x = optind; x < argc; x++){
+      		workingdevice = getdevice(argv[x]);
+		filecount += grokdir(argv[x], &files);
+	}
   }
 
+
   if (!files) {
     if (!ISFLAG(flags, F_HIDEPROGRESS)) fprintf(stderr, "\r%40s\r", " ");
     exit(0);
@@ -1206,7 +1322,7 @@ int main(int argc, char **argv) {
 	continue;
       }
 
-      if (confirmmatch(file1, file2)) {
+      if (ISFLAG(flags, F_SKIPBYTEVERIFY) || confirmmatch(file1, file2)) {
         registerpair(match, curfile,
             (ordertype == ORDER_TIME) ? sort_pairs_by_mtime : sort_pairs_by_filename );