From 5443bcd4060653980ef19e2ee7c39460850e5153 Mon Sep 17 00:00:00 2001 From: moriar1 Date: Sun, 8 Sep 2024 14:31:57 +0500 Subject: [PATCH] `add` subcommand with `-l -d` options --- README.md | 33 ++++++------- download.py | 16 +++++-- src/actions.rs | 113 +++++++++++++++++++++++---------------------- src/cli.rs | 14 +++--- src/dir_actions.rs | 32 ++++++------- src/main.rs | 77 +++++++++++------------------- 6 files changed, 133 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index e5bf5c3..5bd1ecb 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,9 @@ To download project use ```bash git clone https://github.com/moriar1/clvog cd clvog -mkdir vid # required create directory `vid` ``` -### Explaining `new_vid_list.txt` content +### `new_vid_list.txt` content It contains required names and link to video: `--.mp4 ()` (See its content using your text editor for exact names. Types are: AV, AMV, MAD. See [reference](#list-structure) for more.) @@ -52,12 +51,7 @@ In this example you see new files: - `0002-AAA.mp4` - `0003-AMV-20240225-20240303.mp4` -`0002-AAA.mp4` created because it must be downloaded manually. - -Move new videos in `vid/` directory (`clvog` will `check` video file names in it and in `vid_list.txt` next time): -```bash -mv 0* vid/ -``` +empty `0002-AAA.mp4` created because it must be downloaded manually. Before using `add` again clear `new_vid_list.txt` content and write your new videos in it. @@ -68,9 +62,9 @@ See [TODO](#todo). - `-u, --skip-check` - `-v, --verbose` - `-V, --version` -- `insert ` +- `insert ` - `add` -- `add -p ` +- `add -l , -d ` - `move ` - `pull # rename video files from list to dir (-u option disabled)` - `sync # rename records from dir to list (-u option disabled)` @@ -79,15 +73,15 @@ See [TODO](#todo). - `hide [,,..] # files only` - `check # verfiy directory and list names matching (runs before any action)` -Example: `clvog add -vup push_list.txt` +Example: `clvog add -vul push_list.txt` – add new videos from `push_list.txt` to `vid_list.txt` with output debug infromation and without `check` names mathcing ### Main actions: -#### `add` and `add -p ` +#### `add -l , -d ` -Download and insert videos from `new_vid_list.txt` or from `` text file at the end of `vid_list.txt` +Download and insert videos from `new_vid_list.txt` or from `` text file at the end of `vid_list.txt` ##### Example -`new_video_list.txt`: +`new_list_path.txt`: > AMV-31122020-31122020.mp4 https://some_link.com > AMV-31122020-31122020.mp4 https://another_link.com > AMV-01012000-31012002.mp4 https://*some_broken_link.com @@ -101,7 +95,7 @@ Download and insert videos from `new_vid_list.txt` or from `` text file at > `` It also downloads these videos and rename files according the list -`/path/to/video_directory/` (`./` by default): +`/path/to/video_directory/` (`./vid/` by default): > ... > ./23-AMV-31122020-31122020.mp4 > etc. @@ -117,7 +111,7 @@ Clvog will delete file under `` ##### Example -`vid_list.txt` and `path/to/video_dir` (`./vid` by default): +`vid_list.txt` and `path/to/video_dir` (`./vid/` by default): > 34-AV-11-11.mp4 https://0 > 35-AMV-22-22.mp4 https://1 > 36-AMV-22-22.mp4 https://2 @@ -208,7 +202,7 @@ Comments (for example, in `comments.txt` file or in description to last video) # TODO: - [ ] implement all commands - [x] check - - [ ] add (with `-p` option) + - [x] add - [ ] insert - [ ] sync - [ ] pull @@ -218,8 +212,9 @@ Comments (for example, in `comments.txt` file or in description to last video) - [ ] rename - [ ] add caching to `get_entries()` and `get_records()` in `actions.rs` (see [example](./misc/cache.rs)) - [ ] add `version` command with the same output as `--version` or `-V` option -- [ ] download videos in `./vid/` directory - +- [ ] download videos in `./vid/` directory +- [ ] add parallel download requests + ## Future of Anime Music Video Organizer Clvog is the first step to creating fully-featured Anime Music Video Organizer with TUI (looks like [ranger](https://github.com/ranger/ranger) and shows more video previews) diff --git a/download.py b/download.py index 4b3c3ea..0e3e27e 100644 --- a/download.py +++ b/download.py @@ -33,8 +33,16 @@ def debug(msg): print(msg) -with open(sys.argv[1], 'r') as file: - log = open('failed_downloads.log', 'w') +if len(sys.argv) > 1: + list_path = sys.argv[1] +else: + list_path = "vid_list.txt" + +run_from = os.getcwd() +with open(list_path, 'r') as file: + if len(sys.argv) == 3: + os.chdir(sys.argv[2]) + log = open(f'{run_from}/failed_downloads.log', 'w') for line in file: # extract file name and link match = re.match(r'(\S+)\s+(https?://\S+)', line.strip()) @@ -67,8 +75,8 @@ def debug(msg): dummy = video_name[0:5] + "AAA.mp4" open(dummy, 'a').close() log.close() -if (os.stat('failed_downloads.log').st_size == 0): - os.remove('failed_downloads.log') +if (os.stat(f'{run_from}/failed_downloads.log').st_size == 0): + os.remove(f'{run_from}/failed_downloads.log') # Using external dowloader: yt_dlp --downloader aria2c "https://" # coub loop issue: github.com/yt-dlp/yt-dlp/issues/1930 diff --git a/src/actions.rs b/src/actions.rs index a272b66..9857d02 100644 --- a/src/actions.rs +++ b/src/actions.rs @@ -1,28 +1,29 @@ use std::{ fs::{self, OpenOptions}, io::{self, Write}, - path::PathBuf, + path::Path, process::{Command, Stdio}, vec, }; -pub struct Config { +pub struct Config<'a> { pub is_verbose: bool, - pub dir_path: PathBuf, - pub list_path: PathBuf, + pub dir_path: &'a Path, + pub list_path: &'a Path, } -/// Returns vec filenames that start with four digits and length at least 11 symbols: "0001-AA.aaa -fn get_entries(dir_path: &PathBuf) -> Vec { +/// Get vec filenames that start with four digits and length at least 11 symbols: `0001-AA.mp4` +fn get_entries(dir_path: &Path) -> Vec { // TODO: add caching, if get_entries has been already called it shouldn't create vec again (See // clvog/misc/cache.rs as example) + fs::create_dir_all(dir_path).unwrap(); let entries = fs::read_dir(dir_path) .expect("Error: cannot open dir") .map(|e| { e.map(|e| { e.file_name() .into_string() - .map_err(|err| println!("Error: filename: {:#?}", err)) + .map_err(|err| println!("Error: filename: {err:#?}")) .unwrap() }) }) @@ -38,64 +39,70 @@ fn get_entries(dir_path: &PathBuf) -> Vec { entries } -fn get_records(list_path: &PathBuf) -> Vec { +/// Get lines from `list_path` file +fn get_records(list_path: &Path) -> Vec { let Some(records) = fs::read_to_string(list_path).ok() else { return vec![]; }; records.lines().map(String::from).collect::>() } -// Check if all video files and records in list have the same names, if not panic! +/// Check if all video files and records in list have the same names, else **Panics** pub fn check(config: &Config) { - let (dir_entries, list_records) = ( - get_entries(&config.dir_path), - get_records(&config.list_path), - ); + let (dir_entries, list_records) = (get_entries(config.dir_path), get_records(config.list_path)); // NOTE loop check only existing records and dir entries, // so if you have extra records in vid_list.txt, but these videos are not exist in ./vid/ // directory then verifing will still passes. // TODO: Uncomment if needs: - // if list_records.len() != dir_entries.len() { - // panic!("") - // } + assert_eq!( + list_records.len(), + dir_entries.len(), + "list len: {}, dir len: {}", + list_records.len(), + dir_entries.len() + ); let mut i = 0; // TODO: use enumerate() from 1 for (entry, rec) in dir_entries.iter().zip(list_records.iter()) { i += 1; let r = rec.split_whitespace().next().unwrap(); if config.is_verbose { - println!("Comparing. Entry:`{}` record: `{}`", entry, r); - } - if (*entry)[0..4].parse::().unwrap() != i { - panic!("Broken order: `{}`\n record name: `{}`", entry, r); - } - - if entry != r && (*entry)[5..8] != *"AAA" { - panic!("entry name: `{}`\n record name: `{}`", entry, r); + println!("Comparing. Entry:`{entry}` record: `{r}`"); } + #[rustfmt::skip] + assert_eq!( + (*entry)[0..4].parse::().unwrap(), i, + "Broken order: `{entry}`\n record name: `{r}`", + ); + assert!( + !(entry != r && (*entry)[5..8] != *"AAA"), + "entry name: `{entry}`\n record name: `{r}`", + ); } } -/// 1. Get list_records and dir_entries(videos) -/// 2. Get last record number -/// 3. Write new records from file with whole information (name, link, description) to -/// new_list_records vector and append file with it. -/// 4. Or if file do not exists and dir_entries is empty - create it, numbering starts +/// Add and download new videos +/// 1. Get `list_records` and `dir_entries` (videos). +/// 2. Get last record number +/// 3. Write new records from <`new_list_path`> file with whole information (name, link, description) to +/// `new_list_records` vector and append`list_path`> file with it. +/// 4. Or if <`list_path`> file do not exists and `dir_entries` is empty - create it, numbering starts /// with 0001 -/// 5. Rewrite file with numbers +/// 5. Rewrite <`new_list_path`> file with numbers /// 6. Dowload videos using download.py (or create `AAA` dummy): -/// - it gets file +/// - it gets `new_list_path` and `vid_path` /// - uses yt-dlp to download videos, renames them -/// - all error are in stderr -/// - all failed videos written in "failed_downloads.log" for further manual intervention, -/// - empty files with appropriate names are created -pub fn add(config: Config, new_list_path: PathBuf) { - let list_records = get_entries(&config.dir_path); +/// - writes all failed videos in "`failed_downloads.log`" for further manual intervention +/// - creates empty files with appropriate names +/// +/// P.S. `list_path` = "`./vid_list.txt`", `new_list_path` = "`./new_vid_list.txt`", `vid_path` = "`./vid/`" by default +pub fn add(config: &Config, new_list_path: &Path, vid_path: &Path) { + let list_records = get_entries(config.dir_path); // Сreate String of new records with numbers, splited by '\n' let size = list_records.len() + 1; - let new_records = fs::read_to_string(new_list_path.to_owned()) + let new_records = fs::read_to_string(new_list_path) .unwrap_or_else(|_| panic!("Error: cannot read new list: `{:?}`", new_list_path)) .lines() .enumerate() @@ -107,44 +114,40 @@ pub fn add(config: Config, new_list_path: PathBuf) { OpenOptions::new() .write(true) .create(true) - .open(&config.list_path) + .open(config.list_path) .unwrap() } else { OpenOptions::new() .append(true) - .open(&config.list_path) + .open(config.list_path) .unwrap() }; // Create backups of lists before writing // TODO: add option to skip backup - for &list in [&new_list_path, &config.list_path].iter() { - fs::copy(&list, list.to_str().unwrap().to_string() + ".bak") - .unwrap_or_else(|e| panic!("Error: cannot create backup of lists: {:?}", e)); + for &list in &[&new_list_path, &config.list_path] { + fs::copy(list, list.to_str().unwrap().to_string() + ".bak") + .unwrap_or_else(|e| panic!("Error: cannot create backup of lists: {e:?}")); } // Rewriting list with new records with numbers to then use downloader.py - fs::write(&new_list_path, &new_records) - .unwrap_or_else(|_| panic!("Error: cannor rewrite new list file: `{:?}`", new_list_path)); + fs::write(new_list_path, &new_records) + .unwrap_or_else(|_| panic!("Error: cannor rewrite new list file: `{new_list_path:?}`")); // Appending main video list list_file .write_all(new_records.as_bytes()) - .unwrap_or_else(|e| panic!("Error: cannot write in video list: {}", e)); + .unwrap_or_else(|e| panic!("Error: cannot write in video list: {e}")); // run downloader Command::new("python3") - .arg("download.py") - .arg(new_list_path) + .args([ + "download.py", + new_list_path.to_str().unwrap(), + vid_path.to_str().unwrap(), + ]) .stdout(Stdio::inherit()) .stderr(Stdio::inherit()) .output() - // .spawn() - .expect("execute"); - // py.id - // let py_stdout = String::from_utf8(py.stdout).unwrap(); - // let py_stderr = String::from_utf8(py.stderr).unwrap(); - // println!("{py_stdout}"); - // eprintln!("{py_stderr}"); - // + .unwrap(); } diff --git a/src/cli.rs b/src/cli.rs index 5a7fac8..0aabb25 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,5 +1,6 @@ use clap::{arg, crate_version, Command}; +#[must_use] pub fn cli() -> Command { Command::new("clvog") .about("Command line video organizer") @@ -15,12 +16,11 @@ pub fn cli() -> Command { .subcommand( Command::new("add") .about("Add new videos at the end of `vid_list.txt` from `new_vid_list.txt`") - .arg( - arg!(-p --"path" "path to video list to add, \ - by default: `./new_vid_list.txt`") - .required(false), - ) + .arg(arg!(-d --"video-directory-path" "path to the video directory, by default: `./vid/`") .required(false)) + .arg(arg!(-l --"list-path" "path to video list to add, by default: `./new_vid_list.txt`").required(false)) .arg(arg!(-u --"skip-check" "Skip names matching verifing").required(false)) - .arg(arg!(-v --"verbose" "Show debug information").required(false)), - ) + .arg(arg!(-v --"verbose" "Show debug information").required(false))) + .subcommand( + Command::new("version") + .arg(arg!(-v --"verbose" "Show debug information").required(false))) } //subc: help, version diff --git a/src/dir_actions.rs b/src/dir_actions.rs index 2383f10..9fca00b 100644 --- a/src/dir_actions.rs +++ b/src/dir_actions.rs @@ -1,17 +1,17 @@ -#[allow(dead_code)] -pub struct DirOrganizer { - dir_path: &'static str, - dir_entries: Vec, - is_verbose: bool, -} // impl: build, hide +// #[allow(dead_code)] +// pub struct DirOrganizer { +// dir_path: &'static str, +// dir_entries: Vec, +// is_verbose: bool, +// } // impl: build, hide -impl DirOrganizer { - pub fn build() -> DirOrganizer { - println!("DirOrganizer::build() call"); - todo!(); - // use get_entries() from dir_actions - } - pub fn hide(&self) { - println!("hide() call") - } -} +// impl DirOrganizer { +// pub fn build() -> DirOrganizer { +// println!("DirOrganizer::build() call"); +// todo!(); +// // use get_entries() from dir_actions +// } +// pub fn hide(&self) { +// println!("hide() call"); +// } +// } diff --git a/src/main.rs b/src/main.rs index 6ee6418..128d51c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,7 @@ -use std::path::PathBuf; +use std::path::Path; -use clvog::*; +use clap::crate_version; +use clvog::{actions, Config}; fn main() { let matches = clvog::cli().get_matches(); @@ -10,65 +11,39 @@ fn main() { // takes sub_match optional argument let is_verbose = matches.1.get_flag("verbose"); - let (dir_path, list_path) = (PathBuf::from("./vid"), PathBuf::from("./vid_list.txt")); + let (dir_path, list_path) = (Path::new("./vid"), Path::new("./vid_list.txt")); let config = Config { - dir_path, // PathBuf - list_path, // PathBuf is_verbose, + dir_path, + list_path, }; - // verify dir-list names before proceeding - if matches.0 != "check" && !matches.1.get_flag("skip-check") { - check(&config); - } - match matches { ("check", _) => { - check(&config); + actions::check(&config); println!("Verifing passed."); } - ("add", _sub_match) => { - let new_list_path = PathBuf::from("./new_vid_list.txt"); - actions::add(config, new_list_path.to_owned()); + ("add", sub_match) => { + if !sub_match.get_flag("skip-check") { + actions::check(&config); + }; + + let default_path = "./new_vid_list.txt".to_string(); + let new_list_path = sub_match + .get_one::("list-path") + .unwrap_or(&default_path); + let new_list_path = Path::new(new_list_path); + + let default_path = "./vid/".to_string(); + let vid_path = sub_match + .get_one::("video-directory-path") + .unwrap_or(&default_path); + let vid_path = Path::new(vid_path); + + actions::add(&config, new_list_path, vid_path); } + ("version", _) => println!("clvog {}", crate_version!()), // ("hide", sub_match) => dir_actions::DirOrganizer::build().hide(), _ => todo!(), } - - /*if let Some(("hide", sub_match)) = matches.subcommand() { - //Dir only action trait - hide(is_verbose); - return; - } - organizer: Organizer; - if !is_skip_check { - check(); - } - match matches.subcommand() { - // Organizer - Some(("check")) - Some(("add", sub_match)) => add(is_verbose), - Some(("rm", sub_match)) => remove(), // check, del from vec then rewrite - Some(("insert", sub_match)) => insert(), // check, insert into vec then rewrite - Some(("sync", sub_match)) => sync(), // check, rewrite vec then rewrite - }*/ } - -/* // Or create Organizer then use add() - let is_skip_check = sub_match.get_flag("skip"); - let is_verbose = sub_match.get_flag("verbose"); - if !is_skip_check { - check(dir_entries, records); // panic - } - list_records = get_records(list_path); // if exists - dir_entries = get_entries(dir_path) - - add( - new_list_path, - list_path, - dir_path, - dir_entries, - list_records, - is_verbose - ); -*/