Merge pull request #5 from redlib-org/improve_spoofing

Improve spoofing

build.rs

@@ -7,6 +7,7 @@ use std::os::unix::process::ExitStatusExt;
 use std::os::windows::process::ExitStatusExt;
 
 fn main() {
+	println!("cargo:rerun-if-changed=src/");
 	let output = String::from_utf8(
 		Command::new("git")
 			.args(["rev-parse", "HEAD"])

scripts/update_oauth_resources.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+# Requirements
+# - curl
+# - rg
+# - jq
+
+# Fetch iOS app versions
+ios_version_list=$(curl -s "https://ipaarchive.com/app/usa/1064216828" | rg "(20\d{2}\.\d+.\d+) / (\d+)" --only-matching -r "Version \$1/Build \$2" | sort | uniq)
+
+# Count the number of lines in the version list
+ios_app_count=$(echo "$ios_version_list" | wc -l)
+
+echo -e "Fetching \e[34m$ios_app_count iOS app versions...\e[0m"
+
+
+# Specify the filename as a variable
+filename="src/oauth_resources.rs"
+
+# Add comment that it is user generated
+echo "// This file was generated by scripts/update_oauth_resources.sh" >> "$filename"
+echo "// Rerun scripts/update_oauth_resources.sh to update this file" >> "$filename"
+echo "// Please do not edit manually" >> "$filename"
+echo "// Filled in with real app versions" >> "$filename"
+
+# Open the array in the source file
+echo "pub static IOS_APP_VERSION_LIST: &[&str; $ios_app_count] = &[" >> "$filename"
+
+
+# Append the version list to the source file
+echo "$ios_version_list" | while IFS= read -r line; do
+  echo "    \"$line\"," >> "$filename"
+  echo -e "Fetched \e[34m$line\e[0m."
+done
+
+# Close the array in the source file
+echo "];" >> "$filename"
+
+# Fetch Android app versions
+page_1=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions/" | rg "<a class=\"ver-item\" href=\"(/reddit/com\.reddit\.frontpage/download/phone-20\d{2}\.\d+\.\d+-apk)\" rel=\"nofollow\">" -r "https://apkcombo.com\$1" | sort | uniq)
+# Append with pages
+page_2=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=2" | rg "<a class=\"ver-item\" href=\"(/reddit/com\.reddit\.frontpage/download/phone-20\d{2}\.\d+\.\d+-apk)\" rel=\"nofollow\">" -r "https://apkcombo.com\$1" | sort | uniq)
+page_3=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=3" | rg "<a class=\"ver-item\" href=\"(/reddit/com\.reddit\.frontpage/download/phone-20\d{2}\.\d+\.\d+-apk)\" rel=\"nofollow\">" -r "https://apkcombo.com\$1" | sort | uniq)
+page_4=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=4" | rg "<a class=\"ver-item\" href=\"(/reddit/com\.reddit\.frontpage/download/phone-20\d{2}\.\d+\.\d+-apk)\" rel=\"nofollow\">" -r "https://apkcombo.com\$1" | sort | uniq)
+page_5=$(curl -s "https://apkcombo.com/reddit/com.reddit.frontpage/old-versions?page=5" | rg "<a class=\"ver-item\" href=\"(/reddit/com\.reddit\.frontpage/download/phone-20\d{2}\.\d+\.\d+-apk)\" rel=\"nofollow\">" -r "https://apkcombo.com\$1" | sort | uniq)
+
+# Concatenate all pages
+versions="${page_1}"
+versions+=$'\n'
+versions+="${page_2}"
+versions+=$'\n'
+versions+="${page_3}"
+versions+=$'\n'
+versions+="${page_4}"
+versions+=$'\n'
+versions+="${page_5}"
+
+# Count the number of lines in the version list
+android_count=$(echo "$versions" | wc -l)
+
+echo -e "Fetching \e[32m$android_count Android app versions...\e[0m"
+
+# Append to the source file
+echo "pub static ANDROID_APP_VERSION_LIST: &[&str; $android_count] = &[" >> "$filename"
+
+# For each in versions, curl the page and extract the build number
+echo "$versions" | while IFS= read -r line; do
+  fetch_page=$(curl -s "$line")
+  build=$(echo "$fetch_page" | rg "<span class=\"vercode\">\((\d+)\)</span>" --only-matching -r "\$1" | head -n1)
+  version=$(echo "$fetch_page" | rg "<span class=\"vername\">Reddit (20\d{2}\.\d+\.\d+)</span>" --only-matching -r "\$1" | head -n1)
+  echo "    \"Version $version/Build $build\"," >> "$filename"
+  echo -e "Fetched \e[32mVersion $version/Build $build\e[0m"
+done
+
+# Close the array in the source file
+echo "];" >> "$filename"
+
+# Retrieve iOS versions
+table=$(curl -s "https://en.wikipedia.org/w/api.php?action=parse&page=IOS_17&prop=wikitext&section=31&format=json" | jq ".parse.wikitext.\"*\"" | rg "(17\.[\d\.]*)\\\n\|(\w*)\\\n\|" --only-matching -r "Version \$1 (Build \$2)")
+
+# Count the number of lines in the version list
+ios_count=$(echo "$table" | wc -l)
+
+echo -e "Fetching \e[34m$ios_count iOS versions...\e[0m"
+
+# Append to the source file
+echo "pub static IOS_OS_VERSION_LIST: &[&str; $ios_count] = &[" >> "$filename"
+
+# For each in versions, curl the page and extract the build number
+echo "$table" | while IFS= read -r line; do
+  echo "    \"$line\"," >> "$filename"
+  echo -e "Fetched $line\e[0m"
+done
+
+# Close the array in the source file
+echo "];" >> "$filename"
+
+echo -e "\e[34mRetrieved $ios_app_count iOS app versions.\e[0m"
+echo -e "\e[32mRetrieved $android_count Android app versions.\e[0m"
+echo -e "\e[34mRetrieved $ios_count iOS versions.\e[0m"
+
+echo -e "\e[34mTotal: $((ios_app_count + android_count + ios_count))\e[0m"
+
+echo -e "\e[32mSuccess!\e[0m"

src/client.rs

@@ -20,12 +20,12 @@ use crate::server::RequestExt;
 
 const REDDIT_URL_BASE: &str = "https://oauth.reddit.com";
 
-pub(crate) static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| {
+pub static CLIENT: Lazy<Client<HttpsConnector<HttpConnector>>> = Lazy::new(|| {
 	let https = hyper_rustls::HttpsConnectorBuilder::new().with_native_roots().https_only().enable_http1().build();
 	client::Client::builder().build(https)
 });
 
-pub(crate) static OAUTH_CLIENT: Lazy<RwLock<Oauth>> = Lazy::new(|| {
+pub static OAUTH_CLIENT: Lazy<RwLock<Oauth>> = Lazy::new(|| {
 	let client = block_on(Oauth::new());
 	tokio::spawn(token_daemon());
 	RwLock::new(client)
@@ -332,7 +332,7 @@ pub async fn json(path: String, quarantine: bool) -> Result<Value, String> {
 	}
 }
 
-#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
 async fn test_localization_popular() {
 	let val = json("/r/popular/hot.json?&raw_json=1&geo_filter=GLOBAL".to_string(), false).await.unwrap();
 	assert_eq!("GLOBAL", val["data"]["geo_filter"].as_str().unwrap());

src/config.rs

@@ -7,11 +7,11 @@ use std::{env::var, fs::read_to_string};
 //
 // This is the local static that is initialized at runtime (technically at
 // first request) and contains the instance settings.
-pub(crate) static CONFIG: Lazy<Config> = Lazy::new(Config::load);
+pub static CONFIG: Lazy<Config> = Lazy::new(Config::load);
 
 // This serves as the frontend for the Pushshift API - on removed comments, this URL will
 // be the base of a link, to display removed content (on another site).
-pub(crate) const DEFAULT_PUSHSHIFT_FRONTEND: &str = "www.unddit.com";
+pub const DEFAULT_PUSHSHIFT_FRONTEND: &str = "www.unddit.com";
 
 /// Stores the configuration parsed from the environment variables and the
 /// config file. `Config::Default()` contains None for each setting.
@@ -104,7 +104,7 @@ impl Config {
 	pub fn load() -> Self {
 		let load_config = |name: &str| {
 			let new_file = read_to_string(name);
-			new_file.ok().and_then(|new_file| toml::from_str::<Config>(&new_file).ok())
+			new_file.ok().and_then(|new_file| toml::from_str::<Self>(&new_file).ok())
 		};
 
 		let config = load_config("redlib.toml").or(load_config("libreddit.toml")).unwrap_or_default();
@@ -168,7 +168,7 @@ fn get_setting_from_config(name: &str, config: &Config) -> Option<String> {
 }
 
 /// Retrieves setting from environment variable or config file.
-pub(crate) fn get_setting(name: &str) -> Option<String> {
+pub fn get_setting(name: &str) -> Option<String> {
 	get_setting_from_config(name, &CONFIG)
 }
 

src/instance_info.rs

@@ -15,7 +15,7 @@ use time::OffsetDateTime;
 // This is the local static that is intialized at runtime (technically at
 // the first request to the info endpoint) and contains the data
 // retrieved from the info endpoint.
-pub(crate) static INSTANCE_INFO: Lazy<InstanceInfo> = Lazy::new(InstanceInfo::new);
+pub static INSTANCE_INFO: Lazy<InstanceInfo> = Lazy::new(InstanceInfo::new);
 
 /// Handles instance info endpoint
 pub async fn instance_info(req: Request<Body>) -> Result<Response<Body>, String> {
@@ -84,7 +84,7 @@ fn info_html(req: Request<Body>) -> Result<Response<Body>, Error> {
 	Response::builder().status(200).header("content-type", "text/html; charset=utf8").body(Body::from(message))
 }
 #[derive(Serialize, Deserialize, Default)]
-pub(crate) struct InstanceInfo {
+pub struct InstanceInfo {
 	package_name: String,
 	crate_version: String,
 	git_commit: String,

src/main.rs

@@ -7,6 +7,7 @@ mod config;
 mod duplicates;
 mod instance_info;
 mod oauth;
+mod oauth_resources;
 mod post;
 mod search;
 mod settings;