brozek/website-toots
1
0
Fork 0
mirror of https://github.com/Brandon-Rozek/website-toots.git synced 2024-09-16 13:02:29 -04:00
Code Issues Projects Releases Packages Wiki Activity

Added duplication check and sleep

Browse source
This commit is contained in:
Brandon Rozek 2024-07-28 19:28:40 -07:00
parent bc01b85f87
commit 5add4461c2
No known key found for this signature in database
GPG key ID: 26E457DA82C9F480
1 changed files with 31 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
.scripts/src/retrieve_toots.rs
View file

@ -2,7 +2,9 @@ use regex::Regex;
use reqwest::header::HeaderValue;
use std::collections::HashMap;
use std::fs::File;
use std::io::Write;
use std::io::{BufReader, Write};
use std::thread::sleep;
use std::time::Duration;
static RETRIEVE_NUM_TOOTS: usize = 1000;
static SERVER: &str = "fosstodon.org";
@ -144,9 +146,32 @@ async fn main() {
.expect("Failed to create directory");
}
let path = format!("{}/{}.json", data_dir, toot_id);
write_json_to_file(&item, path.as_ref())
let path: String = format!("{}/{}.json", data_dir, toot_id);
let mut update_json = false;
let f = File::open(path.clone());
if f.is_ok() {
let reader = BufReader::new(f.unwrap());
let f_json_result: Result<serde_json::Value, _> = serde_json::from_reader(reader);
if f_json_result.is_ok() {
// If the JSON from the network is different, update
if f_json_result.unwrap() != *item {
update_json = true;
}
}
} else {
// File doesn't exist (probably) so
// try to write the new JSON
update_json = true;
}
if update_json {
write_json_to_file(&item, path.as_ref())
.expect("Failed to write to file");
}
}
println!("Retrieved {} toots from server", json_array.len());
@ -164,5 +189,8 @@ async fn main() {
Ok(next_url) => url = next_url,
Err(_) => more_toots_exist = false,
}
// Sleep for the 1 second to not overload the server
sleep(Duration::new(1, 0));
}
}