vinod/booksman
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.12-searchP3

Browse Source
This commit is contained in:
Vinod J M
2022-11-27 17:48:14 +05:30
parent f297d8e4e0
commit db0d976192
6 changed files with 393 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/Cargo.lock generated
View File

@@ -441,6 +441,7 @@ dependencies = [
"itertools",
"log",
"meilisearch-sdk",
"migration",
"reqwest",
"sea-orm",
"serde",

1
backend/api/Cargo.toml
View File

@@ -7,6 +7,7 @@ edition = "2021"
[dependencies]
booksman-orm = { path = "../orm" }
booksman-search = { path = "../search" }
migration = { path = "../migration" }
entity = { path = "../entity" }
axum = "^0.5"
axum-extra = { version = "^0.3", features = ["spa"] }

352
backend/api/src/lib.rs
View File

@@ -29,6 +29,7 @@ use meilisearch_sdk::client::Client;
//use itertools::Itertools;
use ::entity::entities::{book,book_author,book_person,book_place,book_subject,book_time,book_isbn};
use std::env;
use migration::{Migrator, MigratorTrait};
#[derive(Deserialize, Serialize, Debug, Clone)]
struct BookUI {
@@ -60,7 +61,7 @@ struct Docs {
edition_count: Option<i32>,
first_publish_year: Option<i32>,
number_of_pages_median: Option<i32>,
goodread_id: Option<String>,
id_goodreads: Option<Vec<String>>,
description: Option<String>,
isbn: Option<Vec<String>>,
cover_i: Option<i32>,
@@ -211,6 +212,10 @@ pub async fn main() {
dotenvy::dotenv().ok();
let backend_url = env::var("BACKEND_URL").expect("BACKEND_URL is not set in .env file");
let images_dir = env::var("IMAGES_DIR").expect("IMAGES_DIR is not set in .env file");
let cors_url = env::var("CORS_URL").expect("CORS_URL is not set in .env file");
let db_url = env::var("DATABASE_URL").expect("DATABASE_URL is not set in .env file");
let meili_url = env::var("MEILI_URL").expect("MEILI_URL is not set in .env file");
let meili_key = env::var("MEILI_KEY").expect("MEILI_KEY is not set in .env file");
@@ -219,8 +224,10 @@ pub async fn main() {
let conn = Database::connect(db_url)
.await
.expect("Database connection failed");
//Migrator::up(&conn, None).await.unwrap();
let meili_client = Client::new(meili_url, meili_key);
// Apply all pending migrations
Migrator::up(&conn, None).await.unwrap();
let meili_client = Client::new(meili_url, meili_key);
let app = Router::new()
.route("/api/search_openlibrary", get(search_openlibrary))
@@ -230,7 +237,7 @@ pub async fn main() {
.route("/api/list_search", get(list_search_book))
.route("/api/create", post(create_book))
.route("/api/update", post(update_book))
.nest("/images", get_service(ServeDir::new("../images")).handle_error(handle_error))
.nest("/images", get_service(ServeDir::new(images_dir)).handle_error(handle_error))
.merge(SpaRouter::new("/assets", opt.static_dir))
.layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()))
.layer(Extension(conn))
@@ -244,7 +251,7 @@ pub async fn main() {
// or see this issue https://github.com/tokio-rs/axum/issues/849
CorsLayer::new()
.allow_methods([Method::GET, Method::POST])
.allow_origin("http://localhost:8080".parse::<HeaderValue>().unwrap())
.allow_origin(cors_url.parse::<HeaderValue>().unwrap())
.allow_headers(Any),
);
@@ -267,6 +274,10 @@ async fn create_by_isbn(
Extension(ref meili_client): Extension<Client>,
axum::extract::Query(params): axum::extract::Query<HashMap<String, String>>,
) -> impl IntoResponse {
dotenvy::dotenv().ok();
let images_dir = env::var("IMAGES_DIR").expect("IMAGES_DIR is not set in .env file");
print!("Get items with query params: {:?}\n", params);
let isbns = params.get("isbns").unwrap();
let splitisbns : Vec<String> = isbns.split(",").map(|s| format!("ISBN:{}",s.to_string())).collect();
@@ -275,142 +286,185 @@ async fn create_by_isbn(
let res = reqwest::get(query).await.expect("Unable to request");
let resjson: BooksManyISBN = res.json::<BooksManyISBN>().await.expect("Unable to return value");
for isbnstring in splitisbns.iter() {
let mut cover = resjson.clone().book_items.get(isbnstring).unwrap().cover.clone().unwrap().medium.clone();
if !resjson.book_items.get(isbnstring).unwrap().cover.is_none() {
let img_bytes = reqwest::get(cover.clone()).await.unwrap().bytes().await.unwrap();
let image = image::load_from_memory(&img_bytes).unwrap();
let temp_cover = cover.clone();
let img_id = temp_cover.split("/").last().unwrap();
image.save(format!("../images/{}",img_id)).expect("Failed to save image");
cover = img_id.to_string();
}
let doc_sent = BookUI{
id: 1,
open_library_key: Some(resjson.clone().book_items.get(isbnstring).unwrap().key.clone()),
title: resjson.clone().book_items.get(isbnstring).unwrap().title.clone(),
edition_count: None,
first_publish_year: None,
median_page_count: None,
goodread_id: Some(resjson.clone().book_items.get(isbnstring).unwrap().identifiers.clone().unwrap().goodreads.unwrap().get(0).unwrap().to_string()),
description: None,
cover: Some(cover.clone()),
location: None,
time_added: None,
rating: None,
comments: Some("From openlibrary".to_string()),
author_name: Some(resjson.clone().book_items.get(isbnstring).unwrap().authors.iter().map(|s| s.name.clone()).collect()),
person: Some(resjson.clone().book_items.get(isbnstring).unwrap().subject_people.clone().unwrap().iter().map(|s| s.name.clone()).collect()),
place: Some(resjson.clone().book_items.get(isbnstring).unwrap().subject_places.clone().unwrap().iter().map(|s| s.name.clone()).collect()),
subject: Some(resjson.clone().book_items.get(isbnstring).unwrap().subjects.clone().unwrap().iter().map(|s| s.name.clone()).collect()),
time: Some(resjson.clone().book_items.get(isbnstring).unwrap().subject_times.clone().unwrap().iter().map(|s| s.name.clone()).collect()),
isbn: Some(vec![isbnstring.split("ISBN:").last().unwrap().to_string()])
};
let book: book::Model = book::Model{
open_library_key: doc_sent.open_library_key.to_owned(),
title: (doc_sent.title.to_owned()),
edition_count: doc_sent.edition_count.to_owned(),
first_publish_year: (doc_sent.first_publish_year.to_owned()),
median_page_count: (doc_sent.median_page_count.to_owned()),
goodread_id: doc_sent.goodread_id.to_owned(),
description: doc_sent.description.to_owned(),
comments: doc_sent.comments.to_owned(),
cover: Some(cover.to_owned()),
rating: doc_sent.rating.to_owned(),
time_added: doc_sent.time_added.to_owned(),
id: 1,
location: doc_sent.location.to_owned(),
};
let created_book = MutationCore::create_book(conn, book)
.await
.expect("could not create book");
let bookfound = resjson.book_items.get(isbnstring);
if !bookfound.clone().is_none() {
let mut cover_id = "".to_string();
if !bookfound.unwrap().cover.is_none() {
let coverurl = bookfound.clone().unwrap().cover.clone().unwrap().medium.clone()
let img_bytes = reqwest::get(coverurl.clone()).await.unwrap().bytes().await.unwrap();
let image = image::load_from_memory(&img_bytes).unwrap();
let temp_cover = coverurl.clone();
let img_id = temp_cover.split("/").last().unwrap();
image.save(format!("{}/{}",images_dir,img_id)).expect("Failed to save image");
cover_id = img_id.to_string();
for author_name in doc_sent.author_name.as_ref().unwrap().iter() {
let record : book_author::Model = book_author::Model{
id: 1,
book_id: (created_book.last_insert_id),
author_name: author_name.to_owned(),
};
MutationCore::create_book_author(conn, record)
.await
.expect("could not create book");
}
for person in doc_sent.person.as_ref().unwrap().iter() {
let record : book_person::Model = book_person::Model{
id: 1,
book_id: (created_book.last_insert_id),
person: person.to_owned(),
};
MutationCore::create_book_person(conn, record)
.await
.expect("could not create book");
}
let mut goodread_id = "".to_string();
if !bookfound.unwrap().identifiers.is_none() {
let goodread_identifier = bookfound.clone().unwrap().identifiers.clone().unwrap().goodreads
if !goodread_identifier.is_none() {
goodread_id = goodread_identifier.unwrap().get(0).unwrap().to_string()
}
}
let mut authors_name = Some(vec!["".to_string()]);
//if !bookfound.unwrap().authors.is_none() {
authors_name = Some(bookfound.clone().unwrap().authors.iter().map(|s| s.name.clone()).collect());
//} else {
// authors_name = None
//}
let mut persons = Some(vec!["".to_string()]);
if !bookfound.unwrap().subject_people.is_none() {
persons = Some(bookfound.clone().unwrap().subject_people.clone().unwrap().iter().map(|s| s.name.clone()).collect());
} else {
persons = None;
}
let mut places = Some(vec!["".to_string()]);
if !bookfound.unwrap().subject_places.is_none() {
places = Some(bookfound.clone().unwrap().subject_places.clone().unwrap().iter().map(|s| s.name.clone()).collect());
} else {
places = None;
}
let mut subjects = Some(vec!["".to_string()]);
if !bookfound.unwrap().subjects.is_none() {
subjects = Some(bookfound.clone().unwrap().subjects.clone().unwrap().iter().map(|s| s.name.clone()).collect());
} else {
subjects = None;
}
let mut times = Some(vec!["".to_string()]);
if !bookfound.unwrap().subject_times.is_none() {
times = Some(bookfound.clone().unwrap().subject_times.clone().unwrap().iter().map(|s| s.name.clone()).collect());
} else {
times = None;
}
}
for place in doc_sent.place.as_ref().unwrap().iter() {
let record : book_place::Model = book_place::Model{
let doc_sent = BookUI{
id: 1,
book_id: (created_book.last_insert_id),
place: place.to_owned(),
open_library_key: Some(bookfound.clone().unwrap().key.clone()),
title: bookfound.clone().unwrap().title.clone(),
edition_count: None,
first_publish_year: None,
median_page_count: None,
goodread_id: Some(goodread_id),
description: None,
cover: Some(cover_id.clone()),
location: None,
time_added: None,
rating: None,
comments: Some("From openlibrary".to_string()),
author_name: authors_name,
person: persons,
place: places,
subject: subjects,
time: times,
isbn: Some(vec![isbnstring.split("ISBN:").last().unwrap().to_string()])
};
MutationCore::create_book_place(conn, record)
.await
.expect("could not create book");
}
for subject in doc_sent.subject.as_ref().unwrap().iter() {
let record : book_subject::Model = book_subject::Model{
let book: book::Model = book::Model{
open_library_key: doc_sent.open_library_key.to_owned(),
title: (doc_sent.title.to_owned()),
edition_count: doc_sent.edition_count.to_owned(),
first_publish_year: (doc_sent.first_publish_year.to_owned()),
median_page_count: (doc_sent.median_page_count.to_owned()),
goodread_id: doc_sent.goodread_id.to_owned(),
description: doc_sent.description.to_owned(),
comments: doc_sent.comments.to_owned(),
cover: Some(cover_id.to_owned()),
rating: doc_sent.rating.to_owned(),
time_added: doc_sent.time_added.to_owned(),
id: 1,
book_id: (created_book.last_insert_id),
subject: subject.to_owned(),
location: doc_sent.location.to_owned(),
};
MutationCore::create_book_subject(conn, record)
.await
.expect("could not create book");
let created_book = MutationCore::create_book(conn, book)
.await
.expect("could not create book");
}
for time in doc_sent.time.as_ref().unwrap().iter() {
let record : book_time::Model = book_time::Model{
id: 1,
book_id: (created_book.last_insert_id),
time: time.to_owned(),
for author_name in doc_sent.author_name.as_ref().unwrap().iter() {
let record : book_author::Model = book_author::Model{
id: 1,
book_id: (created_book.last_insert_id),
author_name: author_name.to_owned(),
};
MutationCore::create_book_author(conn, record)
.await
.expect("could not create book");
}
for person in doc_sent.person.as_ref().unwrap().iter() {
let record : book_person::Model = book_person::Model{
id: 1,
book_id: (created_book.last_insert_id),
person: person.to_owned(),
};
MutationCore::create_book_person(conn, record)
.await
.expect("could not create book");
}
for place in doc_sent.place.as_ref().unwrap().iter() {
let record : book_place::Model = book_place::Model{
id: 1,
book_id: (created_book.last_insert_id),
place: place.to_owned(),
};
MutationCore::create_book_place(conn, record)
.await
.expect("could not create book");
}
for subject in doc_sent.subject.as_ref().unwrap().iter() {
let record : book_subject::Model = book_subject::Model{
id: 1,
book_id: (created_book.last_insert_id),
subject: subject.to_owned(),
};
MutationCore::create_book_subject(conn, record)
.await
.expect("could not create book");
}
for time in doc_sent.time.as_ref().unwrap().iter() {
let record : book_time::Model = book_time::Model{
id: 1,
book_id: (created_book.last_insert_id),
time: time.to_owned(),
};
MutationCore::create_book_time(conn, record)
.await
.expect("could not create book");
}
for isbn in doc_sent.isbn.as_ref().unwrap().iter() {
let record : book_isbn::Model = book_isbn::Model{
id: 1,
book_id: (created_book.last_insert_id),
isbn: isbn.to_owned(),
};
MutationCore::create_book_isbn(conn, record)
.await
.expect("could not create book");
}
let book_meili = BookMeili{
id: doc_sent.id,
open_library_key: doc_sent.open_library_key.unwrap_or("".to_string()),
title: doc_sent.title,
edition_count: doc_sent.edition_count.unwrap_or(0),
first_publish_year: doc_sent.first_publish_year.unwrap_or(0),
median_page_count: doc_sent.median_page_count.unwrap_or(0),
goodread_id: doc_sent.goodread_id.unwrap_or("".to_string()),
description: doc_sent.description.unwrap_or("".to_string()),
cover: doc_sent.cover.unwrap_or("".to_string()),
location: doc_sent.location.unwrap_or("".to_string()),
time_added: doc_sent.time_added.unwrap_or("".to_string()),
rating: doc_sent.rating.unwrap_or(0),
comments: doc_sent.comments.unwrap_or("".to_string()),
author_name: doc_sent.author_name.unwrap_or(vec!["".to_string()]),
person: doc_sent.person.unwrap_or(vec!["".to_string()]),
place: doc_sent.place.unwrap_or(vec!["".to_string()]),
subject: doc_sent.subject.unwrap_or(vec!["".to_string()]),
time: doc_sent.time.unwrap_or(vec!["".to_string()]),
isbn: doc_sent.isbn.unwrap_or(vec!["".to_string()]),
};
MutationCore::create_book_time(conn, record)
.await
.expect("could not create book");
booksman_search::create_or_update_book(book_meili, meili_client).await;
}
for isbn in doc_sent.isbn.as_ref().unwrap().iter() {
let record : book_isbn::Model = book_isbn::Model{
id: 1,
book_id: (created_book.last_insert_id),
isbn: isbn.to_owned(),
};
MutationCore::create_book_isbn(conn, record)
.await
.expect("could not create book");
}
let book_meili = BookMeili{
id: doc_sent.id,
open_library_key: doc_sent.open_library_key.unwrap_or("".to_string()),
title: doc_sent.title,
edition_count: doc_sent.edition_count.unwrap_or(0),
first_publish_year: doc_sent.first_publish_year.unwrap_or(0),
median_page_count: doc_sent.median_page_count.unwrap_or(0),
goodread_id: doc_sent.goodread_id.unwrap_or("".to_string()),
description: doc_sent.description.unwrap_or("".to_string()),
cover: doc_sent.cover.unwrap_or("".to_string()),
location: doc_sent.location.unwrap_or("".to_string()),
time_added: doc_sent.time_added.unwrap_or("".to_string()),
rating: doc_sent.rating.unwrap_or(0),
comments: doc_sent.comments.unwrap_or("".to_string()),
author_name: doc_sent.author_name.unwrap_or(vec!["".to_string()]),
person: doc_sent.person.unwrap_or(vec!["".to_string()]),
place: doc_sent.place.unwrap_or(vec!["".to_string()]),
subject: doc_sent.subject.unwrap_or(vec!["".to_string()]),
time: doc_sent.time.unwrap_or(vec!["".to_string()]),
isbn: doc_sent.isbn.unwrap_or(vec!["".to_string()]),
};
booksman_search::create_or_update_book(book_meili, meili_client).await;
}
return "Done";
}
@@ -431,6 +485,10 @@ async fn search_openlibrary(
let mut vec = Vec::with_capacity(12);
for i in 0..12 as usize {
let doc = resjson.docs[i].clone();
let mut goodread = "".to_string();
if doc.id_goodreads.is_some() {
goodread = doc.id_goodreads.unwrap().get(0).unwrap().to_string();
}
vec.push(
BookUI{
id: -((i+1) as i32),
@@ -439,7 +497,7 @@ let mut vec = Vec::with_capacity(12);
edition_count: doc.edition_count,
first_publish_year: doc.first_publish_year,
median_page_count: doc.number_of_pages_median,
goodread_id: doc.goodread_id,
goodread_id: Some(goodread),
description: doc.description,
cover: doc.cover_url,
location: None,
@@ -474,6 +532,11 @@ async fn list_book(
Extension(ref conn): Extension<DatabaseConnection>,
axum::extract::Query(params): axum::extract::Query<HashMap<String, String>>,
) -> impl IntoResponse {
dotenvy::dotenv().ok();
let backend_url = env::var("BACKEND_URL").expect("BACKEND_URL is not set in .env file");
let page: usize = params.get("page").unwrap().parse().unwrap();
let books = QueryCore::find_books_plus_meta_in_page(conn,page,5)
@@ -485,7 +548,9 @@ let mut resbooks: Vec<BookUI> = Vec::with_capacity(5);
for bookandmeta in books.0.into_iter() {
let mut cover = bookandmeta.clone().book.cover;
if !cover.is_none() {
cover = Some(format!("http://localhost:8081/images/{}",cover.unwrap()));
cover = Some(format!("{}/images/{}",backend_url,cover.unwrap()));
} else {
cover = Some(format!("{}/images/placeholder.jpg",backend_url));
}
let bookui = BookUI{
id: bookandmeta.clone().book.id,
@@ -526,6 +591,9 @@ async fn list_search_book(
) -> impl IntoResponse {
let page: usize = params.get("page").unwrap().parse().unwrap();
let search = params.get("search").unwrap();
dotenvy::dotenv().ok();
let backend_url = env::var("BACKEND_URL").expect("BACKEND_URL is not set in .env file");
let books = booksman_search::search_book(search, page,meili_client)
.await
@@ -536,7 +604,9 @@ let mut resbooks: Vec<BookUI> = Vec::with_capacity(24);
for bookmeili in books.into_iter() {
let mut cover = bookmeili.clone().cover;
if cover!="".to_string() {
cover = format!("http://localhost:8081/images/{}",cover);
cover = format!("{}/images/{}",backend_url,cover);
} else {
cover = format!("{}/images/placeholder.jpg",backend_url);
}
let bookui = BookUI{
id: bookmeili.id,
@@ -577,6 +647,10 @@ async fn create_book(
Json(doc_sent): Json<BookUI>,
) -> impl IntoResponse {
println!("Creating book");
dotenvy::dotenv().ok();
let images_dir = env::var("IMAGES_DIR").expect("IMAGES_DIR is not set in .env file");
let mut cover = doc_sent.cover.clone();
if !doc_sent.cover.is_none() {
let img_bytes = reqwest::get(cover.unwrap()).await.unwrap().bytes().await.unwrap();
@@ -585,7 +659,7 @@ async fn create_book(
let image = image::load_from_memory(&img_bytes).unwrap();
let temp_cover = doc_sent.cover.clone().unwrap();
let img_id = temp_cover.split("/").last().unwrap();
image.save(format!("../images/{}",img_id)).expect("Failed to save image");
image.save(format!("{}/{}",images_dir,img_id)).expect("Failed to save image");
cover = Some(img_id.to_string());
}
let book: book::Model = book::Model{
@@ -707,6 +781,10 @@ async fn update_book(
Json(doc_sent): Json<BookUI>,
) -> impl IntoResponse {
println!("Updating book");
dotenvy::dotenv().ok();
let images_dir = env::var("IMAGES_DIR").expect("IMAGES_DIR is not set in .env file");
let mut cover = doc_sent.cover.clone();
if !doc_sent.cover.is_none() {
let img_bytes = reqwest::get(cover.unwrap()).await.unwrap().bytes().await.unwrap();
@@ -715,7 +793,7 @@ if !doc_sent.cover.is_none() {
let image = image::load_from_memory(&img_bytes).unwrap();
let temp_cover = doc_sent.cover.clone().unwrap();
let img_id = temp_cover.split("/").last().unwrap();
image.save(format!("../images/{}",img_id)).expect("Failed to save image");
image.save(format!("{}/{}",images_dir,img_id)).expect("Failed to save image");
cover = Some(img_id.to_string());
}
let book: book::Model = book::Model{