chore: remove redundant metric
This commit is contained in:
@@ -1,15 +1,15 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
use crate::gallery::{list_images, read_index_json};
|
||||
use anyhow::Result;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
use rbv_entity::{ClipEmbedding, FaceDetection, Gallery, GalleryImage, Image, ImageId};
|
||||
use rbv_hash::{face_id, gallery_id, image_id};
|
||||
use std::sync::Arc;
|
||||
use rbv_ml::{AnalysisResult, MlBackend, MlError};
|
||||
use sqlx::PgPool;
|
||||
use crate::gallery::{list_images, read_index_json};
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::Semaphore;
|
||||
use tracing::{info, warn};
|
||||
|
||||
pub struct IngestConfig {
|
||||
pub concurrency: usize,
|
||||
@@ -96,7 +96,10 @@ pub async fn ingest_galleries(
|
||||
let (file_count, total_bytes) = match compute_gallery_stats(&image_paths) {
|
||||
Ok(stats) => stats,
|
||||
Err(e) => {
|
||||
warn!("Cannot stat gallery {}, processing anyway: {e}", gallery_path.display());
|
||||
warn!(
|
||||
"Cannot stat gallery {}, processing anyway: {e}",
|
||||
gallery_path.display()
|
||||
);
|
||||
(-1, -1)
|
||||
}
|
||||
};
|
||||
@@ -110,7 +113,6 @@ pub async fn ingest_galleries(
|
||||
let checked = report.galleries_skipped + report.galleries_processed;
|
||||
if report.galleries_skipped % 100 == 0 {
|
||||
info!(
|
||||
checked,
|
||||
skipped = report.galleries_skipped,
|
||||
indexed = report.galleries_processed,
|
||||
"progress={checked}/{total_galleries}"
|
||||
@@ -125,7 +127,10 @@ pub async fn ingest_galleries(
|
||||
let existing = match rbv_data::image::existing_gallery_images(pool, &gid).await {
|
||||
Ok(m) => Arc::new(m),
|
||||
Err(e) => {
|
||||
warn!("Cannot fetch existing images for {}: {e}", gallery_path.display());
|
||||
warn!(
|
||||
"Cannot fetch existing images for {}: {e}",
|
||||
gallery_path.display()
|
||||
);
|
||||
Arc::new(HashMap::new())
|
||||
}
|
||||
};
|
||||
@@ -182,7 +187,9 @@ pub async fn ingest_galleries(
|
||||
g_errors += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
report.errors.push((gallery_path.clone(), anyhow::anyhow!("task panicked: {e}")));
|
||||
report
|
||||
.errors
|
||||
.push((gallery_path.clone(), anyhow::anyhow!("task panicked: {e}")));
|
||||
g_errors += 1;
|
||||
}
|
||||
}
|
||||
@@ -191,7 +198,10 @@ pub async fn ingest_galleries(
|
||||
// Batch-upsert all skipped gallery_image rows in one query.
|
||||
if !skipped_gis.is_empty() {
|
||||
if let Err(e) = rbv_data::image::upsert_gallery_images_batch(pool, &skipped_gis).await {
|
||||
warn!("batch upsert gallery_images failed for {}: {e:#}", gid.to_hex());
|
||||
warn!(
|
||||
"batch upsert gallery_images failed for {}: {e:#}",
|
||||
gid.to_hex()
|
||||
);
|
||||
g_errors += 1;
|
||||
report.errors.push((gallery_path.clone(), e));
|
||||
}
|
||||
@@ -211,10 +221,8 @@ pub async fn ingest_galleries(
|
||||
);
|
||||
|
||||
if g_errors == 0 && file_count >= 0 {
|
||||
let _ = rbv_data::gallery::update_gallery_stats(
|
||||
pool, &gid, file_count, total_bytes,
|
||||
)
|
||||
.await;
|
||||
let _ =
|
||||
rbv_data::gallery::update_gallery_stats(pool, &gid, file_count, total_bytes).await;
|
||||
}
|
||||
|
||||
report.galleries_processed += 1;
|
||||
@@ -309,7 +317,10 @@ async fn process_image(
|
||||
let result = match analyze_with_backoff(ml, &ml_bytes, image_path).await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
warn!("Skipping {} after ML retries exhausted: {e:#}", image_path.display());
|
||||
warn!(
|
||||
"Skipping {} after ML retries exhausted: {e:#}",
|
||||
image_path.display()
|
||||
);
|
||||
return Ok(ImageResult::Skipped(gi));
|
||||
}
|
||||
};
|
||||
@@ -323,7 +334,10 @@ async fn process_image(
|
||||
image_id: iid.clone(),
|
||||
embedding: result.clip_embedding,
|
||||
};
|
||||
let faces: Vec<FaceDetection> = result.faces.into_iter().map(|detected| {
|
||||
let faces: Vec<FaceDetection> = result
|
||||
.faces
|
||||
.into_iter()
|
||||
.map(|detected| {
|
||||
let fid = face_id(&iid, &detected.bounding_box);
|
||||
FaceDetection {
|
||||
id: fid,
|
||||
@@ -333,7 +347,8 @@ async fn process_image(
|
||||
embedding: detected.embedding,
|
||||
person_id: None,
|
||||
}
|
||||
}).collect();
|
||||
})
|
||||
.collect();
|
||||
let face_count = faces.len();
|
||||
|
||||
let mut tx = pool.begin().await?;
|
||||
@@ -394,6 +409,9 @@ fn prepare_for_ml(bytes: &[u8]) -> anyhow::Result<Vec<u8>> {
|
||||
};
|
||||
|
||||
let mut out = Vec::new();
|
||||
img.write_to(&mut std::io::Cursor::new(&mut out), image::ImageFormat::Jpeg)?;
|
||||
img.write_to(
|
||||
&mut std::io::Cursor::new(&mut out),
|
||||
image::ImageFormat::Jpeg,
|
||||
)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user