From 1bf028d6ee6c2ee620c7b56f6d9d5f5ac7631456 Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Fri, 19 Aug 2011 11:20:55 -0400 Subject: [PATCH] Vandelay: Insufficient Quality Fall-Thru Merge Profile When a user selects "Merge On Single Match" or "Merge On Best Match" and apply a minimum quality ratio, the fall-through profile will be used in a secondary merge attempt for any records that fail the initial merge due to insufficient record quality. This allows the user to choose 2 different outcomes based on record quality. For example, you may wish to do a full overlay for all high-quality records, but only extract subjects or other information from lower quality records. This also allows users to do record+copy imports for high-quality records and copy-only imports for low quality records (using a no-op merge profile like "replace 901c"). Signed-off-by: Bill Erickson Signed-off-by: Mike Rylander --- .../lib/OpenILS/Application/Vandelay.pm | 190 ++++++++++++------ .../web/js/ui/default/vandelay/vandelay.js | 19 ++ Open-ILS/web/opac/locale/en-US/vandelay.dtd | 1 + .../templates/default/vandelay/inc/queue.tt2 | 6 + .../templates/default/vandelay/inc/upload.tt2 | 7 + 5 files changed, 159 insertions(+), 64 deletions(-) diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm index 2b563a85b0..62dca2e073 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm @@ -885,6 +885,7 @@ sub import_record_list_impl { my $auto_overlay_best = $$args{auto_overlay_best_match}; my $match_quality_ratio = $$args{match_quality_ratio}; my $merge_profile = $$args{merge_profile}; + my $ft_merge_profile = $$args{fall_through_merge_profile}; my $bib_source = $$args{bib_source}; my $import_no_match = $$args{import_no_match}; @@ -990,40 +991,17 @@ sub import_record_list_impl { if( scalar(keys %match_recs) == 1) { # all matches point to the same record - # $auto_overlay_best_func will find the 1 match and - # overlay if the quality ratio allows it - - my $res = $e->json_query( - { - from => [ - $auto_overlay_best_func, - $rec_id, - $merge_profile, - $match_quality_ratio - ] - } + ($imported, $error, $rec) = try_auto_overlay( + $e, $type, + $report_args, + $auto_overlay_best_func, + $retrieve_func, + $rec_class, + $rec_id, + $match_quality_ratio, + $merge_profile, + $ft_merge_profile ); - - if($res and ($res = $res->[0])) { - - if($res->{$auto_overlay_best_func} eq 't') { - $logger->info("vl: $type overlay-1match succeeded for queued rec $rec_id"); - $imported = 1; - - # re-fetch the record to pick up the imported_as value from the DB - $$report_args{rec} = $rec = $e->$retrieve_func([ - $rec_id, {flesh => 1, flesh_fields => {$rec_class => ['matches']}}]); - - - } else { - $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; - $logger->info("vl: $type overlay-1match failed for queued rec $rec_id"); - } - - } else { - $error = 1; - $logger->error("vl: Error attempting overlay with func=$auto_overlay_best_func, profile=$merge_profile, record=$rec_id"); - } } } @@ -1063,40 +1041,19 @@ sub import_record_list_impl { } if(!$imported and !$error and $auto_overlay_best and scalar(@{$rec->matches}) > 0 ) { - # caller says to overlay the best match - my $res = $e->json_query( - { - from => [ - $auto_overlay_best_func, - $rec_id, - $merge_profile, - $match_quality_ratio - ] - } + ($imported, $error, $rec) = try_auto_overlay( + $e, $type, + $report_args, + $auto_overlay_best_func, + $retrieve_func, + $rec_class, + $rec_id, + $match_quality_ratio, + $merge_profile, + $ft_merge_profile ); - - if($res and ($res = $res->[0])) { - - if($res->{$auto_overlay_best_func} eq 't') { - $logger->info("vl: $type auto-overlay-best succeeded for queued rec $rec_id"); - $imported = 1; - - # re-fetch the record to pick up the imported_as value from the DB - $$report_args{rec} = $rec = $e->$retrieve_func([ - $rec_id, {flesh => 1, flesh_fields => {$rec_class => ['matches']}}]); - - } else { - $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; - $logger->info("vl: $type auto-overlay-best failed for queued rec $rec_id"); - } - - } else { - $error = 1; - $logger->error("vl: Error attempting overlay with func=$auto_overlay_best_func, ". - "quality_ratio=$match_quality_ratio, profile=$merge_profile, record=$rec_id"); - } } if(!$imported and !$error and $import_no_match and scalar(@{$rec->matches}) == 0) { @@ -1176,6 +1133,111 @@ sub import_record_list_impl { return undef; } + +sub try_auto_overlay { + my $e = shift; + my $type = shift; + my $report_args = shift; + my $overlay_func = shift; + my $retrieve_func = shift; + my $rec_class = shift; + my $rec_id = shift; + my $match_quality_ratio = shift; + my $merge_profile = shift; + my $ft_merge_profile = shift; + + my $imported = 0; + my $error = 0; + + # Find the best match and overlay if the quality ratio allows it. + my $res = $e->json_query( + { + from => [ + $overlay_func, + $rec_id, + $merge_profile, + $match_quality_ratio + ] + } + ); + + if($res and ($res = $res->[0])) { + + if($res->{$overlay_func} eq 't') { + + # first attempt succeeded + $imported = 1; + + } else { + + # quality-limited merge failed with insufficient quality. If there is a + # fall-through merge profile, re-do the merge with the alternate profile + # and no quality restriction. + + if($ft_merge_profile and $match_quality_ratio > 0) { + + $logger->info("vl: $type auto-merge failed with profile $merge_profile; ". + "re-merging with fall-through profile $ft_merge_profile"); + + my $res = $e->json_query( + { + from => [ + $overlay_func, + $rec_id, + $ft_merge_profile, + 0 # minimum quality not required + ] + } + ); + + if($res and ($res = $res->[0])) { + + if($res->{$overlay_func} eq 't') { + + # second attempt succeeded + $imported = 1; + + } else { + + # failed to merge on second attempt + $logger->info("vl: $type auto-merge with fall-through failed for queued rec $rec_id"); + } + } else { + + # second attempt died + $error = 1; + $logger->error("vl: Error attempting overlay with func=$overlay_func, profile=$merge_profile, record=$rec_id"); + } + + } else { + + # failed to merge on first attempt, no fall-through was provided + $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; + $logger->info("vl: $type auto-merge failed for queued rec $rec_id"); + } + } + + } else { + + # first attempt died + $error = 1; + $logger->error("vl: Error attempting overlay with func=$overlay_func, profile=$merge_profile, record=$rec_id"); + } + + if($imported) { + + # at least 1 of the attempts succeeded + $logger->info("vl: $type auto-merge succeeded for queued rec $rec_id"); + + # re-fetch the record to pick up the imported_as value from the DB + $$report_args{rec} = $e->$retrieve_func([ + $rec_id, {flesh => 1, flesh_fields => {$rec_class => ['matches']}}]); + } + + return ($imported, $error, $$report_args{rec}); +} + + # tracks any import errors, commits the current xact, responds to the client sub finish_rec_import_attempt { my $args = shift; diff --git a/Open-ILS/web/js/ui/default/vandelay/vandelay.js b/Open-ILS/web/js/ui/default/vandelay/vandelay.js index 3d6f019ffe..48423c5862 100644 --- a/Open-ILS/web/js/ui/default/vandelay/vandelay.js +++ b/Open-ILS/web/js/ui/default/vandelay/vandelay.js @@ -120,6 +120,16 @@ function vlInit() { vlUploadMergeProfile2.searchAttr = 'name'; vlUploadMergeProfile2.startup(); + vlUploadFtMergeProfile.store = new dojo.data.ItemFileReadStore({data:fieldmapper.vmp.toStoreData(mergeProfiles)}); + vlUploadFtMergeProfile.labelAttr = 'name'; + vlUploadFtMergeProfile.searchAttr = 'name'; + vlUploadFtMergeProfile.startup(); + + vlUploadFtMergeProfile2.store = new dojo.data.ItemFileReadStore({data:fieldmapper.vmp.toStoreData(mergeProfiles)}); + vlUploadFtMergeProfile2.labelAttr = 'name'; + vlUploadFtMergeProfile2.searchAttr = 'name'; + vlUploadFtMergeProfile2.startup(); + // Fetch the bib and authority attribute definitions vlFetchBibAttrDefs(function () { checkInitDone(); }); @@ -1098,6 +1108,7 @@ function vlHandleQueueItemsAction(action) { vlUploadQueueAutoOverlayExact.attr('value', vlUploadQueueAutoOverlayExact2.attr('value')); vlUploadQueueAutoOverlay1Match.attr('value', vlUploadQueueAutoOverlay1Match2.attr('value')); vlUploadMergeProfile.attr('value', vlUploadMergeProfile2.attr('value')); + vlUploadFtMergeProfile.attr('value', vlUploadFtMergeProfile2.attr('value')); vlUploadQueueAutoOverlayBestMatch.attr('value', vlUploadQueueAutoOverlayBestMatch2.attr('value')); vlUploadQueueAutoOverlayBestMatchRatio.attr('value', vlUploadQueueAutoOverlayBestMatchRatio2.attr('value')); @@ -1116,6 +1127,8 @@ function vlHandleQueueItemsAction(action) { vlUploadQueueAutoOverlay1Match2.attr('value', false); vlUploadMergeProfile.attr('value', ''); vlUploadMergeProfile2.attr('value', ''); + vlUploadFtMergeProfile.attr('value', ''); + vlUploadFtMergeProfile2.attr('value', ''); vlUploadQueueAutoOverlayBestMatch.attr('value', false); vlUploadQueueAutoOverlayBestMatch2.attr('value', false); vlUploadQueueAutoOverlayBestMatchRatio.attr('value', '0.0'); @@ -1196,6 +1209,12 @@ function vlImportRecordQueue(type, queueId, recList, onload) { options.merge_profile = profile; } + var ftprofile = vlUploadFtMergeProfile.attr('value'); + if(ftprofile != null && ftprofile != '') { + options.fall_through_merge_profile = ftprofile; + } + + /* determine which method we're calling */ var method = 'open-ils.vandelay.bib_queue.import'; diff --git a/Open-ILS/web/opac/locale/en-US/vandelay.dtd b/Open-ILS/web/opac/locale/en-US/vandelay.dtd index aa9649a3a9..99dac5c969 100644 --- a/Open-ILS/web/opac/locale/en-US/vandelay.dtd +++ b/Open-ILS/web/opac/locale/en-US/vandelay.dtd @@ -9,6 +9,7 @@ + diff --git a/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 b/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 index 466ee4f9fc..749434f0a5 100644 --- a/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 +++ b/Open-ILS/web/templates/default/vandelay/inc/queue.tt2 @@ -230,6 +230,12 @@ (&vandelay.auto.import.auto_overlay_best_ratio.desc;) + + &vandelay.auto.import.ft_merge_profile; + +
+ + diff --git a/Open-ILS/web/templates/default/vandelay/inc/upload.tt2 b/Open-ILS/web/templates/default/vandelay/inc/upload.tt2 index b43225226a..0c1bbd5342 100644 --- a/Open-ILS/web/templates/default/vandelay/inc/upload.tt2 +++ b/Open-ILS/web/templates/default/vandelay/inc/upload.tt2 @@ -82,6 +82,13 @@ (&vandelay.auto.import.auto_overlay_best_ratio.desc;) + + &vandelay.auto.import.ft_merge_profile; + +
+ + + -- 2.43.2