From 12029c8f696863957ddbbdb784c4dda0a8e94f56 Mon Sep 17 00:00:00 2001 From: berick Date: Thu, 19 May 2011 15:26:49 -0400 Subject: [PATCH] Vandelay import improvements / bug fixes * Allow for quality ratio control during import-on-1match * Allow for 901c matching when no match-set is selected * Tidy up some display strings * Numerous small bug fixes --- .../lib/OpenILS/Application/Vandelay.pm | 79 ++++++++++++------- Open-ILS/src/sql/Pg/012.schema.vandelay.sql | 37 +++++---- .../web/js/ui/default/vandelay/vandelay.js | 1 + Open-ILS/web/opac/locale/en-US/vandelay.dtd | 14 ++-- 4 files changed, 80 insertions(+), 51 deletions(-) diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm index 4a0bf99449..65a00138d8 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm @@ -927,8 +927,8 @@ sub import_record_list_impl { $e->requestor($requestor); $$report_args{e} = $e; - $$report_args{import_error} = undef; $$report_args{evt} = undef; + $$report_args{import_error} = undef; my $rec = $e->$retrieve_func([ $rec_id, @@ -962,7 +962,7 @@ sub import_record_list_impl { { from => [ $overlay_func, - $rec->id, + $rec_id, $overlay_target, $merge_profile ] @@ -972,8 +972,8 @@ sub import_record_list_impl { if($res and ($res = $res->[0])) { if($res->{$overlay_func} eq 't') { - $logger->info("vl: $type direct overlay succeeded for queued rec " . - $rec->id . " and overlay target $overlay_target"); + $logger->info("vl: $type direct overlay succeeded for queued rec ". + "$rec_id and overlay target $overlay_target"); $imported = 1; } @@ -984,34 +984,39 @@ sub import_record_list_impl { } else { - if($auto_overlay_1match) { - # caller says to overlay if there is exactly 1 match + if($auto_overlay_1match) { # overlay if there is exactly 1 match my %match_recs = map { $_->eg_record => 1 } @{$rec->matches}; if( scalar(keys %match_recs) == 1) { # all matches point to the same record + # $auto_overlay_best_func will find the 1 match and + # overlay if the quality ratio allows it + my $res = $e->json_query( { from => [ - $overlay_func, - $rec->id, - $rec->matches->[0]->eg_record, - $merge_profile + $auto_overlay_best_func, + $rec_id, + $merge_profile, + $match_quality_ratio ] } ); if($res and ($res = $res->[0])) { - if($res->{$overlay_func} eq 't') { - $logger->info("vl: $type overlay-1match succeeded for queued rec " . $rec->id); + if($res->{$auto_overlay_best_func} eq 't') { + $logger->info("vl: $type overlay-1match succeeded for queued rec $rec_id"); $imported = 1; + } else { + $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; + $logger->info("vl: $type overlay-1match failed for queued rec $rec_id"); } } else { $error = 1; - $logger->error("vl: Error attempting overlay with func=$overlay_func, profile=$merge_profile, record=$rec_id"); + $logger->error("vl: Error attempting overlay with func=$auto_overlay_best_func, profile=$merge_profile, record=$rec_id"); } } } @@ -1019,12 +1024,13 @@ sub import_record_list_impl { if(!$imported and !$error and $auto_overlay_exact and scalar(@{$rec->matches}) == 1 ) { # caller says to overlay if there is an /exact/ match + # $auto_overlay_func only proceeds and returns true on exact matches my $res = $e->json_query( { from => [ $auto_overlay_func, - $rec->id, + $rec_id, $merge_profile ] } @@ -1033,10 +1039,10 @@ sub import_record_list_impl { if($res and ($res = $res->[0])) { if($res->{$auto_overlay_func} eq 't') { - $logger->info("vl: $type auto-overlay succeeded for queued rec " . $rec->id); + $logger->info("vl: $type auto-overlay succeeded for queued rec $rec_id"); $imported = 1; } else { - $logger->info("vl: $type auto-overlay failed for queued rec " . $rec->id); + $logger->info("vl: $type auto-overlay failed for queued rec $rec_id"); } } else { @@ -1053,7 +1059,7 @@ sub import_record_list_impl { { from => [ $auto_overlay_best_func, - $rec->id, + $rec_id, $merge_profile, $match_quality_ratio ] @@ -1063,11 +1069,11 @@ sub import_record_list_impl { if($res and ($res = $res->[0])) { if($res->{$auto_overlay_best_func} eq 't') { - $logger->info("vl: $type auto-overlay-best succeeded for queued rec " . $rec->id); + $logger->info("vl: $type auto-overlay-best succeeded for queued rec $rec_id"); $imported = 1; } else { $$report_args{import_error} = 'overlay.record.quality' if $match_quality_ratio > 0; - $logger->info("vl: $type auto-overlay-best failed for queued rec " . $rec->id); + $logger->info("vl: $type auto-overlay-best failed for queued rec $rec_id"); } } else { @@ -1097,20 +1103,28 @@ sub import_record_list_impl { $logger->info("vl: successfully imported new $type record"); $rec->imported_as($record->id); - $rec->import_time('now'); - $rec->clear_import_error; - $rec->clear_error_detail; - $imported = 1 if $e->$update_func($rec); } } } if($imported) { - push @success_rec_ids, $rec_id; - finish_rec_import_attempt($report_args); - } else { - # Send an update whenever there's an error + $rec->import_time('now'); + $rec->clear_import_error; + $rec->clear_error_detail; + + if($e->$update_func($rec)) { + + push @success_rec_ids, $rec_id; + finish_rec_import_attempt($report_args); + + } else { + $imported = 0; + } + } + + if(!$imported) { + $logger->info("vl: record $rec_id was not imported"); $$report_args{evt} = $e->event unless $$report_args{evt}; finish_rec_import_attempt($report_args); } @@ -1137,7 +1151,7 @@ sub import_record_list_impl { } # import the copies - import_record_asset_list_impl($conn, \@success_rec_ids, $requestor); + import_record_asset_list_impl($conn, \@success_rec_ids, $requestor) if @success_rec_ids; $conn->respond({total => $$report_args{total}, progress => $$report_args{progress}}); return undef; @@ -1402,7 +1416,10 @@ sub import_record_asset_list_impl { $rec_ids = $roe->json_query({ select => {vqbr => ['id']}, from => {vqbr => 'vii'}, - where => {'+vqbr' => {import_time => {'!=' => undef}}}, + where => {'+vqbr' => { + id => $rec_ids, + import_time => {'!=' => undef} + }}, distinct => 1 }); $rec_ids = [map {$_->{id}} @$rec_ids]; @@ -1555,7 +1572,9 @@ sub respond_with_status { if($$args{report_all} or ($$args{progress} % $$args{step}) == 0) { $$args{conn}->respond({ - map { $_ => $args->{$_} } qw/total progress success_count/, + total => $$args{total}, + progress => $$args{progress}, + success_count => $$args{success_count}, err_event => $evt }); $$args{step} *= 2 unless $$args{step} == 256; diff --git a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql index b136e8e035..d461d4fd87 100644 --- a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql +++ b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql @@ -677,9 +677,9 @@ $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION vandelay.match_bib_record() RETURNS TRIGGER AS $func$ DECLARE incoming_existing_id TEXT; - my_bib_queue vandelay.bib_queue%ROWTYPE; test_result vandelay.match_set_test_result%ROWTYPE; tmp_rec BIGINT; + match_set INT; BEGIN IF TG_OP IN ('INSERT','UPDATE') AND NEW.imported_as IS NOT NULL THEN RETURN NEW; @@ -687,14 +687,12 @@ BEGIN DELETE FROM vandelay.bib_match WHERE queued_record = NEW.id; - SELECT * INTO my_bib_queue FROM vandelay.bib_queue WHERE id = NEW.queue; + SELECT q.match_set INTO match_set FROM vandelay.bib_queue q WHERE q.id = NEW.queue; - IF my_bib_queue.match_set IS NULL THEN - RETURN NEW; + IF match_set IS NOT NULL THEN + NEW.quality := vandelay.measure_record_quality( NEW.marc, match_set ); END IF; - NEW.quality := vandelay.measure_record_quality( NEW.marc, my_bib_queue.match_set ); - -- Perfect matches on 901$c exit early with a match with high quality. incoming_existing_id := oils_xpath_string('//*[@tag="901"]/*[@code="c"][1]', NEW.marc); @@ -702,22 +700,33 @@ BEGIN IF incoming_existing_id IS NOT NULL AND incoming_existing_id != '' THEN SELECT id INTO tmp_rec FROM biblio.record_entry WHERE id = incoming_existing_id::bigint; IF tmp_rec IS NOT NULL THEN - INSERT INTO vandelay.bib_match (queued_record, eg_record, quality) VALUES ( NEW.id, incoming_existing_id::bigint, 9999); - RETURN NEW; + INSERT INTO vandelay.bib_match (queued_record, eg_record, match_score, quality) + SELECT + NEW.id, + b.id, + 9999, + -- note: no match_set means quality==0 + vandelay.measure_record_quality( b.marc, match_set ) + FROM biblio.record_entry b + WHERE id = incoming_existing_id::bigint; END IF; END IF; + IF match_set IS NULL THEN + RETURN NEW; + END IF; FOR test_result IN SELECT * FROM - vandelay.match_set_test_marcxml(my_bib_queue.match_set, NEW.marc) LOOP + vandelay.match_set_test_marcxml(match_set, NEW.marc) LOOP INSERT INTO vandelay.bib_match ( queued_record, eg_record, match_score, quality ) - SELECT NEW.id, - test_result.record, + SELECT + NEW.id, + test_result.record, test_result.quality, - vandelay.measure_record_quality( b.marc, my_bib_queue.match_set ) - FROM biblio.record_entry b - WHERE id = test_result.record; + vandelay.measure_record_quality( b.marc, match_set ) + FROM biblio.record_entry b + WHERE id = test_result.record; END LOOP; diff --git a/Open-ILS/web/js/ui/default/vandelay/vandelay.js b/Open-ILS/web/js/ui/default/vandelay/vandelay.js index b67352af5a..e0a7eaf9f8 100644 --- a/Open-ILS/web/js/ui/default/vandelay/vandelay.js +++ b/Open-ILS/web/js/ui/default/vandelay/vandelay.js @@ -1143,6 +1143,7 @@ function vlImportRecordQueue(type, queueId, recList, onload) { if(vlUploadQueueAutoOverlay1Match.checked) { options.auto_overlay_1match = true; vlUploadQueueAutoOverlay1Match.checked = false; + options.match_quality_ratio = vlUploadQueueAutoOverlayBestMatchRatio.attr('value'); mergeOpt = true; } diff --git a/Open-ILS/web/opac/locale/en-US/vandelay.dtd b/Open-ILS/web/opac/locale/en-US/vandelay.dtd index d2d1789d9b..7928620720 100644 --- a/Open-ILS/web/opac/locale/en-US/vandelay.dtd +++ b/Open-ILS/web/opac/locale/en-US/vandelay.dtd @@ -2,12 +2,12 @@ - - - - + + + + - + @@ -52,8 +52,8 @@ - - + + -- 2.43.2