From 1f139d6ebf37cb2941c962d4cc711988c82bda58 Mon Sep 17 00:00:00 2001 From: Liam Whalen Date: Wed, 1 Oct 2014 19:17:34 -0700 Subject: [PATCH] LP#1444644 Copy Import Development Work Two new options for importing holdings have been added to MARC Batch Import/Export: 1) Auto-overlay On-order Cataloguing Copies This is similar to "Auto-overlay In-process Acquisitions Copies," but for copies that were not created from an acquisitions workflow. Holdings information in the incoming record will be used to overlay any existing On Order copies for the matching record which belong to the owning library defined in the Holdings Import Profile. The Holdings Import Profile is also used to match incoming to existing copies, if possible; otherwise, On Order copies are overlaid in the order they were created. The call number will also be overlaid if the incoming record provides one. 2) Use Org Unit Matching in Copy to Determine Best Match When there are multiple potential matching records, this feature allows the user to automatically select the record which has the most copies at libraries near the importing library in the org tree. That is, starting at the importing library, it climbs the org tree, gradually expanding the scope at which it checks for holdings on matching records; once holdings are found, the record with the most holdings at that scope is selected for overlay. If there are no matching records with holdings, then the default best match overlay is attempted. Two new permissions are added to control the use of these new features. Signed-off-by: Liam Whalen Signed-off-by: Jeff Davis Signed-off-by: Galen Charlton --- .../staff/cat/vandelay/import.component.html | 24 ++++++ .../staff/cat/vandelay/import.component.ts | 8 ++ .../lib/OpenILS/Application/Vandelay.pm | 80 +++++++++++++++++++ Open-ILS/src/sql/Pg/012.schema.vandelay.sql | 70 ++++++++++++++++ Open-ILS/src/sql/Pg/950.data.seed-values.sql | 6 +- ....vandelay.auto_overlay_org_unit_copies.sql | 79 ++++++++++++++++++ Open-ILS/src/templates/vandelay/inc/queue.tt2 | 11 +++ .../src/templates/vandelay/inc/upload.tt2 | 10 +++ .../web/js/ui/default/vandelay/vandelay.js | 20 ++++- .../Cataloging/copy-import-options.adoc | 33 ++++++++ 10 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay.auto_overlay_org_unit_copies.sql create mode 100644 docs/RELEASE_NOTES_NEXT/Cataloging/copy-import-options.adoc diff --git a/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.html b/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.html index 47ab53519a..2882bc55d8 100644 --- a/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.html +++ b/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.html @@ -216,6 +216,19 @@ +
+ +
+
+ +
+ +
@@ -231,6 +244,17 @@
+
+ +
+
+ +
diff --git a/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts b/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts index 15b5640fff..391efdeb77 100644 --- a/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts +++ b/Open-ILS/src/eg2/src/app/staff/cat/vandelay/import.component.ts @@ -29,6 +29,8 @@ const TEMPLATE_ATTRS = [ 'mergeOnBestMatch', 'mergeOnSingleMatch', 'autoOverlayAcqCopies', + 'autoOverlayOnOrderCopies', + 'autoOverlayOrgUnitCopies', 'selectedHoldingsProfile', 'selectedMergeProfile', 'selectedFallThruMergeProfile', @@ -44,6 +46,8 @@ interface ImportOptions { auto_overlay_best_match?: boolean; auto_overlay_1match?: boolean; opp_acq_copy_overlay?: boolean; + opp_oo_cat_copy_overlay?: boolean; + auto_overlay_org_unit_copies?: boolean; merge_profile?: any; fall_through_merge_profile?: any; strip_field_groups?: number[]; @@ -83,6 +87,8 @@ export class ImportComponent implements OnInit, AfterViewInit, OnDestroy { mergeOnBestMatch: boolean; minQualityRatio: number; autoOverlayAcqCopies: boolean; + autoOverlayOnOrderCopies: boolean; + autoOverlayOrgUnitCopies: boolean; // True after the first upload, then remains true. showProgress: boolean; @@ -558,6 +564,8 @@ export class ImportComponent implements OnInit, AfterViewInit, OnDestroy { auto_overlay_best_match: this.mergeOnBestMatch, auto_overlay_1match: this.mergeOnSingleMatch, opp_acq_copy_overlay: this.autoOverlayAcqCopies, + opp_oo_cat_copy_overlay: this.autoOverlayOnOrderCopies, + auto_overlay_org_unit_copies: this.autoOverlayOrgUnitCopies, merge_profile: this.selectedMergeProfile, fall_through_merge_profile: this.selectedFallThruMergeProfile, strip_field_groups: this.selectedTrashGroups, diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm index 4ee6b28502..f34c8aeaee 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Vandelay.pm @@ -998,6 +998,7 @@ sub import_record_list_impl { my $auto_overlay_exact = $$args{auto_overlay_exact}; my $auto_overlay_1match = $$args{auto_overlay_1match}; my $auto_overlay_best = $$args{auto_overlay_best_match}; + my $auto_overlay_org_unit_copies = $$args{auto_overlay_org_unit_copies}; my $match_quality_ratio = $$args{match_quality_ratio}; my $merge_profile = $$args{merge_profile}; my $ft_merge_profile = $$args{fall_through_merge_profile}; @@ -1008,6 +1009,7 @@ sub import_record_list_impl { my $overlay_func = 'vandelay.overlay_bib_record'; my $auto_overlay_func = 'vandelay.auto_overlay_bib_record'; my $auto_overlay_best_func = 'vandelay.auto_overlay_bib_record_with_best'; + my $auto_overlay_org_unit_copies_func = 'vandelay.auto_overlay_org_unit_copies'; my $retrieve_func = 'retrieve_vandelay_queued_bib_record'; my $update_func = 'update_vandelay_queued_bib_record'; my $search_func = 'search_vandelay_queued_bib_record'; @@ -1102,6 +1104,7 @@ sub import_record_list_impl { my $record; my $imported = 0; + if ($type eq 'bib') { # strip configured / selected MARC tags from inbound records @@ -1221,6 +1224,30 @@ sub import_record_list_impl { ); } + if(!$imported and !$error and $auto_overlay_org_unit_copies and scalar(@{$rec->matches}) > 0 ) { + # caller says to overlay depending on the number of copies attached to a record, whose OU + # matches the OU of the import record's Holding's Import Profile. + + my $perm = 'IMPORT_USE_ORG_UNIT_COPIES'; + my $rec_ou = $e->requestor->ws_ou; + + if (!$e->allowed($perm, $rec_ou)) { + return $e->die_event; + } + + ($imported, $error, $rec) = try_auto_overlay( + $e, $type, + $report_args, + $auto_overlay_org_unit_copies_func, + $retrieve_func, + $rec_class, + $rec_id, + $match_quality_ratio, + $merge_profile, + $ft_merge_profile + ); + } + if(!$imported and !$error and $import_no_match and scalar(@{$rec->matches}) == 0) { # No overlay / merge occurred. Do a traditional record import by creating a new record @@ -1784,6 +1811,7 @@ sub import_record_asset_list_impl { my $auto_callnumber = {}; my $opp_acq_copy_overlay = $args->{opp_acq_copy_overlay}; + my $opp_oo_cat_copy_overlay = $args->{opp_oo_cat_copy_overlay}; my @overlaid_copy_ids; for my $item_id (@$item_ids) { my $e = new_editor(requestor => $requestor, xact => 1); @@ -1873,6 +1901,58 @@ sub import_record_asset_list_impl { $copy = $acqlid->eg_copy_id; push(@overlaid_copy_ids, $copy->id); } + } elsif ($opp_oo_cat_copy_overlay) { # we are going to "opportunistically" overlay received, On-order catalogue copies + + my $perm = 'IMPORT_ON_ORDER_CAT_COPY'; + my $rec_ou = $e->requestor->ws_ou; + + if (!$e->allowed($perm, $rec_ou)) { + return $e->die_event; + } + + my $query; + if ($item->copy_number) { + $query = [ + { + "status" => OILS_COPY_STATUS_ON_ORDER, + "copy_number" => $item->copy_number, + "+acn" => [{"owning_lib" => $item->owning_lib}, + {"record" => $rec->imported_as}] + }, + { + "join" => "acn", + "flesh" => 1, + "flesh_fields" => { + "acp" => ["call_number"] + } + } + ]; + } else { + #see if we have copies in vandelay.import_item that + #belong to the current record, but do not have $item->copy_number defined + #in their on-order records. Retrieve them by create date from oldest to + #newest ORDER BY acp.create_date ASC + $query = [ + { + "status" => OILS_COPY_STATUS_ON_ORDER, + "+acn" => [{"record" => $rec->imported_as}, + {"owning_lib" => $item->owning_lib}] + }, + { + "join" => "acn", + "flesh" => 1, + "flesh_fields" => { + "acp" => ["call_number"] + }, + "order_by" => { "acp" => "create_date" } + } + ]; + } + # don't overlay the same copy twice + $query->[0]{"+acp"}{"id"} = {"not in" => \@overlaid_copy_ids} if @overlaid_copy_ids; + if ($copy = $e->search_asset_copy($query)->[0]) { + push(@overlaid_copy_ids, $copy->id); + } } if ($copy) { # we found a copy to overlay diff --git a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql index b22677cce7..732be40a6d 100644 --- a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql +++ b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql @@ -1862,6 +1862,76 @@ CREATE OR REPLACE FUNCTION vandelay.auto_overlay_bib_queue ( queue_id BIGINT ) R SELECT * FROM vandelay.auto_overlay_bib_queue( $1, NULL ); $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION vandelay.auto_overlay_org_unit_copies ( import_id BIGINT, merge_profile_id INT, lwm_ratio_value_p NUMERIC ) RETURNS BOOL AS $$ +DECLARE + eg_id BIGINT; + match_count INT; + rec vandelay.bib_match%ROWTYPE; + v_owning_lib INT; + scope_org INT; + scope_orgs INT[]; + copy_count INT := 0; + max_copy_count INT := 0; +BEGIN + + PERFORM * FROM vandelay.queued_bib_record WHERE import_time IS NOT NULL AND id = import_id; + + IF FOUND THEN + -- RAISE NOTICE 'already imported, cannot auto-overlay' + RETURN FALSE; + END IF; + + -- Gather all the owning libs for our import items. + -- These are our initial scope_orgs. + SELECT ARRAY_AGG(DISTINCT owning_lib) INTO scope_orgs + FROM vandelay.import_item + WHERE queued_record = import_id; + + WHILE CARDINALITY(scope_orgs) > 0 LOOP + FOR scope_org IN SELECT * FROM UNNEST(scope_orgs) LOOP + -- For each match, get a count of all copies at descendants of our scope org. + FOR rec IN SELECT * FROM vandelay.bib_match AS vbm + WHERE queued_record = import_id + ORDER BY vbm.eg_record DESC + LOOP + SELECT COUNT(acp.id) INTO copy_count + FROM asset.copy AS acp + INNER JOIN asset.call_number AS acn + ON acp.call_number = acn.id + WHERE acn.owning_lib IN (SELECT id FROM + actor.org_unit_descendants(scope_org)) + AND acn.record = rec.eg_record + AND acp.deleted = FALSE; + IF copy_count > max_copy_count THEN + max_copy_count := copy_count; + eg_id := rec.eg_record; + END IF; + END LOOP; + END LOOP; + + -- If no matching bibs had holdings, gather our next set of orgs to check, and iterate. + IF max_copy_count = 0 THEN + SELECT ARRAY_AGG(DISTINCT parent_ou) INTO scope_orgs + FROM actor.org_unit + WHERE id IN (SELECT * FROM UNNEST(scope_orgs)) + AND parent_ou IS NOT NULL; + END IF; + END LOOP; + + IF eg_id IS NULL THEN + -- Could not determine best match via copy count + -- fall back to default best match + IF (SELECT * FROM vandelay.auto_overlay_bib_record_with_best( import_id, merge_profile_id, lwm_ratio_value_p )) THEN + RETURN TRUE; + ELSE + RETURN FALSE; + END IF; + END IF; + + RETURN vandelay.overlay_bib_record( import_id, eg_id, merge_profile_id ); +END; +$$ LANGUAGE PLPGSQL; + CREATE OR REPLACE FUNCTION vandelay.ingest_bib_marc ( ) RETURNS TRIGGER AS $$ DECLARE value TEXT; diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index fb8f0b5ad6..4bf6099e8b 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -1925,7 +1925,11 @@ INSERT INTO permission.perm_list ( id, code, description ) VALUES ( 614, 'REFRESH_CAROUSEL', oils_i18n_gettext(614, 'Allow a user to refresh carousels', 'ppl', 'description')), ( 615, 'ADMIN_REMOTEAUTH', oils_i18n_gettext( 615, - 'Administer remote patron authentication', 'ppl', 'description' )) + 'Administer remote patron authentication', 'ppl', 'description' )), + ( 616, 'IMPORT_USE_ORG_UNIT_COPIES', oils_i18n_gettext( 616, + 'Allows users to import records based on the number of org unit copies attached to a record', 'ppl', 'description' )), + ( 617, 'IMPORT_ON_ORDER_CAT_COPY', oils_i18n_gettext( 617, + 'Allows users to import copies based on the on-order items attached to a record', 'ppl', 'description' )) ; diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay.auto_overlay_org_unit_copies.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay.auto_overlay_org_unit_copies.sql new file mode 100644 index 0000000000..20ec31d5c1 --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.vandelay.auto_overlay_org_unit_copies.sql @@ -0,0 +1,79 @@ +BEGIN; + +INSERT INTO permission.perm_list ( id, code, description ) VALUES + ( 616, 'IMPORT_USE_ORG_UNIT_COPIES', oils_i18n_gettext( 616, + 'Allows users to import records based on the number of org unit copies attached to a record', 'ppl', 'description' )), + ( 617, 'IMPORT_ON_ORDER_CAT_COPY', oils_i18n_gettext( 617, + 'Allows users to import copies based on the on-order items attached to a record', 'ppl', 'description' )); + +CREATE OR REPLACE FUNCTION vandelay.auto_overlay_org_unit_copies ( import_id BIGINT, merge_profile_id INT, lwm_ratio_value_p NUMERIC ) RETURNS BOOL AS $$ +DECLARE + eg_id BIGINT; + match_count INT; + rec vandelay.bib_match%ROWTYPE; + v_owning_lib INT; + scope_org INT; + scope_orgs INT[]; + copy_count INT := 0; + max_copy_count INT := 0; +BEGIN + + PERFORM * FROM vandelay.queued_bib_record WHERE import_time IS NOT NULL AND id = import_id; + + IF FOUND THEN + -- RAISE NOTICE 'already imported, cannot auto-overlay' + RETURN FALSE; + END IF; + + -- Gather all the owning libs for our import items. + -- These are our initial scope_orgs. + SELECT ARRAY_AGG(DISTINCT owning_lib) INTO scope_orgs + FROM vandelay.import_item + WHERE queued_record = import_id; + + WHILE CARDINALITY(scope_orgs) > 0 LOOP + FOR scope_org IN SELECT * FROM UNNEST(scope_orgs) LOOP + -- For each match, get a count of all copies at descendants of our scope org. + FOR rec IN SELECT * FROM vandelay.bib_match AS vbm + WHERE queued_record = import_id + ORDER BY vbm.eg_record DESC + LOOP + SELECT COUNT(acp.id) INTO copy_count + FROM asset.copy AS acp + INNER JOIN asset.call_number AS acn + ON acp.call_number = acn.id + WHERE acn.owning_lib IN (SELECT id FROM + actor.org_unit_descendants(scope_org)) + AND acn.record = rec.eg_record + AND acp.deleted = FALSE; + IF copy_count > max_copy_count THEN + max_copy_count := copy_count; + eg_id := rec.eg_record; + END IF; + END LOOP; + END LOOP; + + -- If no matching bibs had holdings, gather our next set of orgs to check, and iterate. + IF max_copy_count = 0 THEN + SELECT ARRAY_AGG(DISTINCT parent_ou) INTO scope_orgs + FROM actor.org_unit + WHERE id IN (SELECT * FROM UNNEST(scope_orgs)) + AND parent_ou IS NOT NULL; + END IF; + END LOOP; + + IF eg_id IS NULL THEN + -- Could not determine best match via copy count + -- fall back to default best match + IF (SELECT * FROM vandelay.auto_overlay_bib_record_with_best( import_id, merge_profile_id, lwm_ratio_value_p )) THEN + RETURN TRUE; + ELSE + RETURN FALSE; + END IF; + END IF; + + RETURN vandelay.overlay_bib_record( import_id, eg_id, merge_profile_id ); +END; +$$ LANGUAGE PLPGSQL; + +COMMIT; diff --git a/Open-ILS/src/templates/vandelay/inc/queue.tt2 b/Open-ILS/src/templates/vandelay/inc/queue.tt2 index b2c8655456..c4357f24c7 100644 --- a/Open-ILS/src/templates/vandelay/inc/queue.tt2 +++ b/Open-ILS/src/templates/vandelay/inc/queue.tt2 @@ -261,6 +261,17 @@ [% l('Auto-overlay In-process Acquisition Copies') %] + + [% l('Auto-overlay On-order Cataloguing Copies') %] + + + + + [% l('Use Org Unit Matching in Copy to Determine Best Match') %] + + + + diff --git a/Open-ILS/src/templates/vandelay/inc/upload.tt2 b/Open-ILS/src/templates/vandelay/inc/upload.tt2 index 4db970d7b4..1337304a4c 100644 --- a/Open-ILS/src/templates/vandelay/inc/upload.tt2 +++ b/Open-ILS/src/templates/vandelay/inc/upload.tt2 @@ -110,6 +110,16 @@ [% l('Auto-overlay In-process Acquisitions Copies') %] + + [% l('Auto-overlay On-order Cataloguing Copies') %] + + + + [% l('Use Org Unit Matching in Copy to Determine Best Match') %] + + + + diff --git a/Open-ILS/web/js/ui/default/vandelay/vandelay.js b/Open-ILS/web/js/ui/default/vandelay/vandelay.js index ab03a20954..d223eea8c2 100644 --- a/Open-ILS/web/js/ui/default/vandelay/vandelay.js +++ b/Open-ILS/web/js/ui/default/vandelay/vandelay.js @@ -1350,6 +1350,8 @@ function vlHandleQueueItemsAction(action) { vlUploadQueueAutoOverlayBestMatch.attr('value', vlUploadQueueAutoOverlayBestMatch2.attr('value')); vlUploadQueueAutoOverlayBestMatchRatio.attr('value', vlUploadQueueAutoOverlayBestMatchRatio2.attr('value')); vlUploadQueueAutoOverlayInprocessAcqCopies.attr('value', vlUploadQueueAutoOverlayInprocessAcqCopies2.attr('value')); + vlUploadQueueAutoOverlayOnorderCatCopies.attr('value', vlUploadQueueAutoOverlayOnorderCatCopies2.attr('value')); + vlUploadQueueAutoOverlayOrgUnitCopies.attr('value', vlUploadQueueAutoOverlayOrgUnitCopies2.attr('value')); // attr('value') and various other incantations won't let me set // the value on the checkedmultiselect, so we temporarily swap @@ -1382,6 +1384,10 @@ function vlHandleQueueItemsAction(action) { vlUploadQueueAutoOverlayBestMatchRatio2.attr('value', '0.0'); vlUploadQueueAutoOverlayInprocessAcqCopies.attr('value', false); vlUploadQueueAutoOverlayInprocessAcqCopies2.attr('value', false); + vlUploadQueueAutoOverlayOnorderCatCopies.attr('value', false); + vlUploadQueueAutoOverlayOnorderCatCopies2.attr('value', false); + vlUploadQueueAutoOverlayOrgUnitCopies.attr('value', false); + vlUploadQueueAutoOverlayOrgUnitCopies2.attr('value', false); // and... swap them back vlUploadTrashGroups2 = vlUploadTrashGroups; @@ -1471,6 +1477,17 @@ function vlImportRecordQueue(type, queueId, recList, onload, sessionKey) { vlUploadQueueAutoOverlayInprocessAcqCopies.checked = false; } + if(vlUploadQueueAutoOverlayOnorderCatCopies.checked) { + options.opp_oo_cat_copy_overlay = true; //"opp" for opportunistic "oo" for on order + vlUploadQueueAutoOverlayOnorderCatCopies.checked = false; + } + + if(vlUploadQueueAutoOverlayOrgUnitCopies.checked) { + options.auto_overlay_org_unit_copies = true; + vlUploadQueueAutoOverlayOrgUnitCopies.checked = false; + options.match_quality_ratio = vlUploadQueueAutoOverlayBestMatchRatio.attr('value'); + } + var profile = vlUploadMergeProfile.attr('value'); if(profile != null && profile != '') { options.merge_profile = profile; @@ -1542,7 +1559,8 @@ function batchUpload() { vlUploadQueueImportNoMatch.checked || vlUploadQueueAutoOverlayExact.checked || vlUploadQueueAutoOverlay1Match.checked || - vlUploadQueueAutoOverlayBestMatch.checked ) { + vlUploadQueueAutoOverlayBestMatch.checked || + vlUploadQueueAutoOverlayOrgUnitCopies.checked ) { vlImportRecordQueue( currentType, diff --git a/docs/RELEASE_NOTES_NEXT/Cataloging/copy-import-options.adoc b/docs/RELEASE_NOTES_NEXT/Cataloging/copy-import-options.adoc new file mode 100644 index 0000000000..d20e769431 --- /dev/null +++ b/docs/RELEASE_NOTES_NEXT/Cataloging/copy-import-options.adoc @@ -0,0 +1,33 @@ +New Options for Importing Copies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Two new options for importing holdings have been added to MARC Batch +Import/Export: + +. **Auto-overlay On-order Cataloguing Copies**: This is similar to + "Auto-overlay In-process Acquisitions Copies," but for copies that were not + created from an acquisitions workflow. Holdings information in the incoming + record will be used to overlay any existing On Order copies for the matching + record which belong to the owning library defined in the Holdings Import + Profile. The Holdings Import Profile is also used to match incoming to + existing copies, if possible; otherwise, On Order copies are overlaid in the + order they were created. The call number will also be overlaid if the + incoming record provides one. +. **Use Org Unit Matching in Copy to Determine Best Match**: When there are + multiple potential matching records, this feature allows the user to + automatically select the record which has the most copies at libraries near + the importing library in the org tree. That is, starting at the importing + library, it climbs the org tree, gradually expanding the scope at which it + checks for holdings on matching records; once holdings are found, the record + with the most holdings at that scope is selected for overlay. If there are + no matching records with holdings, then the default best match overlay is + attempted. + +Permissions ++++++++++++ + +Two new permissions control the use of these new features: + +* IMPORT_ON_ORDER_CAT_COPY +* IMPORT_USE_ORG_UNIT_COPIES + -- 2.43.2