From 42019cf16318874cf353ff0c1caab56069016a4c Mon Sep 17 00:00:00 2001 From: miker Date: Mon, 2 Mar 2009 19:24:15 +0000 Subject: [PATCH] initial addition of Conifer-sponsored electronic serials support. tests and docs to come git-svn-id: svn://svn.open-ils.org/ILS/trunk@12354 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/examples/fm_IDL.xml | 27 +++ .../perlmods/OpenILS/Application/Ingest.pm | 187 +++++++++++++++++- .../OpenILS/Application/Search/Biblio.pm | 66 +++++++ Open-ILS/src/sql/Pg/040.schema.asset.sql | 15 ++ Open-ILS/src/sql/Pg/210.schema.serials.sql | 10 - .../src/sql/Pg/300.schema.staged_search.sql | 34 ++++ 6 files changed, 327 insertions(+), 12 deletions(-) diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml index 76ac57aa8f..8383d8e43d 100644 --- a/Open-ILS/examples/fm_IDL.xml +++ b/Open-ILS/examples/fm_IDL.xml @@ -1486,6 +1486,33 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm index 87c9df9445..9e5a84e23d 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm @@ -129,9 +129,64 @@ sub rw_biblio_ingest_single_object { my $cstore = OpenSRF::AppSession->connect('open-ils.cstore'); my $xact = $cstore->request('open-ils.cstore.transaction.begin')->gather(1); + my $tmp; + + # update uri stuff ... + + # gather URI call numbers for this record + my $uri_cns = $u->{call_number} = $cstore->request( + 'open-ils.cstore.direct.asset.call_number.id_list.atomic' => { record => $bib->id, label => '##URI##' } + )->gather(1); + + # gather the maps for those call numbers + my $uri_maps = $u->{call_number} = $cstore->request( + 'open-ils.cstore.direct.asset.uri_call_number_map.id_list.atomic' => { call_number => $uri_cns } + )->gather(1); + + # delete the old maps + $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.delete' => $_ )->gather(1) for (@$uri_maps); + + # and delete the call numbers if there are no more URIs + if (!@{ $blob->{uri} }) { + $cstore->request( 'open-ils.cstore.direct.asset.call_number.delete' => $_ )->gather(1) for (@$uri_cns); + } + + # now, add CNs, URIs and maps + my %new_cns_by_owner; + my %new_uris_by_owner; + for my $u ( @{ $blob->{uri} } ) { + + my $owner = $u->{call_number}->owning_lib; + + if ($u->{call_number}->isnew) { + if ($new_cns_by_owner{$owner}) { + $u->{call_number} = $new_cns_by_owner{$owner}; + } else { + $u->{call_number} = $new_cns_by_owner{$owner} = $cstore->request( + 'open-ils.cstore.direct.asset.call_number.create' => $u->{call_number} + )->gather(1); + } + } + + if ($u->{uri}->isnew) { + if ($new_uris_by_owner{$owner}) { + $u->{uri} = $new_uris_by_owner{$owner}; + } else { + $u->{uri} = $new_uris_by_owner{$owner} = $cstore->request( + 'open-ils.cstore.direct.asset.uri.create' => $u->{uri} + )->gather(1); + } + } + + my $umap = Fieldmapper::asset::uri_call_number_map->new; + $umap->uri($u->{uri}->id); + $umap->call_number($u->{call_number}->id); + + $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.create' => $umap )->gather(1) if (!$tmp); + } # update full_rec stuff ... - my $tmp = $cstore->request( + $tmp = $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.id_list.atomic', { record => $bib->id } )->gather(1); @@ -346,6 +401,7 @@ sub ro_biblio_ingest_single_object { my $document = $parser->parse_string($xml); + my @uris = $self->method_lookup("open-ils.ingest.856_uri.object")->run($bib); my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document); my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document); my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml); @@ -355,7 +411,7 @@ sub ro_biblio_ingest_single_object { $_->record($bib->id) for (@mfr); $rd->record($bib->id) if ($rd); - return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd }; + return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd, uri => \@uris }; } __PACKAGE__->register_method( api_name => "open-ils.ingest.full.biblio.object.readonly", @@ -1019,6 +1075,133 @@ __PACKAGE__->register_method( stream => 1, ); + +# -------------------------------------------------------------------------------- +# URI extraction + +package OpenILS::Application::Ingest::Biblio::URI; +use base qw/OpenILS::Application::Ingest/; +use Unicode::Normalize; +use OpenSRF::EX qw/:try/; + + +sub _extract_856_uris { + + my $recid = shift; + my $marcxml = shift; + my @objects; + + my @nodes = $marcxml->findnodes('//*[local-name()="datafield" and @tag="856" and (@ind1="4" or @ind1="1") and (@ind2="0" or @ind2="1")]'); + + my $cstore = OpenSRF::AppSession->connect('open-ils.cstore'); + + for my $node (@nodes) { + # first, is there a URI? + my $href = $node->findvalue('[local-name()="subfield" and @code="u"]/text()'); + next unless ($href); + + # now, find the best possible label + my $label = $node->findvalue('[local-name()="subfield" and @code="y"]/text()'); + $label ||= $node->findvalue('[local-name()="subfield" and @code="3"]/text()'); + $label ||= $href; + + # look for use info + my $use = $node->findvalue('[local-name()="subfield" and @code="z"]/text()'); + $use ||= $node->findvalue('[local-name()="subfield" and @code="2"]/text()'); + $use ||= $node->findvalue('[local-name()="subfield" and @code="n"]/text()'); + + # moving on to the URI owner + my $owner = $node->findvalue('[local-name()="subfield" and @code="w"]/text()'); + $owner ||= $node->findvalue('[local-name()="subfield" and @code="n"]/text()'); + $owner ||= $node->findvalue('[local-name()="subfield" and @code="9"]/text()'); # Evergreen special sauce + + $owner =~ s/^.*?\((\w+)\).*$/$1/o; # unwrap first paren-enclosed string and then ... + + # no owner? skip it :( + next unless ($owner); + + my $org = $cstore + ->request( 'open-ils.cstore.direct.actor.org_unit.search' => { shortname => $owner} ) + ->gather(1); + + next unless ($org); + + # now we can construct the uri object + my $uri = $cstore + ->request( 'open-ils.cstore.direct.asset.uri.search' => { label => $label, href => $href, use => $use, active => 't' } ) + ->gather(1); + + if (!$uri) { + $uri = Fieldmapper::asset::uri->new; + $uri->isnew( 1 ); + $uri->label($label); + $uri->href($href); + $uri->use($use); + } + + # see if we need to create a call number + my $cn = $cstore + ->request( 'open-ils.cstore.direct.asset.call_number.search' => { owner => $org->id, record => $recid, label => '##URI##' } ) + ->gather(1); + + if (!$cn) { + $cn = Fieldmapper::asset::call_number->new; + $cn->isnew( 1 ); + $cn->owner( $org->id ); + $cn->record( $recid ); + $cn->label( '##URI##' ); + } + + push @objects, { uri => $uri, call_number => $cn }; + } + + $log->debug("Returning ".scalar(@objects)." URI nodes for record $recid"); + return @objects; +} + +sub get_uris_record { + my $self = shift; + my $client = shift; + my $rec = shift; + + OpenILS::Application::Ingest->post_init(); + my $r = OpenSRF::AppSession + ->create('open-ils.cstore') + ->request( "open-ils.cstore.direct.biblio.record_entry.retrieve" => $rec ) + ->gather(1); + + return undef unless ($r and $r->marc); + + $client->respond($_) for (_extract_856_uris($r->id, $r->marc)); + return undef; +} +__PACKAGE__->register_method( + api_name => "open-ils.ingest.856_uri.record", + method => "get_uris_record", + api_level => 1, + argc => 1, + stream => 1, +); + +sub get_uris_object { + my $self = shift; + my $client = shift; + my $obj = shift; + + return undef unless ($obj and $obj->marc); + + $client->respond($_) for (_extract_856_uris($obj->id, $obj->marc)); + return undef; +} +__PACKAGE__->register_method( + api_name => "open-ils.ingest.856_uri.object", + method => "get_uris_object", + api_level => 1, + argc => 1, + stream => 1, +); + + # -------------------------------------------------------------------------------- # Fingerprinting diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm b/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm index 4ca56b5f7d..6d92d7912b 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Search/Biblio.pm @@ -237,6 +237,72 @@ sub biblio_id_to_copy { } +__PACKAGE__->register_method( + method => "biblio_id_to_uris", + api_name=> "open-ils.search.asset.uri.retrieve_by_bib", + argc => 2, + stream => 1, + signature => q# + @param BibID Which bib record contains the URIs + @param OrgID Where to look for URIs + @param OrgDepth Range adjustment for OrgID + @return A stream or list of 'auri' objects + # + +); +sub biblio_id_to_uris { + my( $self, $client, $bib, $org, $depth ) = @_; + die "Org ID required" unless defined($org); + die "Bib ID required" unless defined($bib); + + my @params; + push @params, $depth if (defined $depth); + + my $ids = $U->cstorereq( "open-ils.cstore.json_query.atomic", + { select => { auri => [ 'id' ] }, + from => { + acn => { + auricnm => { + field => 'call_number', + fkey => 'id', + join => { + auri => { + field => 'id', + fkey => 'uri', + filter => { active => 't' } + } + } + } + } + }, + where => { + '+acn' => { + record => $bib, + owning_lib => { + in => { + select => { aou => [ { column => 'id', transform => 'actor.org_unit_descendants', params => \@params, result_field => 'id' } ] }, + from => 'aou', + where => { id => $org }, + distinct=> 1 + } + } + } + }, + distinct=> 1, + } + ); + + my $uris = $U->cstorereq( + "open-ils.cstore.direct.asset.uri.search.atomic", + { id => [ map { (values %$_) } @$ids ] } + ); + + $client->respond($_) for (@$uris); + + return undef; +} + + __PACKAGE__->register_method( method => "copy_retrieve", api_name => "open-ils.search.asset.copy.retrieve",); diff --git a/Open-ILS/src/sql/Pg/040.schema.asset.sql b/Open-ILS/src/sql/Pg/040.schema.asset.sql index 9b078132c6..cea70c8130 100644 --- a/Open-ILS/src/sql/Pg/040.schema.asset.sql +++ b/Open-ILS/src/sql/Pg/040.schema.asset.sql @@ -133,6 +133,14 @@ CREATE TABLE asset.copy_note ( value TEXT NOT NULL ); +CREATE TABLE asset.uri ( + id SERIAL PRIMARY KEY, + href TEXT NOT NULL, + label TEXT, + use TEXT, + active BOOL NOT NULL DEFAULT TRUE +); + CREATE TABLE asset.call_number ( id bigserial PRIMARY KEY, creator BIGINT NOT NULL, @@ -152,6 +160,13 @@ CREATE INDEX asset_call_number_upper_label_id_owning_lib_idx ON asset.call_numbe CREATE UNIQUE INDEX asset_call_number_label_once_per_lib ON asset.call_number (record, owning_lib, label) WHERE deleted IS FALSE; CREATE RULE protect_cn_delete AS ON DELETE TO asset.call_number DO INSTEAD UPDATE asset.call_number SET deleted = TRUE WHERE OLD.id = asset.call_number.id; +CREATE TABLE asset.uri_call_number_map ( + id BIGSERIAL PRIMARY KEY, + uri INT NOT NULL REFERENCES asset.uri (id), + call_number INT NOT NULL REFERENCES asset.call_number (id), + CONSTRAINT uri_cn_once UNIQUE (uri,call_number) +); + CREATE TABLE asset.call_number_note ( id BIGSERIAL PRIMARY KEY, call_number BIGINT NOT NULL, diff --git a/Open-ILS/src/sql/Pg/210.schema.serials.sql b/Open-ILS/src/sql/Pg/210.schema.serials.sql index 9637dada77..03245f5a05 100644 --- a/Open-ILS/src/sql/Pg/210.schema.serials.sql +++ b/Open-ILS/src/sql/Pg/210.schema.serials.sql @@ -2,16 +2,6 @@ DROP SCHEMA serial CASCADE; -CREATE TABLE asset.uri ( - id SERIAL PRIMARY KEY, - href TEXT NOT NULL, - label TEXT, - use TEXT, - active BOOL NOT NULL DEFAULT TRUE -); - -ALTER TABLE asset.call_number ADD COLUMN uri INT REFERENCES asset.uri (id) DEFERRABLE INITIALLY DEFERRED; - BEGIN; CREATE SCHEMA serial; diff --git a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql index ca63fd69db..39e814388a 100644 --- a/Open-ILS/src/sql/Pg/300.schema.staged_search.sql +++ b/Open-ILS/src/sql/Pg/300.schema.staged_search.sql @@ -423,6 +423,40 @@ BEGIN CONTINUE; END IF; + PERFORM 1 + FROM asset.call_number cn + JOIN asset.uri_call_number_map map ON (map.call_number = cn.id) + JOIN asset.uri uri ON (map.uri = uri.id) + WHERE NOT cn.deleted + AND cn.label = '##URI##' + AND uri.active + AND cn.record IN ( SELECT * FROM search.explode_array( core_result.records ) ) + AND cn.owning_lib IN ( SELECT * FROM search.explode_array( search_org_list ) ) + LIMIT 1; + + IF FOUND THEN + -- RAISE NOTICE ' % have at least one URI ... ', core_result.records; + visible_count := visible_count + 1; + + current_res.id = core_result.id; + current_res.rel = core_result.rel; + + tmp_int := 1; + IF metarecord THEN + SELECT COUNT(DISTINCT s.source) INTO tmp_int FROM metabib.metarecord_source_map s WHERE s.metarecord = core_result.id; + END IF; + + IF tmp_int = 1 THEN + current_res.record = core_result.records[1]; + ELSE + current_res.record = NULL; + END IF; + + RETURN NEXT current_res; + + CONTINUE; + END IF; + IF param_statuses IS NOT NULL AND array_upper(param_statuses, 1) > 0 THEN PERFORM 1 -- 2.43.2