Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm

   1 package OpenILS::Application::Ingest;
   2 use OpenILS::Application;
   3 use base qw/OpenILS::Application/;
   4
   5 use Unicode::Normalize;
   6 use OpenSRF::EX qw/:try/;
   7
   8 use OpenSRF::AppSession;
   9 use OpenSRF::Utils::SettingsClient;
  10 use OpenSRF::Utils::Logger qw/:level/;
  11
  12 use OpenILS::Utils::ScriptRunner;
  13 use OpenILS::Utils::Fieldmapper;
  14 use OpenSRF::Utils::JSON;
  15
  16 use OpenILS::Utils::Fieldmapper;
  17
  18 use XML::LibXML;
  19 use XML::LibXSLT;
  20 use Time::HiRes qw(time);
  21
  22 our %supported_formats = (
  23         mods32  => {ns => 'http://www.loc.gov/mods/v3'},
  24         mods3   => {ns => 'http://www.loc.gov/mods/v3'},
  25         mods    => {ns => 'http://www.loc.gov/mods/'},
  26         marcxml => {ns => 'http://www.loc.gov/MARC21/slim'},
  27         srw_dc  => {ns => 'info:srw/schema/1/dc-schema'},
  28         oai_dc  => {ns => 'http://www.openarchives.org/OAI/2.0/oai_dc/'},
  29         rdf_dc  => {ns => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'},
  30         atom    => {ns => 'http://www.w3.org/2005/Atom'},
  31         rss091  => {ns => 'http://my.netscape.com/rdf/simple/0.9/'},
  32         rss092  => {ns => ''},
  33         rss093  => {ns => ''},
  34         rss094  => {ns => ''},
  35         rss10   => {ns => 'http://purl.org/rss/1.0/'},
  36         rss11   => {ns => 'http://purl.org/net/rss1.1#'},
  37         rss2    => {ns => ''},
  38 );
  39
  40
  41 my $log = 'OpenSRF::Utils::Logger';
  42
  43 my  $parser = XML::LibXML->new();
  44 my  $xslt = XML::LibXSLT->new();
  45
  46 my  $mods_sheet;
  47 my  $mads_sheet;
  48 my  $xpathset = {};
  49 sub initialize {}
  50 sub child_init {}
  51
  52 sub post_init {
  53
  54         unless (keys %$xpathset) {
  55                 $log->debug("Running post_init", DEBUG);
  56
  57                 my $xsldir = OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl');
  58
  59                 unless ($supported_formats{mods}{xslt}) {
  60                         $log->debug("Loading MODS XSLT", DEBUG);
  61                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS.xsl");
  62                         $supported_formats{mods}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  63                 }
  64
  65                 unless ($supported_formats{mods3}{xslt}) {
  66                         $log->debug("Loading MODS v3 XSLT", DEBUG);
  67                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS3.xsl");
  68                         $supported_formats{mods3}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  69                 }
  70
  71                 unless ($supported_formats{mods32}{xslt}) {
  72                         $log->debug("Loading MODS v32 XSLT", DEBUG);
  73                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS32.xsl");
  74                         $supported_formats{mods32}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  75                 }
  76
  77                 my $req = OpenSRF::AppSession
  78                                 ->create('open-ils.cstore')
  79
  80                                 # XXX testing new metabib field use for faceting
  81                                 #->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { id => { '!=' => undef } } )
  82                                 ->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { search_field => 't' } )
  83
  84                                 ->gather(1);
  85
  86                 if (ref $req and @$req) {
  87                         for my $f (@$req) {
  88                                 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
  89                                 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
  90                                 $xpathset->{ $f->field_class }->{ $f->name }->{format} = $f->format;
  91                                 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
  92                         }
  93                 }
  94         }
  95 }
  96
  97 sub entityize {
  98         my $stuff = shift;
  99         my $form = shift;
 100
 101         if ($form eq 'D') {
 102                 $stuff = NFD($stuff);
 103         } else {
 104                 $stuff = NFC($stuff);
 105         }
 106
 107         $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
 108         return $stuff;
 109 }
 110
 111 # --------------------------------------------------------------------------------
 112 # Biblio ingest
 113
 114 package OpenILS::Application::Ingest::Biblio;
 115 use base qw/OpenILS::Application::Ingest/;
 116 use Unicode::Normalize;
 117
 118 sub rw_biblio_ingest_single_object {
 119         my $self = shift;
 120         my $client = shift;
 121         my $bib = shift;
 122
 123         my ($blob) = $self->method_lookup("open-ils.ingest.full.biblio.object.readonly")->run($bib);
 124         return undef unless ($blob);
 125
 126         $bib->fingerprint( $blob->{fingerprint}->{fingerprint} );
 127         $bib->quality( $blob->{fingerprint}->{quality} );
 128
 129         my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
 130
 131         my $xact = $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 132     my $tmp;
 133
 134         # update uri stuff ...
 135
 136     # gather URI call numbers for this record
 137     my $uri_cns = $u->{call_number} = $cstore->request(
 138         'open-ils.cstore.direct.asset.call_number.id_list.atomic' => { record => $bib->id, label => '##URI##' }
 139     )->gather(1);
 140
 141     # gather the maps for those call numbers
 142     my $uri_maps = $u->{call_number} = $cstore->request(
 143         'open-ils.cstore.direct.asset.uri_call_number_map.id_list.atomic' => { call_number => $uri_cns }
 144     )->gather(1);
 145
 146     # delete the old maps
 147     $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.delete' => $_ )->gather(1) for (@$uri_maps);
 148
 149     # and delete the call numbers if there are no more URIs
 150     if (!@{ $blob->{uri} }) {
 151         $cstore->request( 'open-ils.cstore.direct.asset.call_number.delete' => $_ )->gather(1) for (@$uri_cns);
 152     }
 153
 154     # now, add CNs, URIs and maps
 155     my %new_cns_by_owner;
 156     my %new_uris_by_owner;
 157     for my $u ( @{ $blob->{uri} } ) {
 158
 159         my $owner = $u->{call_number}->owning_lib;
 160
 161         if ($u->{call_number}->isnew) {
 162             if ($new_cns_by_owner{$owner}) {
 163                 $u->{call_number} = $new_cns_by_owner{$owner};
 164             } else {
 165                 $u->{call_number}->clear_id;
 166                 $u->{call_number} = $new_cns_by_owner{$owner} = $cstore->request(
 167                     'open-ils.cstore.direct.asset.call_number.create' => $u->{call_number}
 168                 )->gather(1);
 169             }
 170         }
 171
 172         if ($u->{uri}->isnew) {
 173             if ($new_uris_by_owner{$owner}) {
 174                     $u->{uri} = $new_uris_by_owner{$owner};
 175             } else {
 176                     $u->{uri} = $new_uris_by_owner{$owner} = $cstore->request(
 177                     'open-ils.cstore.direct.asset.uri.create' => $u->{uri}
 178                 )->gather(1);
 179             }
 180         }
 181
 182         my $umap = Fieldmapper::asset::uri_call_number_map->new;
 183         $umap->uri($u->{uri}->id);
 184         $umap->call_number($u->{call_number}->id);
 185
 186             $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.create' => $umap )->gather(1) if (!$tmp);
 187     }
 188
 189         # update full_rec stuff ...
 190         $tmp = $cstore->request(
 191                 'open-ils.cstore.direct.metabib.full_rec.id_list.atomic',
 192                 { record => $bib->id }
 193         )->gather(1);
 194
 195         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.delete' => $_ )->gather(1) for (@$tmp);
 196         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.create' => $_ )->gather(1) for (@{ $blob->{full_rec} });
 197
 198         # update rec_descriptor stuff ...
 199         $tmp = $cstore->request(
 200                 'open-ils.cstore.direct.metabib.record_descriptor.id_list.atomic',
 201                 { record => $bib->id }
 202         )->gather(1);
 203
 204         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.delete' => $_ )->gather(1) for (@$tmp);
 205         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.create' => $blob->{descriptor} )->gather(1);
 206
 207         # deal with classed fields...
 208         for my $class ( qw/title author subject keyword series/ ) {
 209                 $tmp = $cstore->request(
 210                         "open-ils.cstore.direct.metabib.${class}_field_entry.id_list.atomic",
 211                         { source => $bib->id }
 212                 )->gather(1);
 213
 214                 $cstore->request( "open-ils.cstore.direct.metabib.${class}_field_entry.delete" => $_ )->gather(1) for (@$tmp);
 215         }
 216         for my $obj ( @{ $blob->{field_entries} } ) {
 217                 my $class = $obj->class_name;
 218                 $class =~ s/^Fieldmapper:://o;
 219                 $class =~ s/::/./go;
 220                 $cstore->request( "open-ils.cstore.direct.$class.create" => $obj )->gather(1);
 221         }
 222
 223         # update MR map ...
 224
 225         $tmp = $cstore->request(
 226                 'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 227                 { source => $bib->id }
 228         )->gather(1);
 229
 230         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.delete' => $_->id )->gather(1) for (@$tmp);
 231
 232         # get the old MRs
 233         my $old_mrs = $cstore->request(
 234                 'open-ils.cstore.direct.metabib.metarecord.search.atomic' => { id => [map { $_->metarecord } @$tmp] }
 235         )->gather(1) if (@$tmp);
 236
 237         $old_mrs = [] if (!ref($old_mrs));
 238
 239         my $mr;
 240         for my $m (@$old_mrs) {
 241                 if ($m->fingerprint eq $bib->fingerprint) {
 242                         $mr = $m;
 243                 } else {
 244                         my $others = $cstore->request(
 245                                 'open-ils.cstore.direct.metabib.metarecord_source_map.id_list.atomic' => { metarecord => $m->id }
 246                         )->gather(1);
 247
 248                         if (!@$others) {
 249                                 $cstore->request(
 250                                         'open-ils.cstore.direct.metabib.metarecord.delete' => $m->id
 251                                 )->gather(1);
 252                         }
 253
 254                         $m->isdeleted(1);
 255                 }
 256         }
 257
 258         my $holds;
 259         if (!$mr) {
 260                 # Get the matchin MR, if any.
 261                 $mr = $cstore->request(
 262                         'open-ils.cstore.direct.metabib.metarecord.search',
 263                         { fingerprint => $bib->fingerprint }
 264                 )->gather(1);
 265
 266                 $holds = $cstore->request(
 267                         'open-ils.cstore.direct.action.hold_request.search.atomic',
 268                         { hold_type => 'M', target => [ map { $_->id } grep { $_->isdeleted } @$old_mrs ] }
 269                 )->gather(1) if (@$old_mrs);
 270
 271                 if ($mr) {
 272                         for my $h (@$holds) {
 273                                 $h->target($mr);
 274                                 $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 275                                 $h->ischanged(1);
 276                         }
 277                 }
 278         }
 279
 280         if (!$mr) {
 281                 $mr = new Fieldmapper::metabib::metarecord;
 282                 $mr->fingerprint( $bib->fingerprint );
 283                 $mr->master_record( $bib->id );
 284                 $mr->id(
 285                         $cstore->request(
 286                                 "open-ils.cstore.direct.metabib.metarecord.create",
 287                                 $mr => { quiet => 'true' }
 288                         )->gather(1)
 289                 );
 290
 291                 for my $h (grep { !$_->ischanged } @$holds) {
 292                         $h->target($mr);
 293                         $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 294                 }
 295         } else {
 296                 my $mrm = $cstore->request(
 297                         'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 298                         { metarecord => $mr->id }
 299                 )->gather(1);
 300
 301                 if (@$mrm) {
 302                         my $best = $cstore->request(
 303                                 "open-ils.cstore.direct.biblio.record_entry.search",
 304                                 { id => [ map { $_->source } @$mrm ] },
 305                                 { 'select'      => { bre => [ qw/id quality/ ] },
 306                                 order_by        => { bre => "quality desc" },
 307                                 limit           => 1,
 308                                 }
 309                         )->gather(1);
 310
 311                         if ($best->quality > $bib->quality) {
 312                                 $mr->master_record($best->id);
 313                         } else {
 314                                 $mr->master_record($bib->id);
 315                         }
 316                 } else {
 317                         $mr->master_record($bib->id);
 318                 }
 319
 320                 $mr->clear_mods;
 321
 322                 $cstore->request( 'open-ils.cstore.direct.metabib.metarecord.update' => $mr )->gather(1);
 323         }
 324
 325         my $mrm = new Fieldmapper::metabib::metarecord_source_map;
 326         $mrm->source($bib->id);
 327         $mrm->metarecord($mr->id);
 328
 329         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.create' => $mrm )->gather(1);
 330         $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.update' => $bib )->gather(1);
 331
 332         $cstore->request( 'open-ils.cstore.transaction.commit' )->gather(1) || return undef;;
 333     $cstore->disconnect;
 334
 335         return $bib->id;
 336 }
 337 __PACKAGE__->register_method(
 338         api_name        => "open-ils.ingest.full.biblio.object",
 339         method          => "rw_biblio_ingest_single_object",
 340         api_level       => 1,
 341         argc            => 1,
 342 );
 343
 344 sub rw_biblio_ingest_single_record {
 345         my $self = shift;
 346         my $client = shift;
 347         my $rec = shift;
 348
 349         OpenILS::Application::Ingest->post_init();
 350         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 351         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 352
 353         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
 354
 355         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 356         $cstore->disconnect;
 357
 358         return undef unless ($r and @$r);
 359
 360         return ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($r))[0];
 361 }
 362 __PACKAGE__->register_method(
 363         api_name        => "open-ils.ingest.full.biblio.record",
 364         method          => "rw_biblio_ingest_single_record",
 365         api_level       => 1,
 366         argc            => 1,
 367 );
 368
 369 sub rw_biblio_ingest_record_list {
 370         my $self = shift;
 371         my $client = shift;
 372         my @rec = ref($_[0]) ? @{ $_[0] } : @_ ;
 373
 374         OpenILS::Application::Ingest->post_init();
 375         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 376         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 377
 378         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.search.atomic' => { id => $rec } )->gather(1);
 379
 380         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 381         $cstore->disconnect;
 382
 383         return undef unless ($r and @$r);
 384
 385         my $count = 0;
 386         $count += ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($_))[0] for (@$r);
 387
 388         return $count;
 389 }
 390 __PACKAGE__->register_method(
 391         api_name        => "open-ils.ingest.full.biblio.record_list",
 392         method          => "rw_biblio_ingest_record_list",
 393         api_level       => 1,
 394         argc            => 1,
 395 );
 396
 397 sub ro_biblio_ingest_single_object {
 398         my $self = shift;
 399         my $client = shift;
 400         my $bib = shift;
 401         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 402
 403         my $cn = $cstore->request( 'open-ils.cstore.direct.asset.call_number.search' => { id => { '!=' => undef }, { limit => 1 } } )->gather(1);
 404     my $max_cn = int($cn->id) + 1000;
 405
 406         my $document = $parser->parse_string($xml);
 407
 408         my @uris = $self->method_lookup("open-ils.ingest.856_uri.object")->run($bib, $max_cn);
 409         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 410         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 411         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 412         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 413
 414         $_->source($bib->id) for (@mXfe);
 415         $_->record($bib->id) for (@mfr);
 416         $rd->record($bib->id) if ($rd);
 417
 418         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd, uri => \@uris };
 419 }
 420 __PACKAGE__->register_method(
 421         api_name        => "open-ils.ingest.full.biblio.object.readonly",
 422         method          => "ro_biblio_ingest_single_object",
 423         api_level       => 1,
 424         argc            => 1,
 425 );
 426
 427 sub ro_biblio_ingest_single_xml {
 428         my $self = shift;
 429         my $client = shift;
 430         my $xml = OpenILS::Application::Ingest::entityize(shift);
 431
 432         my $document = $parser->parse_string($xml);
 433
 434         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 435         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 436         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 437         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 438
 439         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd };
 440 }
 441 __PACKAGE__->register_method(
 442         api_name        => "open-ils.ingest.full.biblio.xml.readonly",
 443         method          => "ro_biblio_ingest_single_xml",
 444         api_level       => 1,
 445         argc            => 1,
 446 );
 447
 448 sub ro_biblio_ingest_single_record {
 449         my $self = shift;
 450         my $client = shift;
 451         my $rec = shift;
 452
 453         OpenILS::Application::Ingest->post_init();
 454         my $r = OpenSRF::AppSession
 455                         ->create('open-ils.cstore')
 456                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 457                         ->gather(1);
 458
 459         return undef unless ($r and @$r);
 460
 461         my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($r->marc);
 462
 463         $_->source($rec) for (@{$res->{field_entries}});
 464         $_->record($rec) for (@{$res->{full_rec}});
 465         $res->{descriptor}->record($rec);
 466
 467         return $res;
 468 }
 469 __PACKAGE__->register_method(
 470         api_name        => "open-ils.ingest.full.biblio.record.readonly",
 471         method          => "ro_biblio_ingest_single_record",
 472         api_level       => 1,
 473         argc            => 1,
 474 );
 475
 476 sub ro_biblio_ingest_stream_record {
 477         my $self = shift;
 478         my $client = shift;
 479
 480         OpenILS::Application::Ingest->post_init();
 481
 482         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 483
 484         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 485
 486                 my $rec = $resp->content;
 487                 last unless (defined $rec);
 488
 489                 $log->debug("Running open-ils.ingest.full.biblio.record.readonly ...");
 490                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.record.readonly")->run($rec);
 491
 492                 $_->source($rec) for (@{$res->{field_entries}});
 493                 $_->record($rec) for (@{$res->{full_rec}});
 494
 495                 $client->respond( $res );
 496         }
 497
 498         return undef;
 499 }
 500 __PACKAGE__->register_method(
 501         api_name        => "open-ils.ingest.full.biblio.record_stream.readonly",
 502         method          => "ro_biblio_ingest_stream_record",
 503         api_level       => 1,
 504         stream          => 1,
 505 );
 506
 507 sub ro_biblio_ingest_stream_xml {
 508         my $self = shift;
 509         my $client = shift;
 510
 511         OpenILS::Application::Ingest->post_init();
 512
 513         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 514
 515         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 516
 517                 my $xml = $resp->content;
 518                 last unless (defined $xml);
 519
 520                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 521                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($xml);
 522
 523                 $client->respond( $res );
 524         }
 525
 526         return undef;
 527 }
 528 __PACKAGE__->register_method(
 529         api_name        => "open-ils.ingest.full.biblio.xml_stream.readonly",
 530         method          => "ro_biblio_ingest_stream_xml",
 531         api_level       => 1,
 532         stream          => 1,
 533 );
 534
 535 sub rw_biblio_ingest_stream_import {
 536         my $self = shift;
 537         my $client = shift;
 538
 539         OpenILS::Application::Ingest->post_init();
 540
 541         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 542
 543         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 544
 545                 my $bib = $resp->content;
 546                 last unless (defined $bib);
 547
 548                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 549                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($bib->marc);
 550
 551                 $_->source($bib->id) for (@{$res->{field_entries}});
 552                 $_->record($bib->id) for (@{$res->{full_rec}});
 553
 554                 $client->respond( $res );
 555         }
 556
 557         return undef;
 558 }
 559 __PACKAGE__->register_method(
 560         api_name        => "open-ils.ingest.full.biblio.bib_stream.import",
 561         method          => "rw_biblio_ingest_stream_import",
 562         api_level       => 1,
 563         stream          => 1,
 564 );
 565
 566
 567 # --------------------------------------------------------------------------------
 568 # Authority ingest
 569
 570 package OpenILS::Application::Ingest::Authority;
 571 use base qw/OpenILS::Application::Ingest/;
 572 use Unicode::Normalize;
 573
 574 sub ro_authority_ingest_single_object {
 575         my $self = shift;
 576         my $client = shift;
 577         my $bib = shift;
 578         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 579
 580         my $document = $parser->parse_string($xml);
 581
 582         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 583
 584         $_->record($bib->id) for (@mfr);
 585
 586         return { full_rec => \@mfr };
 587 }
 588 __PACKAGE__->register_method(
 589         api_name        => "open-ils.ingest.full.authority.object.readonly",
 590         method          => "ro_authority_ingest_single_object",
 591         api_level       => 1,
 592         argc            => 1,
 593 );
 594
 595 sub ro_authority_ingest_single_xml {
 596         my $self = shift;
 597         my $client = shift;
 598         my $xml = OpenILS::Application::Ingest::entityize(shift);
 599
 600         my $document = $parser->parse_string($xml);
 601
 602         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 603
 604         return { full_rec => \@mfr };
 605 }
 606 __PACKAGE__->register_method(
 607         api_name        => "open-ils.ingest.full.authority.xml.readonly",
 608         method          => "ro_authority_ingest_single_xml",
 609         api_level       => 1,
 610         argc            => 1,
 611 );
 612
 613 sub ro_authority_ingest_single_record {
 614         my $self = shift;
 615         my $client = shift;
 616         my $rec = shift;
 617
 618         OpenILS::Application::Ingest->post_init();
 619         my $r = OpenSRF::AppSession
 620                         ->create('open-ils.cstore')
 621                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 622                         ->gather(1);
 623
 624         return undef unless ($r and @$r);
 625
 626         my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($r->marc);
 627
 628         $_->record($rec) for (@{$res->{full_rec}});
 629         $res->{descriptor}->record($rec);
 630
 631         return $res;
 632 }
 633 __PACKAGE__->register_method(
 634         api_name        => "open-ils.ingest.full.authority.record.readonly",
 635         method          => "ro_authority_ingest_single_record",
 636         api_level       => 1,
 637         argc            => 1,
 638 );
 639
 640 sub ro_authority_ingest_stream_record {
 641         my $self = shift;
 642         my $client = shift;
 643
 644         OpenILS::Application::Ingest->post_init();
 645
 646         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 647
 648         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 649
 650                 my $rec = $resp->content;
 651                 last unless (defined $rec);
 652
 653                 $log->debug("Running open-ils.ingest.full.authority.record.readonly ...");
 654                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.record.readonly")->run($rec);
 655
 656                 $_->record($rec) for (@{$res->{full_rec}});
 657
 658                 $client->respond( $res );
 659         }
 660
 661         return undef;
 662 }
 663 __PACKAGE__->register_method(
 664         api_name        => "open-ils.ingest.full.authority.record_stream.readonly",
 665         method          => "ro_authority_ingest_stream_record",
 666         api_level       => 1,
 667         stream          => 1,
 668 );
 669
 670 sub ro_authority_ingest_stream_xml {
 671         my $self = shift;
 672         my $client = shift;
 673
 674         OpenILS::Application::Ingest->post_init();
 675
 676         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 677
 678         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 679
 680                 my $xml = $resp->content;
 681                 last unless (defined $xml);
 682
 683                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 684                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($xml);
 685
 686                 $client->respond( $res );
 687         }
 688
 689         return undef;
 690 }
 691 __PACKAGE__->register_method(
 692         api_name        => "open-ils.ingest.full.authority.xml_stream.readonly",
 693         method          => "ro_authority_ingest_stream_xml",
 694         api_level       => 1,
 695         stream          => 1,
 696 );
 697
 698 sub rw_authority_ingest_stream_import {
 699         my $self = shift;
 700         my $client = shift;
 701
 702         OpenILS::Application::Ingest->post_init();
 703
 704         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 705
 706         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 707
 708                 my $bib = $resp->content;
 709                 last unless (defined $bib);
 710
 711                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 712                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($bib->marc);
 713
 714                 $_->record($bib->id) for (@{$res->{full_rec}});
 715
 716                 $client->respond( $res );
 717         }
 718
 719         return undef;
 720 }
 721 __PACKAGE__->register_method(
 722         api_name        => "open-ils.ingest.full.authority.bib_stream.import",
 723         method          => "rw_authority_ingest_stream_import",
 724         api_level       => 1,
 725         stream          => 1,
 726 );
 727
 728
 729 # --------------------------------------------------------------------------------
 730 # MARC index extraction
 731
 732 package OpenILS::Application::Ingest::XPATH;
 733 use base qw/OpenILS::Application::Ingest/;
 734 use Unicode::Normalize;
 735
 736 # give this an XML documentElement and an XPATH expression
 737 sub xpath_to_string {
 738         my $xml = shift;
 739         my $xpath = shift;
 740         my $ns_uri = shift;
 741         my $ns_prefix = shift;
 742         my $unique = shift;
 743
 744         $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
 745
 746         my $string = "";
 747
 748         # grab the set of matching nodes
 749         my @nodes = $xml->findnodes( $xpath );
 750         for my $value (@nodes) {
 751
 752                 # grab all children of the node
 753                 my @children = $value->childNodes();
 754                 for my $child (@children) {
 755
 756                         # add the childs content to the growing buffer
 757                         my $content = quotemeta($child->textContent);
 758                         next if ($unique && $string =~ /$content/);  # uniquify the values
 759                         $string .= $child->textContent . " ";
 760                 }
 761                 if( ! @children ) {
 762                         $string .= $value->textContent . " ";
 763                 }
 764         }
 765
 766     $string =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 767     $string =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 768
 769         return NFD($string);
 770 }
 771
 772 sub class_index_string_xml {
 773         my $self = shift;
 774         my $client = shift;
 775         my $xml = shift;
 776         my @classes = @_;
 777
 778         OpenILS::Application::Ingest->post_init();
 779         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
 780
 781         my %transform_cache;
 782
 783         for my $class (@classes) {
 784                 my $class_constructor = "Fieldmapper::metabib::${class}_field_entry";
 785                 for my $type ( keys %{ $xpathset->{$class} } ) {
 786
 787                         my $def = $xpathset->{$class}->{$type};
 788                         my $sf = $OpenILS::Application::Ingest::supported_formats{$def->{format}};
 789
 790                         my $document = $xml;
 791
 792                         if ($sf->{xslt}) {
 793                                 $document = $transform_cache{$def->{format}} || $sf->{xslt}->transform($xml);
 794                                 $transform_cache{$def->{format}} = $document;
 795                         }
 796
 797                         my $value =  xpath_to_string(
 798                                         $document->documentElement      => $def->{xpath},
 799                                         $sf->{ns}                       => $def->{format},
 800                                         1
 801                         );
 802
 803                         next unless $value;
 804
 805                         $value = NFD($value);
 806                         $value =~ s/\pM+//sgo;
 807                         $value =~ s/\pC+//sgo;
 808                         $value =~ s/\W+$//sgo;
 809
 810                         $value =~ s/\b\.+\b//sgo;
 811                         $value = lc($value);
 812
 813                         my $fm = $class_constructor->new;
 814                         $fm->value( $value );
 815                         $fm->field( $xpathset->{$class}->{$type}->{id} );
 816                         $client->respond($fm);
 817                 }
 818         }
 819         return undef;
 820 }
 821 __PACKAGE__->register_method(
 822         api_name        => "open-ils.ingest.field_entry.class.xml",
 823         method          => "class_index_string_xml",
 824         api_level       => 1,
 825         argc            => 2,
 826         stream          => 1,
 827 );
 828
 829 sub class_index_string_record {
 830         my $self = shift;
 831         my $client = shift;
 832         my $rec = shift;
 833         my @classes = shift;
 834
 835         OpenILS::Application::Ingest->post_init();
 836         my $r = OpenSRF::AppSession
 837                         ->create('open-ils.cstore')
 838                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 839                         ->gather(1);
 840
 841         return undef unless ($r and @$r);
 842
 843         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, @classes)) {
 844                 $fm->source($rec);
 845                 $client->respond($fm);
 846         }
 847         return undef;
 848 }
 849 __PACKAGE__->register_method(
 850         api_name        => "open-ils.ingest.field_entry.class.record",
 851         method          => "class_index_string_record",
 852         api_level       => 1,
 853         argc            => 2,
 854         stream          => 1,
 855 );
 856
 857 sub all_index_string_xml {
 858         my $self = shift;
 859         my $client = shift;
 860         my $xml = shift;
 861
 862         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($xml, keys(%$xpathset))) {
 863                 $client->respond($fm);
 864         }
 865         return undef;
 866 }
 867 __PACKAGE__->register_method(
 868         api_name        => "open-ils.ingest.extract.field_entry.all.xml",
 869         method          => "all_index_string_xml",
 870         api_level       => 1,
 871         argc            => 1,
 872         stream          => 1,
 873 );
 874
 875 sub all_index_string_record {
 876         my $self = shift;
 877         my $client = shift;
 878         my $rec = shift;
 879
 880         OpenILS::Application::Ingest->post_init();
 881         my $r = OpenSRF::AppSession
 882                         ->create('open-ils.cstore')
 883                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 884                         ->gather(1);
 885
 886         return undef unless ($r and @$r);
 887
 888         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, keys(%$xpathset))) {
 889                 $fm->source($rec);
 890                 $client->respond($fm);
 891         }
 892         return undef;
 893 }
 894 __PACKAGE__->register_method(
 895         api_name        => "open-ils.ingest.extract.field_entry.all.record",
 896         method          => "all_index_string_record",
 897         api_level       => 1,
 898         argc            => 1,
 899         stream          => 1,
 900 );
 901
 902 # --------------------------------------------------------------------------------
 903 # Flat MARC
 904
 905 package OpenILS::Application::Ingest::FlatMARC;
 906 use base qw/OpenILS::Application::Ingest/;
 907 use Unicode::Normalize;
 908
 909
 910 sub _marcxml_to_full_rows {
 911
 912         my $marcxml = shift;
 913         my $xmltype = shift || 'metabib';
 914
 915         my $type = "Fieldmapper::${xmltype}::full_rec";
 916
 917         my @ns_list;
 918
 919         my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
 920
 921         for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
 922                 next unless $tagline;
 923
 924                 my $ns = $type->new;
 925
 926                 $ns->tag( 'LDR' );
 927                 my $val = $tagline->textContent;
 928                 $val = NFD($val);
 929                 $val =~ s/\pM+//sgo;
 930                 $val =~ s/\pC+//sgo;
 931                 $val =~ s/\W+$//sgo;
 932                 $ns->value( $val );
 933
 934                 push @ns_list, $ns;
 935         }
 936
 937         for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
 938                 next unless $tagline;
 939
 940                 my $ns = $type->new;
 941
 942                 $ns->tag( $tagline->getAttribute( "tag" ) );
 943                 my $val = $tagline->textContent;
 944                 $val = NFD($val);
 945                 $val =~ s/\pM+//sgo;
 946                 $val =~ s/\pC+//sgo;
 947                 $val =~ s/\W+$//sgo;
 948                 $ns->value( $val );
 949
 950                 push @ns_list, $ns;
 951         }
 952
 953         for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
 954                 next unless $tagline;
 955
 956                 my $tag = $tagline->getAttribute( "tag" );
 957                 my $ind1 = $tagline->getAttribute( "ind1" );
 958                 my $ind2 = $tagline->getAttribute( "ind2" );
 959
 960                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 961                         next unless $data;
 962
 963                         my $ns = $type->new;
 964
 965                         $ns->tag( $tag );
 966                         $ns->ind1( $ind1 );
 967                         $ns->ind2( $ind2 );
 968                         $ns->subfield( $data->getAttribute( "code" ) );
 969                         my $val = $data->textContent;
 970                         $val = NFD($val);
 971                         $val =~ s/\pM+//sgo;
 972                         $val =~ s/\pC+//sgo;
 973                         $val =~ s/\W+$//sgo;
 974             $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 975             $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 976                         $ns->value( lc($val) );
 977
 978                         push @ns_list, $ns;
 979                 }
 980
 981         if ($xmltype eq 'metabib' and $tag eq '245') {
 982                 $tag = 'tnf';
 983
 984                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 985                         next unless ($data and $data->getAttribute( "code" ) eq 'a');
 986
 987                         $ns = $type->new;
 988
 989                         $ns->tag( $tag );
 990                         $ns->ind1( $ind1 );
 991                         $ns->ind2( $ind2 );
 992                         $ns->subfield( $data->getAttribute( "code" ) );
 993                         my $val = substr( $data->textContent, $ind2 );
 994                         $val = NFD($val);
 995                         $val =~ s/\pM+//sgo;
 996                         $val =~ s/\pC+//sgo;
 997                         $val =~ s/\W+$//sgo;
 998                 $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 999                 $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
1000                         $ns->value( lc($val) );
1001
1002                         push @ns_list, $ns;
1003                 }
1004         }
1005         }
1006
1007         $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml");
1008         return @ns_list;
1009 }
1010
1011 sub flat_marc_xml {
1012         my $self = shift;
1013         my $client = shift;
1014         my $xml = shift;
1015
1016         $log->debug("processing [$xml]");
1017
1018         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
1019
1020         my $type = 'metabib';
1021         $type = 'authority' if ($self->api_name =~ /authority/o);
1022
1023         OpenILS::Application::Ingest->post_init();
1024
1025         $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
1026         return undef;
1027 }
1028 __PACKAGE__->register_method(
1029         api_name        => "open-ils.ingest.flat_marc.authority.xml",
1030         method          => "flat_marc_xml",
1031         api_level       => 1,
1032         argc            => 1,
1033         stream          => 1,
1034 );
1035 __PACKAGE__->register_method(
1036         api_name        => "open-ils.ingest.flat_marc.biblio.xml",
1037         method          => "flat_marc_xml",
1038         api_level       => 1,
1039         argc            => 1,
1040         stream          => 1,
1041 );
1042
1043 sub flat_marc_record {
1044         my $self = shift;
1045         my $client = shift;
1046         my $rec = shift;
1047
1048         my $type = 'biblio';
1049         $type = 'authority' if ($self->api_name =~ /authority/o);
1050
1051         OpenILS::Application::Ingest->post_init();
1052         my $r = OpenSRF::AppSession
1053                         ->create('open-ils.cstore')
1054                         ->request( "open-ils.cstore.direct.${type}.record_entry.retrieve" => $rec )
1055                         ->gather(1);
1056
1057
1058         return undef unless ($r and $r->marc);
1059
1060         my @rows = $self->method_lookup("open-ils.ingest.flat_marc.$type.xml")->run($r->marc);
1061         for my $row (@rows) {
1062                 $client->respond($row);
1063                 $log->debug(OpenSRF::Utils::JSON->perl2JSON($row), DEBUG);
1064         }
1065         return undef;
1066 }
1067 __PACKAGE__->register_method(
1068         api_name        => "open-ils.ingest.flat_marc.biblio.record_entry",
1069         method          => "flat_marc_record",
1070         api_level       => 1,
1071         argc            => 1,
1072         stream          => 1,
1073 );
1074 __PACKAGE__->register_method(
1075         api_name        => "open-ils.ingest.flat_marc.authority.record_entry",
1076         method          => "flat_marc_record",
1077         api_level       => 1,
1078         argc            => 1,
1079         stream          => 1,
1080 );
1081
1082
1083 # --------------------------------------------------------------------------------
1084 # URI extraction
1085
1086 package OpenILS::Application::Ingest::Biblio::URI;
1087 use base qw/OpenILS::Application::Ingest/;
1088 use Unicode::Normalize;
1089 use OpenSRF::EX qw/:try/;
1090
1091
1092 sub _extract_856_uris {
1093
1094     my $recid   = shift;
1095         my $marcxml = shift;
1096         my $max_cn = shift;
1097         my @objects;
1098
1099         my $document = $parser->parse_string($marcxml);
1100         my @nodes = $document->findnodes('//*[local-name()="datafield" and @tag="856" and (@ind1="4" or @ind1="1") and (@ind2="0" or @ind2="1")]');
1101
1102     my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
1103
1104     for my $node (@nodes) {
1105         # first, is there a URI?
1106         my $href = $node->findvalue('*[local-name()="subfield" and @code="u"]/text()');
1107         next unless ($href);
1108
1109         # now, find the best possible label
1110         my $label = $node->findvalue('*[local-name()="subfield" and @code="y"]/text()');
1111         $label ||= $node->findvalue('*[local-name()="subfield" and @code="3"]/text()');
1112         $label ||= $href;
1113
1114         # look for use info
1115         my $use = $node->findvalue('*[local-name()="subfield" and @code="z"]/text()');
1116         $use ||= $node->findvalue('*[local-name()="subfield" and @code="2"]/text()');
1117         $use ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1118
1119         # moving on to the URI owner
1120         my $owner = $node->findvalue('*[local-name()="subfield" and @code="w"]/text()');
1121         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1122         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="9"]/text()'); # Evergreen special sauce
1123
1124         $owner =~ s/^.*?\((\w+)\).*$/$1/o; # unwrap first paren-enclosed string and then ...
1125
1126         # no owner? skip it :(
1127         next unless ($owner);
1128
1129         my $org = $cstore
1130             ->request( 'open-ils.cstore.direct.actor.org_unit.search' => { shortname => $owner} )
1131                         ->gather(1);
1132
1133         next unless ($org);
1134
1135         # now we can construct the uri object
1136         my $uri = $cstore
1137             ->request( 'open-ils.cstore.direct.asset.uri.search' => { label => $label, href => $href, use_restriction => $use, active => 't' } )
1138                         ->gather(1);
1139
1140         if (!$uri) {
1141             $uri = Fieldmapper::asset::uri->new;
1142             $uri->isnew( 1 );
1143             $uri->label($label);
1144             $uri->href($href);
1145             $uri->use_restriction($use);
1146         }
1147
1148         # see if we need to create a call number
1149         my $cn = $cstore
1150             ->request( 'open-ils.cstore.direct.asset.call_number.search' => { owning_lib => $org->id, record => $recid, label => '##URI##' } )
1151                         ->gather(1);
1152
1153         if (!$cn) {
1154             $cn = Fieldmapper::asset::call_number->new;
1155             $cn->isnew( 1 );
1156             $cn->id( $$max_cn++ );
1157             $cn->owning_lib( $org->id );
1158             $cn->record( $recid );
1159             $cn->label( '##URI##' );
1160         }
1161
1162         push @objects, { uri => $uri, call_number => $cn };
1163     }
1164
1165         $log->debug("Returning ".scalar(@objects)." URI nodes for record $recid");
1166         return @objects;
1167 }
1168
1169 sub get_uris_record {
1170         my $self = shift;
1171         my $client = shift;
1172         my $rec = shift;
1173
1174         OpenILS::Application::Ingest->post_init();
1175         my $r = OpenSRF::AppSession
1176                         ->create('open-ils.cstore')
1177                         ->request( "open-ils.cstore.direct.biblio.record_entry.retrieve" => $rec )
1178                         ->gather(1);
1179
1180         return undef unless ($r and $r->marc);
1181
1182         $client->respond($_) for (_extract_856_uris($r->id, $r->marc));
1183         return undef;
1184 }
1185 __PACKAGE__->register_method(
1186         api_name        => "open-ils.ingest.856_uri.record",
1187         method          => "get_uris_record",
1188         api_level       => 1,
1189         argc            => 1,
1190         stream          => 1,
1191 );
1192
1193 sub get_uris_object {
1194         my $self = shift;
1195         my $client = shift;
1196         my $obj = shift;
1197         my $max_cn = shift;
1198
1199         return undef unless ($obj and $obj->marc);
1200
1201         $client->respond($_) for (_extract_856_uris($obj->id, $obj->marc, \$max_cn));
1202         return undef;
1203 }
1204 __PACKAGE__->register_method(
1205         api_name        => "open-ils.ingest.856_uri.object",
1206         method          => "get_uris_object",
1207         api_level       => 1,
1208         argc            => 1,
1209         stream          => 1,
1210 );
1211
1212
1213 # --------------------------------------------------------------------------------
1214 # Fingerprinting
1215
1216 package OpenILS::Application::Ingest::Biblio::Fingerprint;
1217 use base qw/OpenILS::Application::Ingest/;
1218 use Unicode::Normalize;
1219 use OpenSRF::EX qw/:try/;
1220
1221 sub biblio_fingerprint_record {
1222         my $self = shift;
1223         my $client = shift;
1224         my $rec = shift;
1225
1226         OpenILS::Application::Ingest->post_init();
1227
1228         my $r = OpenSRF::AppSession
1229                         ->create('open-ils.cstore')
1230                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
1231                         ->gather(1);
1232
1233         return undef unless ($r and $r->marc);
1234
1235         my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
1236         $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
1237         $fp->{quality} = int($fp->{quality});
1238         return $fp;
1239 }
1240 __PACKAGE__->register_method(
1241         api_name        => "open-ils.ingest.fingerprint.record",
1242         method          => "biblio_fingerprint_record",
1243         api_level       => 1,
1244         argc            => 1,
1245 );
1246
1247 our $fp_script;
1248 sub biblio_fingerprint {
1249         my $self = shift;
1250         my $client = shift;
1251         my $xml = OpenILS::Application::Ingest::entityize(shift);
1252
1253         $log->internal("Got MARC [$xml]");
1254
1255         if(!$fp_script) {
1256                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1257                 my $conf = OpenSRF::Utils::SettingsClient->new;
1258
1259                 my $libs        = $conf->config_value(@pfx, 'script_path');
1260                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_fingerprint');
1261                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1262
1263                 $log->debug("Loading script $script_file for biblio fingerprinting...");
1264
1265                 $fp_script = new OpenILS::Utils::ScriptRunner
1266                         ( file          => $script_file,
1267                           paths         => $script_libs,
1268                           reset_count   => 100 );
1269         }
1270
1271         $fp_script->insert('environment' => {marc => $xml} => 1);
1272
1273         my $res = $fp_script->run || ($log->error( "Fingerprint script died!  $@" ) && return undef);
1274         $log->debug("Script for biblio fingerprinting completed successfully...");
1275
1276         return $res;
1277 }
1278 __PACKAGE__->register_method(
1279         api_name        => "open-ils.ingest.fingerprint.xml",
1280         method          => "biblio_fingerprint",
1281         api_level       => 1,
1282         argc            => 1,
1283 );
1284
1285 our $rd_script;
1286 sub biblio_descriptor {
1287         my $self = shift;
1288         my $client = shift;
1289         my $xml = OpenILS::Application::Ingest::entityize(shift);
1290
1291         $log->internal("Got MARC [$xml]");
1292
1293         if(!$rd_script) {
1294                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1295                 my $conf = OpenSRF::Utils::SettingsClient->new;
1296
1297                 my $libs        = $conf->config_value(@pfx, 'script_path');
1298                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_descriptor');
1299                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1300
1301                 $log->debug("Loading script $script_file for biblio descriptor extraction...");
1302
1303                 $rd_script = new OpenILS::Utils::ScriptRunner
1304                         ( file          => $script_file,
1305                           paths         => $script_libs,
1306                           reset_count   => 100 );
1307         }
1308
1309         $log->debug("Setting up environment for descriptor extraction script...");
1310         $rd_script->insert('environment.marc' => $xml => 1);
1311         $log->debug("Environment building complete...");
1312
1313         my $res = $rd_script->run || ($log->error( "Descriptor script died!  $@" ) && return undef);
1314         $log->debug("Script for biblio descriptor extraction completed successfully");
1315
1316     my $d1 = $res->date1;
1317     if ($d1 && $d1 ne '    ') {
1318         $d1 =~ tr/ux/00/;
1319         $res->date1( $d1 );
1320     }
1321
1322     my $d2 = $res->date2;
1323     if ($d2 && $d2 ne '    ') {
1324         $d2 =~ tr/ux/99/;
1325         $res->date2( $d2 );
1326     }
1327
1328         return $res;
1329 }
1330 __PACKAGE__->register_method(
1331         api_name        => "open-ils.ingest.descriptor.xml",
1332         method          => "biblio_descriptor",
1333         api_level       => 1,
1334         argc            => 1,
1335 );
1336
1337
1338 1;
1339