Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm

   1 package OpenILS::Application::Ingest;
   2 use OpenILS::Application;
   3 use base qw/OpenILS::Application/;
   4
   5 use Unicode::Normalize;
   6 use OpenSRF::EX qw/:try/;
   7
   8 use OpenSRF::AppSession;
   9 use OpenSRF::Utils::SettingsClient;
  10 use OpenSRF::Utils::Logger qw/:level/;
  11
  12 use OpenILS::Utils::ScriptRunner;
  13 use OpenILS::Utils::Fieldmapper;
  14 use OpenSRF::Utils::JSON;
  15
  16 use OpenILS::Utils::Fieldmapper;
  17
  18 use XML::LibXML;
  19 use XML::LibXSLT;
  20 use Time::HiRes qw(time);
  21
  22 our %supported_formats = (
  23         mods32  => {ns => 'http://www.loc.gov/mods/v3'},
  24         mods3   => {ns => 'http://www.loc.gov/mods/v3'},
  25         mods    => {ns => 'http://www.loc.gov/mods/'},
  26         marcxml => {ns => 'http://www.loc.gov/MARC21/slim'},
  27         srw_dc  => {ns => 'info:srw/schema/1/dc-schema'},
  28         oai_dc  => {ns => 'http://www.openarchives.org/OAI/2.0/oai_dc/'},
  29         rdf_dc  => {ns => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'},
  30         atom    => {ns => 'http://www.w3.org/2005/Atom'},
  31         rss091  => {ns => 'http://my.netscape.com/rdf/simple/0.9/'},
  32         rss092  => {ns => ''},
  33         rss093  => {ns => ''},
  34         rss094  => {ns => ''},
  35         rss10   => {ns => 'http://purl.org/rss/1.0/'},
  36         rss11   => {ns => 'http://purl.org/net/rss1.1#'},
  37         rss2    => {ns => ''},
  38 );
  39
  40
  41 my $log = 'OpenSRF::Utils::Logger';
  42
  43 my  $parser = XML::LibXML->new();
  44 my  $xslt = XML::LibXSLT->new();
  45
  46 my  $mods_sheet;
  47 my  $mads_sheet;
  48 my  $xpathset = {};
  49 sub initialize {}
  50 sub child_init {}
  51
  52 sub post_init {
  53
  54         unless (keys %$xpathset) {
  55                 $log->debug("Running post_init", DEBUG);
  56
  57                 my $xsldir = OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl');
  58
  59                 unless ($supported_formats{mods}{xslt}) {
  60                         $log->debug("Loading MODS XSLT", DEBUG);
  61                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS.xsl");
  62                         $supported_formats{mods}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  63                 }
  64
  65                 unless ($supported_formats{mods3}{xslt}) {
  66                         $log->debug("Loading MODS v3 XSLT", DEBUG);
  67                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS3.xsl");
  68                         $supported_formats{mods3}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  69                 }
  70
  71                 unless ($supported_formats{mods32}{xslt}) {
  72                         $log->debug("Loading MODS v32 XSLT", DEBUG);
  73                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS32.xsl");
  74                         $supported_formats{mods32}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  75                 }
  76
  77                 my $req = OpenSRF::AppSession
  78                                 ->create('open-ils.cstore')
  79
  80                                 # XXX testing new metabib field use for faceting
  81                                 #->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { id => { '!=' => undef } } )
  82                                 ->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { search_field => 't' } )
  83
  84                                 ->gather(1);
  85
  86                 if (ref $req and @$req) {
  87                         for my $f (@$req) {
  88                                 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
  89                                 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
  90                                 $xpathset->{ $f->field_class }->{ $f->name }->{format} = $f->format;
  91                                 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
  92                         }
  93                 }
  94         }
  95 }
  96
  97 sub entityize {
  98         my $stuff = shift;
  99         my $form = shift;
 100
 101         if ($form eq 'D') {
 102                 $stuff = NFD($stuff);
 103         } else {
 104                 $stuff = NFC($stuff);
 105         }
 106
 107         $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
 108         return $stuff;
 109 }
 110
 111 # --------------------------------------------------------------------------------
 112 # Biblio ingest
 113
 114 package OpenILS::Application::Ingest::Biblio;
 115 use base qw/OpenILS::Application::Ingest/;
 116 use Unicode::Normalize;
 117
 118 sub rw_biblio_ingest_single_object {
 119         my $self = shift;
 120         my $client = shift;
 121         my $bib = shift;
 122
 123         my ($blob) = $self->method_lookup("open-ils.ingest.full.biblio.object.readonly")->run($bib);
 124         return undef unless ($blob);
 125
 126         $bib->fingerprint( $blob->{fingerprint}->{fingerprint} );
 127         $bib->quality( $blob->{fingerprint}->{quality} );
 128
 129         my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
 130
 131         my $xact = $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 132     my $tmp;
 133
 134         # update uri stuff ...
 135
 136     # gather URI call numbers for this record
 137     my $uri_cns = $u->{call_number} = $cstore->request(
 138         'open-ils.cstore.direct.asset.call_number.id_list.atomic' => { record => $bib->id, label => '##URI##' }
 139     )->gather(1);
 140
 141     # gather the maps for those call numbers
 142     my $uri_maps = $u->{call_number} = $cstore->request(
 143         'open-ils.cstore.direct.asset.uri_call_number_map.id_list.atomic' => { call_number => $uri_cns }
 144     )->gather(1);
 145
 146     # delete the old maps
 147     $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.delete' => $_ )->gather(1) for (@$uri_maps);
 148
 149     # and delete the call numbers if there are no more URIs
 150     if (!@{ $blob->{uri} }) {
 151         $cstore->request( 'open-ils.cstore.direct.asset.call_number.delete' => $_ )->gather(1) for (@$uri_cns);
 152     }
 153
 154     # now, add CNs, URIs and maps
 155     my %new_cns_by_owner;
 156     my %new_uris_by_owner;
 157     for my $u ( @{ $blob->{uri} } ) {
 158
 159         my $owner = $u->{call_number}->owning_lib;
 160
 161         if ($u->{call_number}->isnew) {
 162             if ($new_cns_by_owner{$owner}) {
 163                 $u->{call_number} = $new_cns_by_owner{$owner};
 164             } else {
 165                 $u->{call_number} = $new_cns_by_owner{$owner} = $cstore->request(
 166                     'open-ils.cstore.direct.asset.call_number.create' => $u->{call_number}
 167                 )->gather(1);
 168             }
 169         }
 170
 171         if ($u->{uri}->isnew) {
 172             if ($new_uris_by_owner{$owner}) {
 173                     $u->{uri} = $new_uris_by_owner{$owner};
 174             } else {
 175                     $u->{uri} = $new_uris_by_owner{$owner} = $cstore->request(
 176                     'open-ils.cstore.direct.asset.uri.create' => $u->{uri}
 177                 )->gather(1);
 178             }
 179         }
 180
 181         my $umap = Fieldmapper::asset::uri_call_number_map->new;
 182         $umap->uri($u->{uri}->id);
 183         $umap->call_number($u->{call_number}->id);
 184
 185             $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.create' => $umap )->gather(1) if (!$tmp);
 186     }
 187
 188         # update full_rec stuff ...
 189         $tmp = $cstore->request(
 190                 'open-ils.cstore.direct.metabib.full_rec.id_list.atomic',
 191                 { record => $bib->id }
 192         )->gather(1);
 193
 194         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.delete' => $_ )->gather(1) for (@$tmp);
 195         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.create' => $_ )->gather(1) for (@{ $blob->{full_rec} });
 196
 197         # update rec_descriptor stuff ...
 198         $tmp = $cstore->request(
 199                 'open-ils.cstore.direct.metabib.record_descriptor.id_list.atomic',
 200                 { record => $bib->id }
 201         )->gather(1);
 202
 203         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.delete' => $_ )->gather(1) for (@$tmp);
 204         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.create' => $blob->{descriptor} )->gather(1);
 205
 206         # deal with classed fields...
 207         for my $class ( qw/title author subject keyword series/ ) {
 208                 $tmp = $cstore->request(
 209                         "open-ils.cstore.direct.metabib.${class}_field_entry.id_list.atomic",
 210                         { source => $bib->id }
 211                 )->gather(1);
 212
 213                 $cstore->request( "open-ils.cstore.direct.metabib.${class}_field_entry.delete" => $_ )->gather(1) for (@$tmp);
 214         }
 215         for my $obj ( @{ $blob->{field_entries} } ) {
 216                 my $class = $obj->class_name;
 217                 $class =~ s/^Fieldmapper:://o;
 218                 $class =~ s/::/./go;
 219                 $cstore->request( "open-ils.cstore.direct.$class.create" => $obj )->gather(1);
 220         }
 221
 222         # update MR map ...
 223
 224         $tmp = $cstore->request(
 225                 'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 226                 { source => $bib->id }
 227         )->gather(1);
 228
 229         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.delete' => $_->id )->gather(1) for (@$tmp);
 230
 231         # get the old MRs
 232         my $old_mrs = $cstore->request(
 233                 'open-ils.cstore.direct.metabib.metarecord.search.atomic' => { id => [map { $_->metarecord } @$tmp] }
 234         )->gather(1) if (@$tmp);
 235
 236         $old_mrs = [] if (!ref($old_mrs));
 237
 238         my $mr;
 239         for my $m (@$old_mrs) {
 240                 if ($m->fingerprint eq $bib->fingerprint) {
 241                         $mr = $m;
 242                 } else {
 243                         my $others = $cstore->request(
 244                                 'open-ils.cstore.direct.metabib.metarecord_source_map.id_list.atomic' => { metarecord => $m->id }
 245                         )->gather(1);
 246
 247                         if (!@$others) {
 248                                 $cstore->request(
 249                                         'open-ils.cstore.direct.metabib.metarecord.delete' => $m->id
 250                                 )->gather(1);
 251                         }
 252
 253                         $m->isdeleted(1);
 254                 }
 255         }
 256
 257         my $holds;
 258         if (!$mr) {
 259                 # Get the matchin MR, if any.
 260                 $mr = $cstore->request(
 261                         'open-ils.cstore.direct.metabib.metarecord.search',
 262                         { fingerprint => $bib->fingerprint }
 263                 )->gather(1);
 264
 265                 $holds = $cstore->request(
 266                         'open-ils.cstore.direct.action.hold_request.search.atomic',
 267                         { hold_type => 'M', target => [ map { $_->id } grep { $_->isdeleted } @$old_mrs ] }
 268                 )->gather(1) if (@$old_mrs);
 269
 270                 if ($mr) {
 271                         for my $h (@$holds) {
 272                                 $h->target($mr);
 273                                 $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 274                                 $h->ischanged(1);
 275                         }
 276                 }
 277         }
 278
 279         if (!$mr) {
 280                 $mr = new Fieldmapper::metabib::metarecord;
 281                 $mr->fingerprint( $bib->fingerprint );
 282                 $mr->master_record( $bib->id );
 283                 $mr->id(
 284                         $cstore->request(
 285                                 "open-ils.cstore.direct.metabib.metarecord.create",
 286                                 $mr => { quiet => 'true' }
 287                         )->gather(1)
 288                 );
 289
 290                 for my $h (grep { !$_->ischanged } @$holds) {
 291                         $h->target($mr);
 292                         $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 293                 }
 294         } else {
 295                 my $mrm = $cstore->request(
 296                         'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 297                         { metarecord => $mr->id }
 298                 )->gather(1);
 299
 300                 if (@$mrm) {
 301                         my $best = $cstore->request(
 302                                 "open-ils.cstore.direct.biblio.record_entry.search",
 303                                 { id => [ map { $_->source } @$mrm ] },
 304                                 { 'select'      => { bre => [ qw/id quality/ ] },
 305                                 order_by        => { bre => "quality desc" },
 306                                 limit           => 1,
 307                                 }
 308                         )->gather(1);
 309
 310                         if ($best->quality > $bib->quality) {
 311                                 $mr->master_record($best->id);
 312                         } else {
 313                                 $mr->master_record($bib->id);
 314                         }
 315                 } else {
 316                         $mr->master_record($bib->id);
 317                 }
 318
 319                 $mr->clear_mods;
 320
 321                 $cstore->request( 'open-ils.cstore.direct.metabib.metarecord.update' => $mr )->gather(1);
 322         }
 323
 324         my $mrm = new Fieldmapper::metabib::metarecord_source_map;
 325         $mrm->source($bib->id);
 326         $mrm->metarecord($mr->id);
 327
 328         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.create' => $mrm )->gather(1);
 329         $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.update' => $bib )->gather(1);
 330
 331         $cstore->request( 'open-ils.cstore.transaction.commit' )->gather(1) || return undef;;
 332     $cstore->disconnect;
 333
 334         return $bib->id;
 335 }
 336 __PACKAGE__->register_method(
 337         api_name        => "open-ils.ingest.full.biblio.object",
 338         method          => "rw_biblio_ingest_single_object",
 339         api_level       => 1,
 340         argc            => 1,
 341 );
 342
 343 sub rw_biblio_ingest_single_record {
 344         my $self = shift;
 345         my $client = shift;
 346         my $rec = shift;
 347
 348         OpenILS::Application::Ingest->post_init();
 349         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 350         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 351
 352         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
 353
 354         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 355         $cstore->disconnect;
 356
 357         return undef unless ($r and @$r);
 358
 359         return ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($r))[0];
 360 }
 361 __PACKAGE__->register_method(
 362         api_name        => "open-ils.ingest.full.biblio.record",
 363         method          => "rw_biblio_ingest_single_record",
 364         api_level       => 1,
 365         argc            => 1,
 366 );
 367
 368 sub rw_biblio_ingest_record_list {
 369         my $self = shift;
 370         my $client = shift;
 371         my @rec = ref($_[0]) ? @{ $_[0] } : @_ ;
 372
 373         OpenILS::Application::Ingest->post_init();
 374         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 375         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 376
 377         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.search.atomic' => { id => $rec } )->gather(1);
 378
 379         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 380         $cstore->disconnect;
 381
 382         return undef unless ($r and @$r);
 383
 384         my $count = 0;
 385         $count += ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($_))[0] for (@$r);
 386
 387         return $count;
 388 }
 389 __PACKAGE__->register_method(
 390         api_name        => "open-ils.ingest.full.biblio.record_list",
 391         method          => "rw_biblio_ingest_record_list",
 392         api_level       => 1,
 393         argc            => 1,
 394 );
 395
 396 sub ro_biblio_ingest_single_object {
 397         my $self = shift;
 398         my $client = shift;
 399         my $bib = shift;
 400         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 401
 402         my $document = $parser->parse_string($xml);
 403
 404         my @uris = $self->method_lookup("open-ils.ingest.856_uri.object")->run($bib);
 405         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 406         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 407         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 408         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 409
 410         $_->source($bib->id) for (@mXfe);
 411         $_->record($bib->id) for (@mfr);
 412         $rd->record($bib->id) if ($rd);
 413
 414         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd, uri => \@uris };
 415 }
 416 __PACKAGE__->register_method(
 417         api_name        => "open-ils.ingest.full.biblio.object.readonly",
 418         method          => "ro_biblio_ingest_single_object",
 419         api_level       => 1,
 420         argc            => 1,
 421 );
 422
 423 sub ro_biblio_ingest_single_xml {
 424         my $self = shift;
 425         my $client = shift;
 426         my $xml = OpenILS::Application::Ingest::entityize(shift);
 427
 428         my $document = $parser->parse_string($xml);
 429
 430         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 431         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 432         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 433         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 434
 435         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd };
 436 }
 437 __PACKAGE__->register_method(
 438         api_name        => "open-ils.ingest.full.biblio.xml.readonly",
 439         method          => "ro_biblio_ingest_single_xml",
 440         api_level       => 1,
 441         argc            => 1,
 442 );
 443
 444 sub ro_biblio_ingest_single_record {
 445         my $self = shift;
 446         my $client = shift;
 447         my $rec = shift;
 448
 449         OpenILS::Application::Ingest->post_init();
 450         my $r = OpenSRF::AppSession
 451                         ->create('open-ils.cstore')
 452                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 453                         ->gather(1);
 454
 455         return undef unless ($r and @$r);
 456
 457         my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($r->marc);
 458
 459         $_->source($rec) for (@{$res->{field_entries}});
 460         $_->record($rec) for (@{$res->{full_rec}});
 461         $res->{descriptor}->record($rec);
 462
 463         return $res;
 464 }
 465 __PACKAGE__->register_method(
 466         api_name        => "open-ils.ingest.full.biblio.record.readonly",
 467         method          => "ro_biblio_ingest_single_record",
 468         api_level       => 1,
 469         argc            => 1,
 470 );
 471
 472 sub ro_biblio_ingest_stream_record {
 473         my $self = shift;
 474         my $client = shift;
 475
 476         OpenILS::Application::Ingest->post_init();
 477
 478         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 479
 480         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 481
 482                 my $rec = $resp->content;
 483                 last unless (defined $rec);
 484
 485                 $log->debug("Running open-ils.ingest.full.biblio.record.readonly ...");
 486                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.record.readonly")->run($rec);
 487
 488                 $_->source($rec) for (@{$res->{field_entries}});
 489                 $_->record($rec) for (@{$res->{full_rec}});
 490
 491                 $client->respond( $res );
 492         }
 493
 494         return undef;
 495 }
 496 __PACKAGE__->register_method(
 497         api_name        => "open-ils.ingest.full.biblio.record_stream.readonly",
 498         method          => "ro_biblio_ingest_stream_record",
 499         api_level       => 1,
 500         stream          => 1,
 501 );
 502
 503 sub ro_biblio_ingest_stream_xml {
 504         my $self = shift;
 505         my $client = shift;
 506
 507         OpenILS::Application::Ingest->post_init();
 508
 509         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 510
 511         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 512
 513                 my $xml = $resp->content;
 514                 last unless (defined $xml);
 515
 516                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 517                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($xml);
 518
 519                 $client->respond( $res );
 520         }
 521
 522         return undef;
 523 }
 524 __PACKAGE__->register_method(
 525         api_name        => "open-ils.ingest.full.biblio.xml_stream.readonly",
 526         method          => "ro_biblio_ingest_stream_xml",
 527         api_level       => 1,
 528         stream          => 1,
 529 );
 530
 531 sub rw_biblio_ingest_stream_import {
 532         my $self = shift;
 533         my $client = shift;
 534
 535         OpenILS::Application::Ingest->post_init();
 536
 537         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 538
 539         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 540
 541                 my $bib = $resp->content;
 542                 last unless (defined $bib);
 543
 544                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 545                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($bib->marc);
 546
 547                 $_->source($bib->id) for (@{$res->{field_entries}});
 548                 $_->record($bib->id) for (@{$res->{full_rec}});
 549
 550                 $client->respond( $res );
 551         }
 552
 553         return undef;
 554 }
 555 __PACKAGE__->register_method(
 556         api_name        => "open-ils.ingest.full.biblio.bib_stream.import",
 557         method          => "rw_biblio_ingest_stream_import",
 558         api_level       => 1,
 559         stream          => 1,
 560 );
 561
 562
 563 # --------------------------------------------------------------------------------
 564 # Authority ingest
 565
 566 package OpenILS::Application::Ingest::Authority;
 567 use base qw/OpenILS::Application::Ingest/;
 568 use Unicode::Normalize;
 569
 570 sub ro_authority_ingest_single_object {
 571         my $self = shift;
 572         my $client = shift;
 573         my $bib = shift;
 574         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 575
 576         my $document = $parser->parse_string($xml);
 577
 578         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 579
 580         $_->record($bib->id) for (@mfr);
 581
 582         return { full_rec => \@mfr };
 583 }
 584 __PACKAGE__->register_method(
 585         api_name        => "open-ils.ingest.full.authority.object.readonly",
 586         method          => "ro_authority_ingest_single_object",
 587         api_level       => 1,
 588         argc            => 1,
 589 );
 590
 591 sub ro_authority_ingest_single_xml {
 592         my $self = shift;
 593         my $client = shift;
 594         my $xml = OpenILS::Application::Ingest::entityize(shift);
 595
 596         my $document = $parser->parse_string($xml);
 597
 598         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 599
 600         return { full_rec => \@mfr };
 601 }
 602 __PACKAGE__->register_method(
 603         api_name        => "open-ils.ingest.full.authority.xml.readonly",
 604         method          => "ro_authority_ingest_single_xml",
 605         api_level       => 1,
 606         argc            => 1,
 607 );
 608
 609 sub ro_authority_ingest_single_record {
 610         my $self = shift;
 611         my $client = shift;
 612         my $rec = shift;
 613
 614         OpenILS::Application::Ingest->post_init();
 615         my $r = OpenSRF::AppSession
 616                         ->create('open-ils.cstore')
 617                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 618                         ->gather(1);
 619
 620         return undef unless ($r and @$r);
 621
 622         my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($r->marc);
 623
 624         $_->record($rec) for (@{$res->{full_rec}});
 625         $res->{descriptor}->record($rec);
 626
 627         return $res;
 628 }
 629 __PACKAGE__->register_method(
 630         api_name        => "open-ils.ingest.full.authority.record.readonly",
 631         method          => "ro_authority_ingest_single_record",
 632         api_level       => 1,
 633         argc            => 1,
 634 );
 635
 636 sub ro_authority_ingest_stream_record {
 637         my $self = shift;
 638         my $client = shift;
 639
 640         OpenILS::Application::Ingest->post_init();
 641
 642         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 643
 644         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 645
 646                 my $rec = $resp->content;
 647                 last unless (defined $rec);
 648
 649                 $log->debug("Running open-ils.ingest.full.authority.record.readonly ...");
 650                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.record.readonly")->run($rec);
 651
 652                 $_->record($rec) for (@{$res->{full_rec}});
 653
 654                 $client->respond( $res );
 655         }
 656
 657         return undef;
 658 }
 659 __PACKAGE__->register_method(
 660         api_name        => "open-ils.ingest.full.authority.record_stream.readonly",
 661         method          => "ro_authority_ingest_stream_record",
 662         api_level       => 1,
 663         stream          => 1,
 664 );
 665
 666 sub ro_authority_ingest_stream_xml {
 667         my $self = shift;
 668         my $client = shift;
 669
 670         OpenILS::Application::Ingest->post_init();
 671
 672         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 673
 674         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 675
 676                 my $xml = $resp->content;
 677                 last unless (defined $xml);
 678
 679                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 680                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($xml);
 681
 682                 $client->respond( $res );
 683         }
 684
 685         return undef;
 686 }
 687 __PACKAGE__->register_method(
 688         api_name        => "open-ils.ingest.full.authority.xml_stream.readonly",
 689         method          => "ro_authority_ingest_stream_xml",
 690         api_level       => 1,
 691         stream          => 1,
 692 );
 693
 694 sub rw_authority_ingest_stream_import {
 695         my $self = shift;
 696         my $client = shift;
 697
 698         OpenILS::Application::Ingest->post_init();
 699
 700         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 701
 702         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 703
 704                 my $bib = $resp->content;
 705                 last unless (defined $bib);
 706
 707                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 708                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($bib->marc);
 709
 710                 $_->record($bib->id) for (@{$res->{full_rec}});
 711
 712                 $client->respond( $res );
 713         }
 714
 715         return undef;
 716 }
 717 __PACKAGE__->register_method(
 718         api_name        => "open-ils.ingest.full.authority.bib_stream.import",
 719         method          => "rw_authority_ingest_stream_import",
 720         api_level       => 1,
 721         stream          => 1,
 722 );
 723
 724
 725 # --------------------------------------------------------------------------------
 726 # MARC index extraction
 727
 728 package OpenILS::Application::Ingest::XPATH;
 729 use base qw/OpenILS::Application::Ingest/;
 730 use Unicode::Normalize;
 731
 732 # give this an XML documentElement and an XPATH expression
 733 sub xpath_to_string {
 734         my $xml = shift;
 735         my $xpath = shift;
 736         my $ns_uri = shift;
 737         my $ns_prefix = shift;
 738         my $unique = shift;
 739
 740         $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
 741
 742         my $string = "";
 743
 744         # grab the set of matching nodes
 745         my @nodes = $xml->findnodes( $xpath );
 746         for my $value (@nodes) {
 747
 748                 # grab all children of the node
 749                 my @children = $value->childNodes();
 750                 for my $child (@children) {
 751
 752                         # add the childs content to the growing buffer
 753                         my $content = quotemeta($child->textContent);
 754                         next if ($unique && $string =~ /$content/);  # uniquify the values
 755                         $string .= $child->textContent . " ";
 756                 }
 757                 if( ! @children ) {
 758                         $string .= $value->textContent . " ";
 759                 }
 760         }
 761
 762     $string =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 763     $string =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 764
 765         return NFD($string);
 766 }
 767
 768 sub class_index_string_xml {
 769         my $self = shift;
 770         my $client = shift;
 771         my $xml = shift;
 772         my @classes = @_;
 773
 774         OpenILS::Application::Ingest->post_init();
 775         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
 776
 777         my %transform_cache;
 778
 779         for my $class (@classes) {
 780                 my $class_constructor = "Fieldmapper::metabib::${class}_field_entry";
 781                 for my $type ( keys %{ $xpathset->{$class} } ) {
 782
 783                         my $def = $xpathset->{$class}->{$type};
 784                         my $sf = $OpenILS::Application::Ingest::supported_formats{$def->{format}};
 785
 786                         my $document = $xml;
 787
 788                         if ($sf->{xslt}) {
 789                                 $document = $transform_cache{$def->{format}} || $sf->{xslt}->transform($xml);
 790                                 $transform_cache{$def->{format}} = $document;
 791                         }
 792
 793                         my $value =  xpath_to_string(
 794                                         $document->documentElement      => $def->{xpath},
 795                                         $sf->{ns}                       => $def->{format},
 796                                         1
 797                         );
 798
 799                         next unless $value;
 800
 801                         $value = NFD($value);
 802                         $value =~ s/\pM+//sgo;
 803                         $value =~ s/\pC+//sgo;
 804                         $value =~ s/\W+$//sgo;
 805
 806                         $value =~ s/\b\.+\b//sgo;
 807                         $value = lc($value);
 808
 809                         my $fm = $class_constructor->new;
 810                         $fm->value( $value );
 811                         $fm->field( $xpathset->{$class}->{$type}->{id} );
 812                         $client->respond($fm);
 813                 }
 814         }
 815         return undef;
 816 }
 817 __PACKAGE__->register_method(
 818         api_name        => "open-ils.ingest.field_entry.class.xml",
 819         method          => "class_index_string_xml",
 820         api_level       => 1,
 821         argc            => 2,
 822         stream          => 1,
 823 );
 824
 825 sub class_index_string_record {
 826         my $self = shift;
 827         my $client = shift;
 828         my $rec = shift;
 829         my @classes = shift;
 830
 831         OpenILS::Application::Ingest->post_init();
 832         my $r = OpenSRF::AppSession
 833                         ->create('open-ils.cstore')
 834                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 835                         ->gather(1);
 836
 837         return undef unless ($r and @$r);
 838
 839         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, @classes)) {
 840                 $fm->source($rec);
 841                 $client->respond($fm);
 842         }
 843         return undef;
 844 }
 845 __PACKAGE__->register_method(
 846         api_name        => "open-ils.ingest.field_entry.class.record",
 847         method          => "class_index_string_record",
 848         api_level       => 1,
 849         argc            => 2,
 850         stream          => 1,
 851 );
 852
 853 sub all_index_string_xml {
 854         my $self = shift;
 855         my $client = shift;
 856         my $xml = shift;
 857
 858         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($xml, keys(%$xpathset))) {
 859                 $client->respond($fm);
 860         }
 861         return undef;
 862 }
 863 __PACKAGE__->register_method(
 864         api_name        => "open-ils.ingest.extract.field_entry.all.xml",
 865         method          => "all_index_string_xml",
 866         api_level       => 1,
 867         argc            => 1,
 868         stream          => 1,
 869 );
 870
 871 sub all_index_string_record {
 872         my $self = shift;
 873         my $client = shift;
 874         my $rec = shift;
 875
 876         OpenILS::Application::Ingest->post_init();
 877         my $r = OpenSRF::AppSession
 878                         ->create('open-ils.cstore')
 879                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 880                         ->gather(1);
 881
 882         return undef unless ($r and @$r);
 883
 884         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, keys(%$xpathset))) {
 885                 $fm->source($rec);
 886                 $client->respond($fm);
 887         }
 888         return undef;
 889 }
 890 __PACKAGE__->register_method(
 891         api_name        => "open-ils.ingest.extract.field_entry.all.record",
 892         method          => "all_index_string_record",
 893         api_level       => 1,
 894         argc            => 1,
 895         stream          => 1,
 896 );
 897
 898 # --------------------------------------------------------------------------------
 899 # Flat MARC
 900
 901 package OpenILS::Application::Ingest::FlatMARC;
 902 use base qw/OpenILS::Application::Ingest/;
 903 use Unicode::Normalize;
 904
 905
 906 sub _marcxml_to_full_rows {
 907
 908         my $marcxml = shift;
 909         my $xmltype = shift || 'metabib';
 910
 911         my $type = "Fieldmapper::${xmltype}::full_rec";
 912
 913         my @ns_list;
 914
 915         my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
 916
 917         for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
 918                 next unless $tagline;
 919
 920                 my $ns = $type->new;
 921
 922                 $ns->tag( 'LDR' );
 923                 my $val = $tagline->textContent;
 924                 $val = NFD($val);
 925                 $val =~ s/\pM+//sgo;
 926                 $val =~ s/\pC+//sgo;
 927                 $val =~ s/\W+$//sgo;
 928                 $ns->value( $val );
 929
 930                 push @ns_list, $ns;
 931         }
 932
 933         for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
 934                 next unless $tagline;
 935
 936                 my $ns = $type->new;
 937
 938                 $ns->tag( $tagline->getAttribute( "tag" ) );
 939                 my $val = $tagline->textContent;
 940                 $val = NFD($val);
 941                 $val =~ s/\pM+//sgo;
 942                 $val =~ s/\pC+//sgo;
 943                 $val =~ s/\W+$//sgo;
 944                 $ns->value( $val );
 945
 946                 push @ns_list, $ns;
 947         }
 948
 949         for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
 950                 next unless $tagline;
 951
 952                 my $tag = $tagline->getAttribute( "tag" );
 953                 my $ind1 = $tagline->getAttribute( "ind1" );
 954                 my $ind2 = $tagline->getAttribute( "ind2" );
 955
 956                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 957                         next unless $data;
 958
 959                         my $ns = $type->new;
 960
 961                         $ns->tag( $tag );
 962                         $ns->ind1( $ind1 );
 963                         $ns->ind2( $ind2 );
 964                         $ns->subfield( $data->getAttribute( "code" ) );
 965                         my $val = $data->textContent;
 966                         $val = NFD($val);
 967                         $val =~ s/\pM+//sgo;
 968                         $val =~ s/\pC+//sgo;
 969                         $val =~ s/\W+$//sgo;
 970             $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 971             $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 972                         $ns->value( lc($val) );
 973
 974                         push @ns_list, $ns;
 975                 }
 976
 977         if ($xmltype eq 'metabib' and $tag eq '245') {
 978                 $tag = 'tnf';
 979
 980                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 981                         next unless ($data and $data->getAttribute( "code" ) eq 'a');
 982
 983                         $ns = $type->new;
 984
 985                         $ns->tag( $tag );
 986                         $ns->ind1( $ind1 );
 987                         $ns->ind2( $ind2 );
 988                         $ns->subfield( $data->getAttribute( "code" ) );
 989                         my $val = substr( $data->textContent, $ind2 );
 990                         $val = NFD($val);
 991                         $val =~ s/\pM+//sgo;
 992                         $val =~ s/\pC+//sgo;
 993                         $val =~ s/\W+$//sgo;
 994                 $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 995                 $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 996                         $ns->value( lc($val) );
 997
 998                         push @ns_list, $ns;
 999                 }
1000         }
1001         }
1002
1003         $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml");
1004         return @ns_list;
1005 }
1006
1007 sub flat_marc_xml {
1008         my $self = shift;
1009         my $client = shift;
1010         my $xml = shift;
1011
1012         $log->debug("processing [$xml]");
1013
1014         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
1015
1016         my $type = 'metabib';
1017         $type = 'authority' if ($self->api_name =~ /authority/o);
1018
1019         OpenILS::Application::Ingest->post_init();
1020
1021         $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
1022         return undef;
1023 }
1024 __PACKAGE__->register_method(
1025         api_name        => "open-ils.ingest.flat_marc.authority.xml",
1026         method          => "flat_marc_xml",
1027         api_level       => 1,
1028         argc            => 1,
1029         stream          => 1,
1030 );
1031 __PACKAGE__->register_method(
1032         api_name        => "open-ils.ingest.flat_marc.biblio.xml",
1033         method          => "flat_marc_xml",
1034         api_level       => 1,
1035         argc            => 1,
1036         stream          => 1,
1037 );
1038
1039 sub flat_marc_record {
1040         my $self = shift;
1041         my $client = shift;
1042         my $rec = shift;
1043
1044         my $type = 'biblio';
1045         $type = 'authority' if ($self->api_name =~ /authority/o);
1046
1047         OpenILS::Application::Ingest->post_init();
1048         my $r = OpenSRF::AppSession
1049                         ->create('open-ils.cstore')
1050                         ->request( "open-ils.cstore.direct.${type}.record_entry.retrieve" => $rec )
1051                         ->gather(1);
1052
1053
1054         return undef unless ($r and $r->marc);
1055
1056         my @rows = $self->method_lookup("open-ils.ingest.flat_marc.$type.xml")->run($r->marc);
1057         for my $row (@rows) {
1058                 $client->respond($row);
1059                 $log->debug(OpenSRF::Utils::JSON->perl2JSON($row), DEBUG);
1060         }
1061         return undef;
1062 }
1063 __PACKAGE__->register_method(
1064         api_name        => "open-ils.ingest.flat_marc.biblio.record_entry",
1065         method          => "flat_marc_record",
1066         api_level       => 1,
1067         argc            => 1,
1068         stream          => 1,
1069 );
1070 __PACKAGE__->register_method(
1071         api_name        => "open-ils.ingest.flat_marc.authority.record_entry",
1072         method          => "flat_marc_record",
1073         api_level       => 1,
1074         argc            => 1,
1075         stream          => 1,
1076 );
1077
1078
1079 # --------------------------------------------------------------------------------
1080 # URI extraction
1081
1082 package OpenILS::Application::Ingest::Biblio::URI;
1083 use base qw/OpenILS::Application::Ingest/;
1084 use Unicode::Normalize;
1085 use OpenSRF::EX qw/:try/;
1086
1087
1088 sub _extract_856_uris {
1089
1090     my $recid   = shift;
1091         my $marcxml = shift;
1092         my @objects;
1093
1094         my @nodes = $marcxml->findnodes('//*[local-name()="datafield" and @tag="856" and (@ind1="4" or @ind1="1") and (@ind2="0" or @ind2="1")]');
1095
1096     my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
1097
1098     for my $node (@nodes) {
1099         # first, is there a URI?
1100         my $href = $node->findvalue('[local-name()="subfield" and @code="u"]/text()');
1101         next unless ($href);
1102
1103         # now, find the best possible label
1104         my $label = $node->findvalue('[local-name()="subfield" and @code="y"]/text()');
1105         $label ||= $node->findvalue('[local-name()="subfield" and @code="3"]/text()');
1106         $label ||= $href;
1107
1108         # look for use info
1109         my $use = $node->findvalue('[local-name()="subfield" and @code="z"]/text()');
1110         $use ||= $node->findvalue('[local-name()="subfield" and @code="2"]/text()');
1111         $use ||= $node->findvalue('[local-name()="subfield" and @code="n"]/text()');
1112
1113         # moving on to the URI owner
1114         my $owner = $node->findvalue('[local-name()="subfield" and @code="w"]/text()');
1115         $owner ||= $node->findvalue('[local-name()="subfield" and @code="n"]/text()');
1116         $owner ||= $node->findvalue('[local-name()="subfield" and @code="9"]/text()'); # Evergreen special sauce
1117
1118         $owner =~ s/^.*?\((\w+)\).*$/$1/o; # unwrap first paren-enclosed string and then ...
1119
1120         # no owner? skip it :(
1121         next unless ($owner);
1122
1123         my $org = $cstore
1124             ->request( 'open-ils.cstore.direct.actor.org_unit.search' => { shortname => $owner} )
1125                         ->gather(1);
1126
1127         next unless ($org);
1128
1129         # now we can construct the uri object
1130         my $uri = $cstore
1131             ->request( 'open-ils.cstore.direct.asset.uri.search' => { label => $label, href => $href, use => $use, active => 't' } )
1132                         ->gather(1);
1133
1134         if (!$uri) {
1135             $uri = Fieldmapper::asset::uri->new;
1136             $uri->isnew( 1 );
1137             $uri->label($label);
1138             $uri->href($href);
1139             $uri->use($use);
1140         }
1141
1142         # see if we need to create a call number
1143         my $cn = $cstore
1144             ->request( 'open-ils.cstore.direct.asset.call_number.search' => { owner => $org->id, record => $recid, label => '##URI##' } )
1145                         ->gather(1);
1146
1147         if (!$cn) {
1148             $cn = Fieldmapper::asset::call_number->new;
1149             $cn->isnew( 1 );
1150             $cn->owner( $org->id );
1151             $cn->record( $recid );
1152             $cn->label( '##URI##' );
1153         }
1154
1155         push @objects, { uri => $uri, call_number => $cn };
1156     }
1157
1158         $log->debug("Returning ".scalar(@objects)." URI nodes for record $recid");
1159         return @objects;
1160 }
1161
1162 sub get_uris_record {
1163         my $self = shift;
1164         my $client = shift;
1165         my $rec = shift;
1166
1167         OpenILS::Application::Ingest->post_init();
1168         my $r = OpenSRF::AppSession
1169                         ->create('open-ils.cstore')
1170                         ->request( "open-ils.cstore.direct.biblio.record_entry.retrieve" => $rec )
1171                         ->gather(1);
1172
1173         return undef unless ($r and $r->marc);
1174
1175         $client->respond($_) for (_extract_856_uris($r->id, $r->marc));
1176         return undef;
1177 }
1178 __PACKAGE__->register_method(
1179         api_name        => "open-ils.ingest.856_uri.record",
1180         method          => "get_uris_record",
1181         api_level       => 1,
1182         argc            => 1,
1183         stream          => 1,
1184 );
1185
1186 sub get_uris_object {
1187         my $self = shift;
1188         my $client = shift;
1189         my $obj = shift;
1190
1191         return undef unless ($obj and $obj->marc);
1192
1193         $client->respond($_) for (_extract_856_uris($obj->id, $obj->marc));
1194         return undef;
1195 }
1196 __PACKAGE__->register_method(
1197         api_name        => "open-ils.ingest.856_uri.object",
1198         method          => "get_uris_object",
1199         api_level       => 1,
1200         argc            => 1,
1201         stream          => 1,
1202 );
1203
1204
1205 # --------------------------------------------------------------------------------
1206 # Fingerprinting
1207
1208 package OpenILS::Application::Ingest::Biblio::Fingerprint;
1209 use base qw/OpenILS::Application::Ingest/;
1210 use Unicode::Normalize;
1211 use OpenSRF::EX qw/:try/;
1212
1213 sub biblio_fingerprint_record {
1214         my $self = shift;
1215         my $client = shift;
1216         my $rec = shift;
1217
1218         OpenILS::Application::Ingest->post_init();
1219
1220         my $r = OpenSRF::AppSession
1221                         ->create('open-ils.cstore')
1222                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
1223                         ->gather(1);
1224
1225         return undef unless ($r and $r->marc);
1226
1227         my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
1228         $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
1229         $fp->{quality} = int($fp->{quality});
1230         return $fp;
1231 }
1232 __PACKAGE__->register_method(
1233         api_name        => "open-ils.ingest.fingerprint.record",
1234         method          => "biblio_fingerprint_record",
1235         api_level       => 1,
1236         argc            => 1,
1237 );
1238
1239 our $fp_script;
1240 sub biblio_fingerprint {
1241         my $self = shift;
1242         my $client = shift;
1243         my $xml = OpenILS::Application::Ingest::entityize(shift);
1244
1245         $log->internal("Got MARC [$xml]");
1246
1247         if(!$fp_script) {
1248                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1249                 my $conf = OpenSRF::Utils::SettingsClient->new;
1250
1251                 my $libs        = $conf->config_value(@pfx, 'script_path');
1252                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_fingerprint');
1253                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1254
1255                 $log->debug("Loading script $script_file for biblio fingerprinting...");
1256
1257                 $fp_script = new OpenILS::Utils::ScriptRunner
1258                         ( file          => $script_file,
1259                           paths         => $script_libs,
1260                           reset_count   => 100 );
1261         }
1262
1263         $fp_script->insert('environment' => {marc => $xml} => 1);
1264
1265         my $res = $fp_script->run || ($log->error( "Fingerprint script died!  $@" ) && return undef);
1266         $log->debug("Script for biblio fingerprinting completed successfully...");
1267
1268         return $res;
1269 }
1270 __PACKAGE__->register_method(
1271         api_name        => "open-ils.ingest.fingerprint.xml",
1272         method          => "biblio_fingerprint",
1273         api_level       => 1,
1274         argc            => 1,
1275 );
1276
1277 our $rd_script;
1278 sub biblio_descriptor {
1279         my $self = shift;
1280         my $client = shift;
1281         my $xml = OpenILS::Application::Ingest::entityize(shift);
1282
1283         $log->internal("Got MARC [$xml]");
1284
1285         if(!$rd_script) {
1286                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1287                 my $conf = OpenSRF::Utils::SettingsClient->new;
1288
1289                 my $libs        = $conf->config_value(@pfx, 'script_path');
1290                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_descriptor');
1291                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1292
1293                 $log->debug("Loading script $script_file for biblio descriptor extraction...");
1294
1295                 $rd_script = new OpenILS::Utils::ScriptRunner
1296                         ( file          => $script_file,
1297                           paths         => $script_libs,
1298                           reset_count   => 100 );
1299         }
1300
1301         $log->debug("Setting up environment for descriptor extraction script...");
1302         $rd_script->insert('environment.marc' => $xml => 1);
1303         $log->debug("Environment building complete...");
1304
1305         my $res = $rd_script->run || ($log->error( "Descriptor script died!  $@" ) && return undef);
1306         $log->debug("Script for biblio descriptor extraction completed successfully");
1307
1308     my $d1 = $res->date1;
1309     if ($d1 && $d1 ne '    ') {
1310         $d1 =~ tr/ux/00/;
1311         $res->date1( $d1 );
1312     }
1313
1314     my $d2 = $res->date2;
1315     if ($d2 && $d2 ne '    ') {
1316         $d2 =~ tr/ux/99/;
1317         $res->date2( $d2 );
1318     }
1319
1320         return $res;
1321 }
1322 __PACKAGE__->register_method(
1323         api_name        => "open-ils.ingest.descriptor.xml",
1324         method          => "biblio_descriptor",
1325         api_level       => 1,
1326         argc            => 1,
1327 );
1328
1329
1330 1;
1331