Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm

   1 package OpenILS::Application::Ingest;
   2 use OpenILS::Application;
   3 use base qw/OpenILS::Application/;
   4
   5 use Unicode::Normalize;
   6 use OpenSRF::EX qw/:try/;
   7
   8 use OpenSRF::AppSession;
   9 use OpenSRF::Utils::SettingsClient;
  10 use OpenSRF::Utils::Logger qw/:level/;
  11
  12 use OpenILS::Utils::ScriptRunner;
  13 use OpenILS::Utils::Fieldmapper;
  14 use OpenSRF::Utils::JSON;
  15
  16 use OpenILS::Utils::Fieldmapper;
  17
  18 use XML::LibXML;
  19 use XML::LibXSLT;
  20 use Time::HiRes qw(time);
  21
  22 our %supported_formats = (
  23         mods32  => {ns => 'http://www.loc.gov/mods/v3'},
  24         mods3   => {ns => 'http://www.loc.gov/mods/v3'},
  25         mods    => {ns => 'http://www.loc.gov/mods/'},
  26         marcxml => {ns => 'http://www.loc.gov/MARC21/slim'},
  27         srw_dc  => {ns => 'info:srw/schema/1/dc-schema'},
  28         oai_dc  => {ns => 'http://www.openarchives.org/OAI/2.0/oai_dc/'},
  29         rdf_dc  => {ns => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'},
  30         atom    => {ns => 'http://www.w3.org/2005/Atom'},
  31         rss091  => {ns => 'http://my.netscape.com/rdf/simple/0.9/'},
  32         rss092  => {ns => ''},
  33         rss093  => {ns => ''},
  34         rss094  => {ns => ''},
  35         rss10   => {ns => 'http://purl.org/rss/1.0/'},
  36         rss11   => {ns => 'http://purl.org/net/rss1.1#'},
  37         rss2    => {ns => ''},
  38 );
  39
  40
  41 my $log = 'OpenSRF::Utils::Logger';
  42
  43 my  $parser = XML::LibXML->new();
  44 my  $xslt = XML::LibXSLT->new();
  45
  46 my  $mods_sheet;
  47 my  $mads_sheet;
  48 my  $xpathset = {};
  49 sub initialize {}
  50 sub child_init {}
  51
  52 sub post_init {
  53
  54         unless (keys %$xpathset) {
  55                 $log->debug("Running post_init", DEBUG);
  56
  57                 my $xsldir = OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl');
  58
  59                 unless ($supported_formats{mods}{xslt}) {
  60                         $log->debug("Loading MODS XSLT", DEBUG);
  61                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS.xsl");
  62                         $supported_formats{mods}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  63                 }
  64
  65                 unless ($supported_formats{mods3}{xslt}) {
  66                         $log->debug("Loading MODS v3 XSLT", DEBUG);
  67                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS3.xsl");
  68                         $supported_formats{mods3}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  69                 }
  70
  71                 unless ($supported_formats{mods32}{xslt}) {
  72                         $log->debug("Loading MODS v32 XSLT", DEBUG);
  73                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS32.xsl");
  74                         $supported_formats{mods32}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  75                 }
  76
  77                 my $req = OpenSRF::AppSession
  78                                 ->create('open-ils.cstore')
  79
  80                                 # XXX testing new metabib field use for faceting
  81                                 #->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { id => { '!=' => undef } } )
  82                                 ->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { search_field => 't' } )
  83
  84                                 ->gather(1);
  85
  86                 if (ref $req and @$req) {
  87                         for my $f (@$req) {
  88                                 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
  89                                 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
  90                                 $xpathset->{ $f->field_class }->{ $f->name }->{format} = $f->format;
  91                                 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
  92                         }
  93                 }
  94         }
  95 }
  96
  97 sub entityize {
  98         my $stuff = shift;
  99         my $form = shift;
 100
 101         if ($form eq 'D') {
 102                 $stuff = NFD($stuff);
 103         } else {
 104                 $stuff = NFC($stuff);
 105         }
 106
 107         $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
 108         return $stuff;
 109 }
 110
 111 # --------------------------------------------------------------------------------
 112 # Biblio ingest
 113
 114 package OpenILS::Application::Ingest::Biblio;
 115 use base qw/OpenILS::Application::Ingest/;
 116 use Unicode::Normalize;
 117
 118 sub rw_biblio_ingest_single_object {
 119         my $self = shift;
 120         my $client = shift;
 121         my $bib = shift;
 122
 123         my ($blob) = $self->method_lookup("open-ils.ingest.full.biblio.object.readonly")->run($bib);
 124         return undef unless ($blob);
 125
 126         $bib->fingerprint( $blob->{fingerprint}->{fingerprint} );
 127         $bib->quality( $blob->{fingerprint}->{quality} );
 128
 129         my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
 130
 131         my $xact = $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 132     my $tmp;
 133
 134         # update uri stuff ...
 135
 136     # gather URI call numbers for this record
 137     my $uri_cns = $u->{call_number} = $cstore->request(
 138         'open-ils.cstore.direct.asset.call_number.id_list.atomic' => { record => $bib->id, label => '##URI##' }
 139     )->gather(1);
 140
 141     # gather the maps for those call numbers
 142     my $uri_maps = $u->{call_number} = $cstore->request(
 143         'open-ils.cstore.direct.asset.uri_call_number_map.id_list.atomic' => { call_number => $uri_cns }
 144     )->gather(1);
 145
 146     # delete the old maps
 147     $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.delete' => $_ )->gather(1) for (@$uri_maps);
 148
 149     # and delete the call numbers if there are no more URIs
 150     if (!@{ $blob->{uri} }) {
 151         $cstore->request( 'open-ils.cstore.direct.asset.call_number.delete' => $_ )->gather(1) for (@$uri_cns);
 152     }
 153
 154     # now, add CNs, URIs and maps
 155     my %new_cns_by_owner;
 156     my %new_uris_by_owner;
 157     for my $u ( @{ $blob->{uri} } ) {
 158
 159         my $owner = $u->{call_number}->owning_lib;
 160
 161         if ($u->{call_number}->isnew) {
 162             if ($new_cns_by_owner{$owner}) {
 163                 $u->{call_number} = $new_cns_by_owner{$owner};
 164             } else {
 165                 $u->{call_number}->clear_id;
 166                 $u->{call_number} = $new_cns_by_owner{$owner} = $cstore->request(
 167                     'open-ils.cstore.direct.asset.call_number.create' => $u->{call_number}
 168                 )->gather(1);
 169             }
 170         }
 171
 172         if ($u->{uri}->isnew) {
 173             if ($new_uris_by_owner{$owner}) {
 174                     $u->{uri} = $new_uris_by_owner{$owner};
 175             } else {
 176                     $u->{uri} = $new_uris_by_owner{$owner} = $cstore->request(
 177                     'open-ils.cstore.direct.asset.uri.create' => $u->{uri}
 178                 )->gather(1);
 179             }
 180         }
 181
 182         my $umap = Fieldmapper::asset::uri_call_number_map->new;
 183         $umap->uri($u->{uri}->id);
 184         $umap->call_number($u->{call_number}->id);
 185
 186             $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.create' => $umap )->gather(1) if (!$tmp);
 187     }
 188
 189         # update full_rec stuff ...
 190         $tmp = $cstore->request(
 191                 'open-ils.cstore.direct.metabib.full_rec.id_list.atomic',
 192                 { record => $bib->id }
 193         )->gather(1);
 194
 195         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.delete' => $_ )->gather(1) for (@$tmp);
 196         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.create' => $_ )->gather(1) for (@{ $blob->{full_rec} });
 197
 198         # update rec_descriptor stuff ...
 199         $tmp = $cstore->request(
 200                 'open-ils.cstore.direct.metabib.record_descriptor.id_list.atomic',
 201                 { record => $bib->id }
 202         )->gather(1);
 203
 204         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.delete' => $_ )->gather(1) for (@$tmp);
 205         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.create' => $blob->{descriptor} )->gather(1);
 206
 207         # deal with classed fields...
 208         for my $class ( qw/title author subject keyword series/ ) {
 209                 $tmp = $cstore->request(
 210                         "open-ils.cstore.direct.metabib.${class}_field_entry.id_list.atomic",
 211                         { source => $bib->id }
 212                 )->gather(1);
 213
 214                 $cstore->request( "open-ils.cstore.direct.metabib.${class}_field_entry.delete" => $_ )->gather(1) for (@$tmp);
 215         }
 216         for my $obj ( @{ $blob->{field_entries} } ) {
 217                 my $class = $obj->class_name;
 218                 $class =~ s/^Fieldmapper:://o;
 219                 $class =~ s/::/./go;
 220                 $cstore->request( "open-ils.cstore.direct.$class.create" => $obj )->gather(1);
 221         }
 222
 223         # update MR map ...
 224
 225         $tmp = $cstore->request(
 226                 'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 227                 { source => $bib->id }
 228         )->gather(1);
 229
 230         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.delete' => $_->id )->gather(1) for (@$tmp);
 231
 232         # get the old MRs
 233         my $old_mrs = $cstore->request(
 234                 'open-ils.cstore.direct.metabib.metarecord.search.atomic' => { id => [map { $_->metarecord } @$tmp] }
 235         )->gather(1) if (@$tmp);
 236
 237         $old_mrs = [] if (!ref($old_mrs));
 238
 239         my $mr;
 240         for my $m (@$old_mrs) {
 241                 if ($m->fingerprint eq $bib->fingerprint) {
 242                         $mr = $m;
 243                 } else {
 244                         my $others = $cstore->request(
 245                                 'open-ils.cstore.direct.metabib.metarecord_source_map.id_list.atomic' => { metarecord => $m->id }
 246                         )->gather(1);
 247
 248                         if (!@$others) {
 249                                 $cstore->request(
 250                                         'open-ils.cstore.direct.metabib.metarecord.delete' => $m->id
 251                                 )->gather(1);
 252                         }
 253
 254                         $m->isdeleted(1);
 255                 }
 256         }
 257
 258         my $holds;
 259         if (!$mr) {
 260                 # Get the matchin MR, if any.
 261                 $mr = $cstore->request(
 262                         'open-ils.cstore.direct.metabib.metarecord.search',
 263                         { fingerprint => $bib->fingerprint }
 264                 )->gather(1);
 265
 266                 $holds = $cstore->request(
 267                         'open-ils.cstore.direct.action.hold_request.search.atomic',
 268                         { hold_type => 'M', target => [ map { $_->id } grep { $_->isdeleted } @$old_mrs ] }
 269                 )->gather(1) if (@$old_mrs);
 270
 271                 if ($mr) {
 272                         for my $h (@$holds) {
 273                                 $h->target($mr);
 274                                 $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 275                                 $h->ischanged(1);
 276                         }
 277                 }
 278         }
 279
 280         if (!$mr) {
 281                 $mr = new Fieldmapper::metabib::metarecord;
 282                 $mr->fingerprint( $bib->fingerprint );
 283                 $mr->master_record( $bib->id );
 284                 $mr->id(
 285                         $cstore->request(
 286                                 "open-ils.cstore.direct.metabib.metarecord.create",
 287                                 $mr => { quiet => 'true' }
 288                         )->gather(1)
 289                 );
 290
 291                 for my $h (grep { !$_->ischanged } @$holds) {
 292                         $h->target($mr);
 293                         $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 294                 }
 295         } else {
 296                 my $mrm = $cstore->request(
 297                         'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 298                         { metarecord => $mr->id }
 299                 )->gather(1);
 300
 301                 if (@$mrm) {
 302                         my $best = $cstore->request(
 303                                 "open-ils.cstore.direct.biblio.record_entry.search",
 304                                 { id => [ map { $_->source } @$mrm ] },
 305                                 { 'select'      => { bre => [ qw/id quality/ ] },
 306                                 order_by        => { bre => "quality desc" },
 307                                 limit           => 1,
 308                                 }
 309                         )->gather(1);
 310
 311                         if ($best->quality > $bib->quality) {
 312                                 $mr->master_record($best->id);
 313                         } else {
 314                                 $mr->master_record($bib->id);
 315                         }
 316                 } else {
 317                         $mr->master_record($bib->id);
 318                 }
 319
 320                 $mr->clear_mods;
 321
 322                 $cstore->request( 'open-ils.cstore.direct.metabib.metarecord.update' => $mr )->gather(1);
 323         }
 324
 325         my $mrm = new Fieldmapper::metabib::metarecord_source_map;
 326         $mrm->source($bib->id);
 327         $mrm->metarecord($mr->id);
 328
 329         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.create' => $mrm )->gather(1);
 330         $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.update' => $bib )->gather(1);
 331
 332         $cstore->request( 'open-ils.cstore.transaction.commit' )->gather(1) || return undef;;
 333     $cstore->disconnect;
 334
 335         return $bib->id;
 336 }
 337 __PACKAGE__->register_method(
 338         api_name        => "open-ils.ingest.full.biblio.object",
 339         method          => "rw_biblio_ingest_single_object",
 340         api_level       => 1,
 341         argc            => 1,
 342 );
 343
 344 sub rw_biblio_ingest_single_record {
 345         my $self = shift;
 346         my $client = shift;
 347         my $rec = shift;
 348
 349         OpenILS::Application::Ingest->post_init();
 350         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 351         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 352
 353         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
 354
 355         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 356         $cstore->disconnect;
 357
 358         return undef unless ($r and @$r);
 359
 360         return ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($r))[0];
 361 }
 362 __PACKAGE__->register_method(
 363         api_name        => "open-ils.ingest.full.biblio.record",
 364         method          => "rw_biblio_ingest_single_record",
 365         api_level       => 1,
 366         argc            => 1,
 367 );
 368
 369 sub rw_biblio_ingest_record_list {
 370         my $self = shift;
 371         my $client = shift;
 372         my @rec = ref($_[0]) ? @{ $_[0] } : @_ ;
 373
 374         OpenILS::Application::Ingest->post_init();
 375         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 376         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 377
 378         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.search.atomic' => { id => $rec } )->gather(1);
 379
 380         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 381         $cstore->disconnect;
 382
 383         return undef unless ($r and @$r);
 384
 385         my $count = 0;
 386         $count += ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($_))[0] for (@$r);
 387
 388         return $count;
 389 }
 390 __PACKAGE__->register_method(
 391         api_name        => "open-ils.ingest.full.biblio.record_list",
 392         method          => "rw_biblio_ingest_record_list",
 393         api_level       => 1,
 394         argc            => 1,
 395 );
 396
 397 sub ro_biblio_ingest_single_object {
 398         my $self = shift;
 399         my $client = shift;
 400         my $bib = shift;
 401         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 402
 403         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 404         my $cn = $cstore->request( 'open-ils.cstore.direct.asset.call_number.search' => { id => { '!=' => undef }, { limit => 1 } } )->gather(1);
 405     $cstore->disconnect;
 406
 407     my $max_cn = int($cn->id) + 1000;
 408
 409         my $document = $parser->parse_string($xml);
 410
 411         my @uris = $self->method_lookup("open-ils.ingest.856_uri.object")->run($bib, $max_cn);
 412         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 413         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 414         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 415         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 416
 417         $_->source($bib->id) for (@mXfe);
 418         $_->record($bib->id) for (@mfr);
 419         $rd->record($bib->id) if ($rd);
 420
 421         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd, uri => \@uris };
 422 }
 423 __PACKAGE__->register_method(
 424         api_name        => "open-ils.ingest.full.biblio.object.readonly",
 425         method          => "ro_biblio_ingest_single_object",
 426         api_level       => 1,
 427         argc            => 1,
 428 );
 429
 430 sub ro_biblio_ingest_single_xml {
 431         my $self = shift;
 432         my $client = shift;
 433         my $xml = OpenILS::Application::Ingest::entityize(shift);
 434
 435         my $document = $parser->parse_string($xml);
 436
 437         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 438         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 439         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 440         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 441
 442         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd };
 443 }
 444 __PACKAGE__->register_method(
 445         api_name        => "open-ils.ingest.full.biblio.xml.readonly",
 446         method          => "ro_biblio_ingest_single_xml",
 447         api_level       => 1,
 448         argc            => 1,
 449 );
 450
 451 sub ro_biblio_ingest_single_record {
 452         my $self = shift;
 453         my $client = shift;
 454         my $rec = shift;
 455
 456         OpenILS::Application::Ingest->post_init();
 457         my $r = OpenSRF::AppSession
 458                         ->create('open-ils.cstore')
 459                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 460                         ->gather(1);
 461
 462         return undef unless ($r and @$r);
 463
 464         my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($r->marc);
 465
 466         $_->source($rec) for (@{$res->{field_entries}});
 467         $_->record($rec) for (@{$res->{full_rec}});
 468         $res->{descriptor}->record($rec);
 469
 470         return $res;
 471 }
 472 __PACKAGE__->register_method(
 473         api_name        => "open-ils.ingest.full.biblio.record.readonly",
 474         method          => "ro_biblio_ingest_single_record",
 475         api_level       => 1,
 476         argc            => 1,
 477 );
 478
 479 sub ro_biblio_ingest_stream_record {
 480         my $self = shift;
 481         my $client = shift;
 482
 483         OpenILS::Application::Ingest->post_init();
 484
 485         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 486
 487         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 488
 489                 my $rec = $resp->content;
 490                 last unless (defined $rec);
 491
 492                 $log->debug("Running open-ils.ingest.full.biblio.record.readonly ...");
 493                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.record.readonly")->run($rec);
 494
 495                 $_->source($rec) for (@{$res->{field_entries}});
 496                 $_->record($rec) for (@{$res->{full_rec}});
 497
 498                 $client->respond( $res );
 499         }
 500
 501         return undef;
 502 }
 503 __PACKAGE__->register_method(
 504         api_name        => "open-ils.ingest.full.biblio.record_stream.readonly",
 505         method          => "ro_biblio_ingest_stream_record",
 506         api_level       => 1,
 507         stream          => 1,
 508 );
 509
 510 sub ro_biblio_ingest_stream_xml {
 511         my $self = shift;
 512         my $client = shift;
 513
 514         OpenILS::Application::Ingest->post_init();
 515
 516         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 517
 518         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 519
 520                 my $xml = $resp->content;
 521                 last unless (defined $xml);
 522
 523                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 524                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($xml);
 525
 526                 $client->respond( $res );
 527         }
 528
 529         return undef;
 530 }
 531 __PACKAGE__->register_method(
 532         api_name        => "open-ils.ingest.full.biblio.xml_stream.readonly",
 533         method          => "ro_biblio_ingest_stream_xml",
 534         api_level       => 1,
 535         stream          => 1,
 536 );
 537
 538 sub rw_biblio_ingest_stream_import {
 539         my $self = shift;
 540         my $client = shift;
 541
 542         OpenILS::Application::Ingest->post_init();
 543
 544         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 545
 546         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 547
 548                 my $bib = $resp->content;
 549                 last unless (defined $bib);
 550
 551                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 552                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($bib->marc);
 553
 554                 $_->source($bib->id) for (@{$res->{field_entries}});
 555                 $_->record($bib->id) for (@{$res->{full_rec}});
 556
 557                 $client->respond( $res );
 558         }
 559
 560         return undef;
 561 }
 562 __PACKAGE__->register_method(
 563         api_name        => "open-ils.ingest.full.biblio.bib_stream.import",
 564         method          => "rw_biblio_ingest_stream_import",
 565         api_level       => 1,
 566         stream          => 1,
 567 );
 568
 569
 570 # --------------------------------------------------------------------------------
 571 # Authority ingest
 572
 573 package OpenILS::Application::Ingest::Authority;
 574 use base qw/OpenILS::Application::Ingest/;
 575 use Unicode::Normalize;
 576
 577 sub ro_authority_ingest_single_object {
 578         my $self = shift;
 579         my $client = shift;
 580         my $bib = shift;
 581         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 582
 583         my $document = $parser->parse_string($xml);
 584
 585         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 586
 587         $_->record($bib->id) for (@mfr);
 588
 589         return { full_rec => \@mfr };
 590 }
 591 __PACKAGE__->register_method(
 592         api_name        => "open-ils.ingest.full.authority.object.readonly",
 593         method          => "ro_authority_ingest_single_object",
 594         api_level       => 1,
 595         argc            => 1,
 596 );
 597
 598 sub ro_authority_ingest_single_xml {
 599         my $self = shift;
 600         my $client = shift;
 601         my $xml = OpenILS::Application::Ingest::entityize(shift);
 602
 603         my $document = $parser->parse_string($xml);
 604
 605         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 606
 607         return { full_rec => \@mfr };
 608 }
 609 __PACKAGE__->register_method(
 610         api_name        => "open-ils.ingest.full.authority.xml.readonly",
 611         method          => "ro_authority_ingest_single_xml",
 612         api_level       => 1,
 613         argc            => 1,
 614 );
 615
 616 sub ro_authority_ingest_single_record {
 617         my $self = shift;
 618         my $client = shift;
 619         my $rec = shift;
 620
 621         OpenILS::Application::Ingest->post_init();
 622         my $r = OpenSRF::AppSession
 623                         ->create('open-ils.cstore')
 624                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 625                         ->gather(1);
 626
 627         return undef unless ($r and @$r);
 628
 629         my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($r->marc);
 630
 631         $_->record($rec) for (@{$res->{full_rec}});
 632         $res->{descriptor}->record($rec);
 633
 634         return $res;
 635 }
 636 __PACKAGE__->register_method(
 637         api_name        => "open-ils.ingest.full.authority.record.readonly",
 638         method          => "ro_authority_ingest_single_record",
 639         api_level       => 1,
 640         argc            => 1,
 641 );
 642
 643 sub ro_authority_ingest_stream_record {
 644         my $self = shift;
 645         my $client = shift;
 646
 647         OpenILS::Application::Ingest->post_init();
 648
 649         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 650
 651         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 652
 653                 my $rec = $resp->content;
 654                 last unless (defined $rec);
 655
 656                 $log->debug("Running open-ils.ingest.full.authority.record.readonly ...");
 657                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.record.readonly")->run($rec);
 658
 659                 $_->record($rec) for (@{$res->{full_rec}});
 660
 661                 $client->respond( $res );
 662         }
 663
 664         return undef;
 665 }
 666 __PACKAGE__->register_method(
 667         api_name        => "open-ils.ingest.full.authority.record_stream.readonly",
 668         method          => "ro_authority_ingest_stream_record",
 669         api_level       => 1,
 670         stream          => 1,
 671 );
 672
 673 sub ro_authority_ingest_stream_xml {
 674         my $self = shift;
 675         my $client = shift;
 676
 677         OpenILS::Application::Ingest->post_init();
 678
 679         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 680
 681         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 682
 683                 my $xml = $resp->content;
 684                 last unless (defined $xml);
 685
 686                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 687                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($xml);
 688
 689                 $client->respond( $res );
 690         }
 691
 692         return undef;
 693 }
 694 __PACKAGE__->register_method(
 695         api_name        => "open-ils.ingest.full.authority.xml_stream.readonly",
 696         method          => "ro_authority_ingest_stream_xml",
 697         api_level       => 1,
 698         stream          => 1,
 699 );
 700
 701 sub rw_authority_ingest_stream_import {
 702         my $self = shift;
 703         my $client = shift;
 704
 705         OpenILS::Application::Ingest->post_init();
 706
 707         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 708
 709         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 710
 711                 my $bib = $resp->content;
 712                 last unless (defined $bib);
 713
 714                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 715                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($bib->marc);
 716
 717                 $_->record($bib->id) for (@{$res->{full_rec}});
 718
 719                 $client->respond( $res );
 720         }
 721
 722         return undef;
 723 }
 724 __PACKAGE__->register_method(
 725         api_name        => "open-ils.ingest.full.authority.bib_stream.import",
 726         method          => "rw_authority_ingest_stream_import",
 727         api_level       => 1,
 728         stream          => 1,
 729 );
 730
 731
 732 # --------------------------------------------------------------------------------
 733 # MARC index extraction
 734
 735 package OpenILS::Application::Ingest::XPATH;
 736 use base qw/OpenILS::Application::Ingest/;
 737 use Unicode::Normalize;
 738
 739 # give this an XML documentElement and an XPATH expression
 740 sub xpath_to_string {
 741         my $xml = shift;
 742         my $xpath = shift;
 743         my $ns_uri = shift;
 744         my $ns_prefix = shift;
 745         my $unique = shift;
 746
 747         $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
 748
 749         my $string = "";
 750
 751         # grab the set of matching nodes
 752         my @nodes = $xml->findnodes( $xpath );
 753         for my $value (@nodes) {
 754
 755                 # grab all children of the node
 756                 my @children = $value->childNodes();
 757                 for my $child (@children) {
 758
 759                         # add the childs content to the growing buffer
 760                         my $content = quotemeta($child->textContent);
 761                         next if ($unique && $string =~ /$content/);  # uniquify the values
 762                         $string .= $child->textContent . " ";
 763                 }
 764                 if( ! @children ) {
 765                         $string .= $value->textContent . " ";
 766                 }
 767         }
 768
 769     $string =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 770     $string =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 771
 772         return NFD($string);
 773 }
 774
 775 sub class_index_string_xml {
 776         my $self = shift;
 777         my $client = shift;
 778         my $xml = shift;
 779         my @classes = @_;
 780
 781         OpenILS::Application::Ingest->post_init();
 782         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
 783
 784         my %transform_cache;
 785
 786         for my $class (@classes) {
 787                 my $class_constructor = "Fieldmapper::metabib::${class}_field_entry";
 788                 for my $type ( keys %{ $xpathset->{$class} } ) {
 789
 790                         my $def = $xpathset->{$class}->{$type};
 791                         my $sf = $OpenILS::Application::Ingest::supported_formats{$def->{format}};
 792
 793                         my $document = $xml;
 794
 795                         if ($sf->{xslt}) {
 796                                 $document = $transform_cache{$def->{format}} || $sf->{xslt}->transform($xml);
 797                                 $transform_cache{$def->{format}} = $document;
 798                         }
 799
 800                         my $value =  xpath_to_string(
 801                                         $document->documentElement      => $def->{xpath},
 802                                         $sf->{ns}                       => $def->{format},
 803                                         1
 804                         );
 805
 806                         next unless $value;
 807
 808                         $value = NFD($value);
 809                         $value =~ s/\pM+//sgo;
 810                         $value =~ s/\pC+//sgo;
 811                         $value =~ s/\W+$//sgo;
 812
 813                         $value =~ s/\b\.+\b//sgo;
 814                         $value = lc($value);
 815
 816                         my $fm = $class_constructor->new;
 817                         $fm->value( $value );
 818                         $fm->field( $xpathset->{$class}->{$type}->{id} );
 819                         $client->respond($fm);
 820                 }
 821         }
 822         return undef;
 823 }
 824 __PACKAGE__->register_method(
 825         api_name        => "open-ils.ingest.field_entry.class.xml",
 826         method          => "class_index_string_xml",
 827         api_level       => 1,
 828         argc            => 2,
 829         stream          => 1,
 830 );
 831
 832 sub class_index_string_record {
 833         my $self = shift;
 834         my $client = shift;
 835         my $rec = shift;
 836         my @classes = shift;
 837
 838         OpenILS::Application::Ingest->post_init();
 839         my $r = OpenSRF::AppSession
 840                         ->create('open-ils.cstore')
 841                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 842                         ->gather(1);
 843
 844         return undef unless ($r and @$r);
 845
 846         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, @classes)) {
 847                 $fm->source($rec);
 848                 $client->respond($fm);
 849         }
 850         return undef;
 851 }
 852 __PACKAGE__->register_method(
 853         api_name        => "open-ils.ingest.field_entry.class.record",
 854         method          => "class_index_string_record",
 855         api_level       => 1,
 856         argc            => 2,
 857         stream          => 1,
 858 );
 859
 860 sub all_index_string_xml {
 861         my $self = shift;
 862         my $client = shift;
 863         my $xml = shift;
 864
 865         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($xml, keys(%$xpathset))) {
 866                 $client->respond($fm);
 867         }
 868         return undef;
 869 }
 870 __PACKAGE__->register_method(
 871         api_name        => "open-ils.ingest.extract.field_entry.all.xml",
 872         method          => "all_index_string_xml",
 873         api_level       => 1,
 874         argc            => 1,
 875         stream          => 1,
 876 );
 877
 878 sub all_index_string_record {
 879         my $self = shift;
 880         my $client = shift;
 881         my $rec = shift;
 882
 883         OpenILS::Application::Ingest->post_init();
 884         my $r = OpenSRF::AppSession
 885                         ->create('open-ils.cstore')
 886                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 887                         ->gather(1);
 888
 889         return undef unless ($r and @$r);
 890
 891         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, keys(%$xpathset))) {
 892                 $fm->source($rec);
 893                 $client->respond($fm);
 894         }
 895         return undef;
 896 }
 897 __PACKAGE__->register_method(
 898         api_name        => "open-ils.ingest.extract.field_entry.all.record",
 899         method          => "all_index_string_record",
 900         api_level       => 1,
 901         argc            => 1,
 902         stream          => 1,
 903 );
 904
 905 # --------------------------------------------------------------------------------
 906 # Flat MARC
 907
 908 package OpenILS::Application::Ingest::FlatMARC;
 909 use base qw/OpenILS::Application::Ingest/;
 910 use Unicode::Normalize;
 911
 912
 913 sub _marcxml_to_full_rows {
 914
 915         my $marcxml = shift;
 916         my $xmltype = shift || 'metabib';
 917
 918         my $type = "Fieldmapper::${xmltype}::full_rec";
 919
 920         my @ns_list;
 921
 922         my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
 923
 924         for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
 925                 next unless $tagline;
 926
 927                 my $ns = $type->new;
 928
 929                 $ns->tag( 'LDR' );
 930                 my $val = $tagline->textContent;
 931                 $val = NFD($val);
 932                 $val =~ s/\pM+//sgo;
 933                 $val =~ s/\pC+//sgo;
 934                 $val =~ s/\W+$//sgo;
 935                 $ns->value( $val );
 936
 937                 push @ns_list, $ns;
 938         }
 939
 940         for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
 941                 next unless $tagline;
 942
 943                 my $ns = $type->new;
 944
 945                 $ns->tag( $tagline->getAttribute( "tag" ) );
 946                 my $val = $tagline->textContent;
 947                 $val = NFD($val);
 948                 $val =~ s/\pM+//sgo;
 949                 $val =~ s/\pC+//sgo;
 950                 $val =~ s/\W+$//sgo;
 951                 $ns->value( $val );
 952
 953                 push @ns_list, $ns;
 954         }
 955
 956         for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
 957                 next unless $tagline;
 958
 959                 my $tag = $tagline->getAttribute( "tag" );
 960                 my $ind1 = $tagline->getAttribute( "ind1" );
 961                 my $ind2 = $tagline->getAttribute( "ind2" );
 962
 963                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 964                         next unless $data;
 965
 966                         my $ns = $type->new;
 967
 968                         $ns->tag( $tag );
 969                         $ns->ind1( $ind1 );
 970                         $ns->ind2( $ind2 );
 971                         $ns->subfield( $data->getAttribute( "code" ) );
 972                         my $val = $data->textContent;
 973                         $val = NFD($val);
 974                         $val =~ s/\pM+//sgo;
 975                         $val =~ s/\pC+//sgo;
 976                         $val =~ s/\W+$//sgo;
 977             $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 978             $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 979                         $ns->value( lc($val) );
 980
 981                         push @ns_list, $ns;
 982                 }
 983
 984         if ($xmltype eq 'metabib' and $tag eq '245') {
 985                 $tag = 'tnf';
 986
 987                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 988                         next unless ($data and $data->getAttribute( "code" ) eq 'a');
 989
 990                         $ns = $type->new;
 991
 992                         $ns->tag( $tag );
 993                         $ns->ind1( $ind1 );
 994                         $ns->ind2( $ind2 );
 995                         $ns->subfield( $data->getAttribute( "code" ) );
 996                         my $val = substr( $data->textContent, $ind2 );
 997                         $val = NFD($val);
 998                         $val =~ s/\pM+//sgo;
 999                         $val =~ s/\pC+//sgo;
1000                         $val =~ s/\W+$//sgo;
1001                 $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
1002                 $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
1003                         $ns->value( lc($val) );
1004
1005                         push @ns_list, $ns;
1006                 }
1007         }
1008         }
1009
1010         $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml");
1011         return @ns_list;
1012 }
1013
1014 sub flat_marc_xml {
1015         my $self = shift;
1016         my $client = shift;
1017         my $xml = shift;
1018
1019         $log->debug("processing [$xml]");
1020
1021         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
1022
1023         my $type = 'metabib';
1024         $type = 'authority' if ($self->api_name =~ /authority/o);
1025
1026         OpenILS::Application::Ingest->post_init();
1027
1028         $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
1029         return undef;
1030 }
1031 __PACKAGE__->register_method(
1032         api_name        => "open-ils.ingest.flat_marc.authority.xml",
1033         method          => "flat_marc_xml",
1034         api_level       => 1,
1035         argc            => 1,
1036         stream          => 1,
1037 );
1038 __PACKAGE__->register_method(
1039         api_name        => "open-ils.ingest.flat_marc.biblio.xml",
1040         method          => "flat_marc_xml",
1041         api_level       => 1,
1042         argc            => 1,
1043         stream          => 1,
1044 );
1045
1046 sub flat_marc_record {
1047         my $self = shift;
1048         my $client = shift;
1049         my $rec = shift;
1050
1051         my $type = 'biblio';
1052         $type = 'authority' if ($self->api_name =~ /authority/o);
1053
1054         OpenILS::Application::Ingest->post_init();
1055         my $r = OpenSRF::AppSession
1056                         ->create('open-ils.cstore')
1057                         ->request( "open-ils.cstore.direct.${type}.record_entry.retrieve" => $rec )
1058                         ->gather(1);
1059
1060
1061         return undef unless ($r and $r->marc);
1062
1063         my @rows = $self->method_lookup("open-ils.ingest.flat_marc.$type.xml")->run($r->marc);
1064         for my $row (@rows) {
1065                 $client->respond($row);
1066                 $log->debug(OpenSRF::Utils::JSON->perl2JSON($row), DEBUG);
1067         }
1068         return undef;
1069 }
1070 __PACKAGE__->register_method(
1071         api_name        => "open-ils.ingest.flat_marc.biblio.record_entry",
1072         method          => "flat_marc_record",
1073         api_level       => 1,
1074         argc            => 1,
1075         stream          => 1,
1076 );
1077 __PACKAGE__->register_method(
1078         api_name        => "open-ils.ingest.flat_marc.authority.record_entry",
1079         method          => "flat_marc_record",
1080         api_level       => 1,
1081         argc            => 1,
1082         stream          => 1,
1083 );
1084
1085
1086 # --------------------------------------------------------------------------------
1087 # URI extraction
1088
1089 package OpenILS::Application::Ingest::Biblio::URI;
1090 use base qw/OpenILS::Application::Ingest/;
1091 use Unicode::Normalize;
1092 use OpenSRF::EX qw/:try/;
1093
1094
1095 sub _extract_856_uris {
1096
1097     my $recid   = shift;
1098         my $marcxml = shift;
1099         my $max_cn = shift;
1100         my @objects;
1101
1102         my $document = $parser->parse_string($marcxml);
1103         my @nodes = $document->findnodes('//*[local-name()="datafield" and @tag="856" and (@ind1="4" or @ind1="1") and (@ind2="0" or @ind2="1")]');
1104
1105     my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
1106
1107     for my $node (@nodes) {
1108         # first, is there a URI?
1109         my $href = $node->findvalue('*[local-name()="subfield" and @code="u"]/text()');
1110         next unless ($href);
1111
1112         # now, find the best possible label
1113         my $label = $node->findvalue('*[local-name()="subfield" and @code="y"]/text()');
1114         $label ||= $node->findvalue('*[local-name()="subfield" and @code="3"]/text()');
1115         $label ||= $href;
1116
1117         # look for use info
1118         my $use = $node->findvalue('*[local-name()="subfield" and @code="z"]/text()');
1119         $use ||= $node->findvalue('*[local-name()="subfield" and @code="2"]/text()');
1120         $use ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1121
1122         # moving on to the URI owner
1123         my $owner = $node->findvalue('*[local-name()="subfield" and @code="w"]/text()');
1124         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1125         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="9"]/text()'); # Evergreen special sauce
1126
1127         $owner =~ s/^.*?\((\w+)\).*$/$1/o; # unwrap first paren-enclosed string and then ...
1128
1129         # no owner? skip it :(
1130         next unless ($owner);
1131
1132         my $org = $cstore
1133             ->request( 'open-ils.cstore.direct.actor.org_unit.search' => { shortname => $owner} )
1134                         ->gather(1);
1135
1136         next unless ($org);
1137
1138         # now we can construct the uri object
1139         my $uri = $cstore
1140             ->request( 'open-ils.cstore.direct.asset.uri.search' => { label => $label, href => $href, use_restriction => $use, active => 't' } )
1141                         ->gather(1);
1142
1143         if (!$uri) {
1144             $uri = Fieldmapper::asset::uri->new;
1145             $uri->isnew( 1 );
1146             $uri->label($label);
1147             $uri->href($href);
1148             $uri->use_restriction($use);
1149         }
1150
1151         # see if we need to create a call number
1152         my $cn = $cstore
1153             ->request( 'open-ils.cstore.direct.asset.call_number.search' => { owning_lib => $org->id, record => $recid, label => '##URI##' } )
1154                         ->gather(1);
1155
1156         if (!$cn) {
1157             $cn = Fieldmapper::asset::call_number->new;
1158             $cn->isnew( 1 );
1159             $cn->id( $$max_cn++ );
1160             $cn->owning_lib( $org->id );
1161             $cn->record( $recid );
1162             $cn->label( '##URI##' );
1163         }
1164
1165         push @objects, { uri => $uri, call_number => $cn };
1166     }
1167
1168         $log->debug("Returning ".scalar(@objects)." URI nodes for record $recid");
1169         return @objects;
1170 }
1171
1172 sub get_uris_record {
1173         my $self = shift;
1174         my $client = shift;
1175         my $rec = shift;
1176
1177         OpenILS::Application::Ingest->post_init();
1178         my $r = OpenSRF::AppSession
1179                         ->create('open-ils.cstore')
1180                         ->request( "open-ils.cstore.direct.biblio.record_entry.retrieve" => $rec )
1181                         ->gather(1);
1182
1183         return undef unless ($r and $r->marc);
1184
1185         $client->respond($_) for (_extract_856_uris($r->id, $r->marc));
1186         return undef;
1187 }
1188 __PACKAGE__->register_method(
1189         api_name        => "open-ils.ingest.856_uri.record",
1190         method          => "get_uris_record",
1191         api_level       => 1,
1192         argc            => 1,
1193         stream          => 1,
1194 );
1195
1196 sub get_uris_object {
1197         my $self = shift;
1198         my $client = shift;
1199         my $obj = shift;
1200         my $max_cn = shift;
1201
1202         return undef unless ($obj and $obj->marc);
1203
1204         $client->respond($_) for (_extract_856_uris($obj->id, $obj->marc, \$max_cn));
1205         return undef;
1206 }
1207 __PACKAGE__->register_method(
1208         api_name        => "open-ils.ingest.856_uri.object",
1209         method          => "get_uris_object",
1210         api_level       => 1,
1211         argc            => 1,
1212         stream          => 1,
1213 );
1214
1215
1216 # --------------------------------------------------------------------------------
1217 # Fingerprinting
1218
1219 package OpenILS::Application::Ingest::Biblio::Fingerprint;
1220 use base qw/OpenILS::Application::Ingest/;
1221 use Unicode::Normalize;
1222 use OpenSRF::EX qw/:try/;
1223
1224 sub biblio_fingerprint_record {
1225         my $self = shift;
1226         my $client = shift;
1227         my $rec = shift;
1228
1229         OpenILS::Application::Ingest->post_init();
1230
1231         my $r = OpenSRF::AppSession
1232                         ->create('open-ils.cstore')
1233                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
1234                         ->gather(1);
1235
1236         return undef unless ($r and $r->marc);
1237
1238         my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
1239         $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
1240         $fp->{quality} = int($fp->{quality});
1241         return $fp;
1242 }
1243 __PACKAGE__->register_method(
1244         api_name        => "open-ils.ingest.fingerprint.record",
1245         method          => "biblio_fingerprint_record",
1246         api_level       => 1,
1247         argc            => 1,
1248 );
1249
1250 our $fp_script;
1251 sub biblio_fingerprint {
1252         my $self = shift;
1253         my $client = shift;
1254         my $xml = OpenILS::Application::Ingest::entityize(shift);
1255
1256         $log->internal("Got MARC [$xml]");
1257
1258         if(!$fp_script) {
1259                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1260                 my $conf = OpenSRF::Utils::SettingsClient->new;
1261
1262                 my $libs        = $conf->config_value(@pfx, 'script_path');
1263                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_fingerprint');
1264                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1265
1266                 $log->debug("Loading script $script_file for biblio fingerprinting...");
1267
1268                 $fp_script = new OpenILS::Utils::ScriptRunner
1269                         ( file          => $script_file,
1270                           paths         => $script_libs,
1271                           reset_count   => 100 );
1272         }
1273
1274         $fp_script->insert('environment' => {marc => $xml} => 1);
1275
1276         my $res = $fp_script->run || ($log->error( "Fingerprint script died!  $@" ) && return undef);
1277         $log->debug("Script for biblio fingerprinting completed successfully...");
1278
1279         return $res;
1280 }
1281 __PACKAGE__->register_method(
1282         api_name        => "open-ils.ingest.fingerprint.xml",
1283         method          => "biblio_fingerprint",
1284         api_level       => 1,
1285         argc            => 1,
1286 );
1287
1288 our $rd_script;
1289 sub biblio_descriptor {
1290         my $self = shift;
1291         my $client = shift;
1292         my $xml = OpenILS::Application::Ingest::entityize(shift);
1293
1294         $log->internal("Got MARC [$xml]");
1295
1296         if(!$rd_script) {
1297                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1298                 my $conf = OpenSRF::Utils::SettingsClient->new;
1299
1300                 my $libs        = $conf->config_value(@pfx, 'script_path');
1301                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_descriptor');
1302                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1303
1304                 $log->debug("Loading script $script_file for biblio descriptor extraction...");
1305
1306                 $rd_script = new OpenILS::Utils::ScriptRunner
1307                         ( file          => $script_file,
1308                           paths         => $script_libs,
1309                           reset_count   => 100 );
1310         }
1311
1312         $log->debug("Setting up environment for descriptor extraction script...");
1313         $rd_script->insert('environment.marc' => $xml => 1);
1314         $log->debug("Environment building complete...");
1315
1316         my $res = $rd_script->run || ($log->error( "Descriptor script died!  $@" ) && return undef);
1317         $log->debug("Script for biblio descriptor extraction completed successfully");
1318
1319     my $d1 = $res->date1;
1320     if ($d1 && $d1 ne '    ') {
1321         $d1 =~ tr/ux/00/;
1322         $res->date1( $d1 );
1323     }
1324
1325     my $d2 = $res->date2;
1326     if ($d2 && $d2 ne '    ') {
1327         $d2 =~ tr/ux/99/;
1328         $res->date2( $d2 );
1329     }
1330
1331         return $res;
1332 }
1333 __PACKAGE__->register_method(
1334         api_name        => "open-ils.ingest.descriptor.xml",
1335         method          => "biblio_descriptor",
1336         api_level       => 1,
1337         argc            => 1,
1338 );
1339
1340
1341 1;
1342