Open-ILS/src/perlmods/OpenILS/Application/Ingest.pm

   1 package OpenILS::Application::Ingest;
   2 use OpenILS::Application;
   3 use base qw/OpenILS::Application/;
   4
   5 use Unicode::Normalize;
   6 use OpenSRF::EX qw/:try/;
   7
   8 use OpenSRF::AppSession;
   9 use OpenSRF::Utils::SettingsClient;
  10 use OpenSRF::Utils::Logger qw/:level/;
  11
  12 use OpenILS::Utils::ScriptRunner;
  13 use OpenILS::Utils::Fieldmapper;
  14 use OpenSRF::Utils::JSON;
  15
  16 use OpenILS::Utils::Fieldmapper;
  17
  18 use XML::LibXML;
  19 use XML::LibXSLT;
  20 use Time::HiRes qw(time);
  21
  22 our %supported_formats = (
  23         mods32  => {ns => 'http://www.loc.gov/mods/v3'},
  24         mods3   => {ns => 'http://www.loc.gov/mods/v3'},
  25         mods    => {ns => 'http://www.loc.gov/mods/'},
  26         marcxml => {ns => 'http://www.loc.gov/MARC21/slim'},
  27         srw_dc  => {ns => 'info:srw/schema/1/dc-schema'},
  28         oai_dc  => {ns => 'http://www.openarchives.org/OAI/2.0/oai_dc/'},
  29         rdf_dc  => {ns => 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'},
  30         atom    => {ns => 'http://www.w3.org/2005/Atom'},
  31         rss091  => {ns => 'http://my.netscape.com/rdf/simple/0.9/'},
  32         rss092  => {ns => ''},
  33         rss093  => {ns => ''},
  34         rss094  => {ns => ''},
  35         rss10   => {ns => 'http://purl.org/rss/1.0/'},
  36         rss11   => {ns => 'http://purl.org/net/rss1.1#'},
  37         rss2    => {ns => ''},
  38 );
  39
  40
  41 my $log = 'OpenSRF::Utils::Logger';
  42
  43 my  $parser = XML::LibXML->new();
  44 my  $xslt = XML::LibXSLT->new();
  45
  46 my  $mods_sheet;
  47 my  $mads_sheet;
  48 my  $xpathset = {};
  49 sub initialize {}
  50 sub child_init {}
  51
  52 sub post_init {
  53
  54         unless (keys %$xpathset) {
  55                 $log->debug("Running post_init", DEBUG);
  56
  57                 my $xsldir = OpenSRF::Utils::SettingsClient->new->config_value(dirs => 'xsl');
  58
  59                 unless ($supported_formats{mods}{xslt}) {
  60                         $log->debug("Loading MODS XSLT", DEBUG);
  61                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS.xsl");
  62                         $supported_formats{mods}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  63                 }
  64
  65                 unless ($supported_formats{mods3}{xslt}) {
  66                         $log->debug("Loading MODS v3 XSLT", DEBUG);
  67                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS3.xsl");
  68                         $supported_formats{mods3}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  69                 }
  70
  71                 unless ($supported_formats{mods32}{xslt}) {
  72                         $log->debug("Loading MODS v32 XSLT", DEBUG);
  73                         my $xslt_doc = $parser->parse_file( $xsldir . "/MARC21slim2MODS32.xsl");
  74                         $supported_formats{mods32}{xslt} = $xslt->parse_stylesheet( $xslt_doc );
  75                 }
  76
  77                 my $req = OpenSRF::AppSession
  78                                 ->create('open-ils.cstore')
  79
  80                                 # XXX testing new metabib field use for faceting
  81                                 #->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { id => { '!=' => undef } } )
  82                                 ->request( 'open-ils.cstore.direct.config.metabib_field.search.atomic', { search_field => 't' } )
  83
  84                                 ->gather(1);
  85
  86                 if (ref $req and @$req) {
  87                         for my $f (@$req) {
  88                                 $xpathset->{ $f->field_class }->{ $f->name }->{xpath} = $f->xpath;
  89                                 $xpathset->{ $f->field_class }->{ $f->name }->{id} = $f->id;
  90                                 $xpathset->{ $f->field_class }->{ $f->name }->{format} = $f->format;
  91                                 $log->debug("Loaded XPath from DB: ".$f->field_class." => ".$f->name." : ".$f->xpath, DEBUG);
  92                         }
  93                 }
  94         }
  95 }
  96
  97 sub entityize {
  98         my $stuff = shift;
  99         my $form = shift;
 100
 101         if ($form eq 'D') {
 102                 $stuff = NFD($stuff);
 103         } else {
 104                 $stuff = NFC($stuff);
 105         }
 106
 107         $stuff =~ s/([\x{0080}-\x{fffd}])/sprintf('&#x%X;',ord($1))/sgoe;
 108         return $stuff;
 109 }
 110
 111 # --------------------------------------------------------------------------------
 112 # Biblio ingest
 113
 114 package OpenILS::Application::Ingest::Biblio;
 115 use base qw/OpenILS::Application::Ingest/;
 116 use Unicode::Normalize;
 117
 118 sub rw_biblio_ingest_single_object {
 119     my $self = shift;
 120     my $client = shift;
 121     my $bib = shift;
 122
 123     my ($blob) = $self->method_lookup("open-ils.ingest.full.biblio.object.readonly")->run($bib);
 124     return undef unless ($blob);
 125
 126     $bib->fingerprint( $blob->{fingerprint}->{fingerprint} );
 127     $bib->quality( $blob->{fingerprint}->{quality} );
 128
 129     my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
 130
 131     my $xact = $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 132     my $tmp;
 133
 134     # update uri stuff ...
 135
 136     # gather URI call numbers for this record
 137     my $uri_cns = $u->{call_number} = $cstore->request(
 138         'open-ils.cstore.direct.asset.call_number.id_list.atomic' => { record => $bib->id, label => '##URI##' }
 139     )->gather(1);
 140
 141     if (@$uri_cns) {
 142         # gather the maps for those call numbers
 143         my $uri_maps = $u->{call_number} = $cstore->request(
 144             'open-ils.cstore.direct.asset.uri_call_number_map.id_list.atomic' => { call_number => $uri_cns }
 145         )->gather(1);
 146
 147         # delete the old maps
 148         $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.delete' => $_ )->gather(1) for (@$uri_maps);
 149
 150         # and delete the call numbers if there are no more URIs
 151         if (!@{ $blob->{uri} }) {
 152             $cstore->request( 'open-ils.cstore.direct.asset.call_number.delete' => $_ )->gather(1) for (@$uri_cns);
 153         }
 154     }
 155
 156     # now, add CNs, URIs and maps
 157     my %new_cns_by_owner;
 158     my %new_uris_by_owner;
 159     for my $u ( @{ $blob->{uri} } ) {
 160
 161         my $owner = $u->{call_number}->owning_lib;
 162
 163         if ($u->{call_number}->isnew) {
 164             if ($new_cns_by_owner{$owner}) {
 165                 $u->{call_number} = $new_cns_by_owner{$owner};
 166             } else {
 167                 $u->{call_number}->clear_id;
 168                 $u->{call_number} = $new_cns_by_owner{$owner} = $cstore->request(
 169                     'open-ils.cstore.direct.asset.call_number.create' => $u->{call_number}
 170                 )->gather(1);
 171             }
 172         }
 173
 174         if ($u->{uri}->isnew) {
 175             if ($new_uris_by_owner{$owner}) {
 176                     $u->{uri} = $new_uris_by_owner{$owner};
 177             } else {
 178                     $u->{uri} = $new_uris_by_owner{$owner} = $cstore->request(
 179                     'open-ils.cstore.direct.asset.uri.create' => $u->{uri}
 180                 )->gather(1);
 181             }
 182         }
 183
 184         my $umap = Fieldmapper::asset::uri_call_number_map->new;
 185         $umap->uri($u->{uri}->id);
 186         $umap->call_number($u->{call_number}->id);
 187
 188             $cstore->request( 'open-ils.cstore.direct.asset.uri_call_number_map.create' => $umap )->gather(1) if (!$tmp);
 189     }
 190
 191         # update full_rec stuff ...
 192         $tmp = $cstore->request(
 193                 'open-ils.cstore.direct.metabib.full_rec.id_list.atomic',
 194                 { record => $bib->id }
 195         )->gather(1);
 196
 197         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.delete' => $_ )->gather(1) for (@$tmp);
 198         $cstore->request( 'open-ils.cstore.direct.metabib.full_rec.create' => $_ )->gather(1) for (@{ $blob->{full_rec} });
 199
 200         # update rec_descriptor stuff ...
 201         $tmp = $cstore->request(
 202                 'open-ils.cstore.direct.metabib.record_descriptor.id_list.atomic',
 203                 { record => $bib->id }
 204         )->gather(1);
 205
 206         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.delete' => $_ )->gather(1) for (@$tmp);
 207         $cstore->request( 'open-ils.cstore.direct.metabib.record_descriptor.create' => $blob->{descriptor} )->gather(1);
 208
 209         # deal with classed fields...
 210         for my $class ( qw/title author subject keyword series/ ) {
 211                 $tmp = $cstore->request(
 212                         "open-ils.cstore.direct.metabib.${class}_field_entry.id_list.atomic",
 213                         { source => $bib->id }
 214                 )->gather(1);
 215
 216                 $cstore->request( "open-ils.cstore.direct.metabib.${class}_field_entry.delete" => $_ )->gather(1) for (@$tmp);
 217         }
 218         for my $obj ( @{ $blob->{field_entries} } ) {
 219                 my $class = $obj->class_name;
 220                 $class =~ s/^Fieldmapper:://o;
 221                 $class =~ s/::/./go;
 222                 $cstore->request( "open-ils.cstore.direct.$class.create" => $obj )->gather(1);
 223         }
 224
 225         # update MR map ...
 226
 227         $tmp = $cstore->request(
 228                 'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 229                 { source => $bib->id }
 230         )->gather(1);
 231
 232         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.delete' => $_->id )->gather(1) for (@$tmp);
 233
 234         # get the old MRs
 235         my $old_mrs = $cstore->request(
 236                 'open-ils.cstore.direct.metabib.metarecord.search.atomic' => { id => [map { $_->metarecord } @$tmp] }
 237         )->gather(1) if (@$tmp);
 238
 239         $old_mrs = [] if (!ref($old_mrs));
 240
 241         my $mr;
 242         for my $m (@$old_mrs) {
 243                 if ($m->fingerprint eq $bib->fingerprint) {
 244                         $mr = $m;
 245                 } else {
 246                         my $others = $cstore->request(
 247                                 'open-ils.cstore.direct.metabib.metarecord_source_map.id_list.atomic' => { metarecord => $m->id }
 248                         )->gather(1);
 249
 250                         if (!@$others) {
 251                                 $cstore->request(
 252                                         'open-ils.cstore.direct.metabib.metarecord.delete' => $m->id
 253                                 )->gather(1);
 254                         }
 255
 256                         $m->isdeleted(1);
 257                 }
 258         }
 259
 260         my $holds;
 261         if (!$mr) {
 262                 # Get the matchin MR, if any.
 263                 $mr = $cstore->request(
 264                         'open-ils.cstore.direct.metabib.metarecord.search',
 265                         { fingerprint => $bib->fingerprint }
 266                 )->gather(1);
 267
 268                 $holds = $cstore->request(
 269                         'open-ils.cstore.direct.action.hold_request.search.atomic',
 270                         { hold_type => 'M', target => [ map { $_->id } grep { $_->isdeleted } @$old_mrs ] }
 271                 )->gather(1) if (@$old_mrs);
 272
 273                 if ($mr) {
 274                         for my $h (@$holds) {
 275                                 $h->target($mr);
 276                                 $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 277                                 $h->ischanged(1);
 278                         }
 279                 }
 280         }
 281
 282         if (!$mr) {
 283                 $mr = new Fieldmapper::metabib::metarecord;
 284                 $mr->fingerprint( $bib->fingerprint );
 285                 $mr->master_record( $bib->id );
 286                 $mr->id(
 287                         $cstore->request(
 288                                 "open-ils.cstore.direct.metabib.metarecord.create",
 289                                 $mr => { quiet => 'true' }
 290                         )->gather(1)
 291                 );
 292
 293                 for my $h (grep { !$_->ischanged } @$holds) {
 294                         $h->target($mr);
 295                         $cstore->request( 'open-ils.cstore.direct.action.hold_request.update' => $h )->gather(1);
 296                 }
 297         } else {
 298                 my $mrm = $cstore->request(
 299                         'open-ils.cstore.direct.metabib.metarecord_source_map.search.atomic',
 300                         { metarecord => $mr->id }
 301                 )->gather(1);
 302
 303                 if (@$mrm) {
 304                         my $best = $cstore->request(
 305                                 "open-ils.cstore.direct.biblio.record_entry.search",
 306                                 { id => [ map { $_->source } @$mrm ] },
 307                                 { 'select'      => { bre => [ qw/id quality/ ] },
 308                                 order_by        => { bre => "quality desc" },
 309                                 limit           => 1,
 310                                 }
 311                         )->gather(1);
 312
 313                         if ($best->quality > $bib->quality) {
 314                                 $mr->master_record($best->id);
 315                         } else {
 316                                 $mr->master_record($bib->id);
 317                         }
 318                 } else {
 319                         $mr->master_record($bib->id);
 320                 }
 321
 322                 $mr->clear_mods;
 323
 324                 $cstore->request( 'open-ils.cstore.direct.metabib.metarecord.update' => $mr )->gather(1);
 325         }
 326
 327         my $mrm = new Fieldmapper::metabib::metarecord_source_map;
 328         $mrm->source($bib->id);
 329         $mrm->metarecord($mr->id);
 330
 331         $cstore->request( 'open-ils.cstore.direct.metabib.metarecord_source_map.create' => $mrm )->gather(1);
 332         $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.update' => $bib )->gather(1);
 333
 334         $cstore->request( 'open-ils.cstore.transaction.commit' )->gather(1) || return undef;;
 335     $cstore->disconnect;
 336
 337         return $bib->id;
 338 }
 339 __PACKAGE__->register_method(
 340         api_name        => "open-ils.ingest.full.biblio.object",
 341         method          => "rw_biblio_ingest_single_object",
 342         api_level       => 1,
 343         argc            => 1,
 344 );
 345
 346 sub rw_biblio_ingest_single_record {
 347         my $self = shift;
 348         my $client = shift;
 349         my $rec = shift;
 350
 351         OpenILS::Application::Ingest->post_init();
 352         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 353         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 354
 355         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )->gather(1);
 356
 357         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 358         $cstore->disconnect;
 359
 360         return undef unless ($r and @$r);
 361
 362         return ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($r))[0];
 363 }
 364 __PACKAGE__->register_method(
 365         api_name        => "open-ils.ingest.full.biblio.record",
 366         method          => "rw_biblio_ingest_single_record",
 367         api_level       => 1,
 368         argc            => 1,
 369 );
 370
 371 sub rw_biblio_ingest_record_list {
 372         my $self = shift;
 373         my $client = shift;
 374         my @rec = ref($_[0]) ? @{ $_[0] } : @_ ;
 375
 376         OpenILS::Application::Ingest->post_init();
 377         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 378         $cstore->request('open-ils.cstore.transaction.begin')->gather(1);
 379
 380         my $r = $cstore->request( 'open-ils.cstore.direct.biblio.record_entry.search.atomic' => { id => $rec } )->gather(1);
 381
 382         $cstore->request('open-ils.cstore.transaction.rollback')->gather(1);
 383         $cstore->disconnect;
 384
 385         return undef unless ($r and @$r);
 386
 387         my $count = 0;
 388         $count += ($self->method_lookup("open-ils.ingest.full.biblio.object")->run($_))[0] for (@$r);
 389
 390         return $count;
 391 }
 392 __PACKAGE__->register_method(
 393         api_name        => "open-ils.ingest.full.biblio.record_list",
 394         method          => "rw_biblio_ingest_record_list",
 395         api_level       => 1,
 396         argc            => 1,
 397 );
 398
 399 sub ro_biblio_ingest_single_object {
 400         my $self = shift;
 401         my $client = shift;
 402         my $bib = shift;
 403         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 404     my $max_cn = shift;
 405     my $max_uri = shift;
 406
 407         my $cstore = OpenSRF::AppSession->connect( 'open-ils.cstore' );
 408
 409     if (!$max_cn) {
 410         my $cn = $cstore->request( 'open-ils.cstore.direct.asset.call_number.search' => { id => { '!=' => undef } }, { limit => 1, order_by => { acn => 'id desc' } } )->gather(1);
 411         $max_cn = int($cn->id) + 1000;
 412     }
 413
 414     if (!$max_uri) {
 415         my $cn = $cstore->request( 'open-ils.cstore.direct.asset.call_number.search' => { id => { '!=' => undef } }, { limit => 1, order_by => { acn => 'id desc' } } )->gather(1);
 416         $max_uri = int($cn->id) + 1000;
 417     }
 418
 419     $cstore->disconnect;
 420
 421         my $document = $parser->parse_string($xml);
 422
 423         my @uris = $self->method_lookup("open-ils.ingest.856_uri.object")->run($bib, $max_cn, $max_uri);
 424         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 425         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 426         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 427         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 428
 429         $_->source($bib->id) for (@mXfe);
 430         $_->record($bib->id) for (@mfr);
 431         $rd->record($bib->id) if ($rd);
 432
 433         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd, uri => \@uris };
 434 }
 435 __PACKAGE__->register_method(
 436         api_name        => "open-ils.ingest.full.biblio.object.readonly",
 437         method          => "ro_biblio_ingest_single_object",
 438         api_level       => 1,
 439         argc            => 1,
 440 );
 441
 442 sub ro_biblio_ingest_single_xml {
 443         my $self = shift;
 444         my $client = shift;
 445         my $xml = OpenILS::Application::Ingest::entityize(shift);
 446
 447         my $document = $parser->parse_string($xml);
 448
 449         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.biblio.xml")->run($document);
 450         my @mXfe = $self->method_lookup("open-ils.ingest.extract.field_entry.all.xml")->run($document);
 451         my ($fp) = $self->method_lookup("open-ils.ingest.fingerprint.xml")->run($xml);
 452         my ($rd) = $self->method_lookup("open-ils.ingest.descriptor.xml")->run($xml);
 453
 454         return { full_rec => \@mfr, field_entries => \@mXfe, fingerprint => $fp, descriptor => $rd };
 455 }
 456 __PACKAGE__->register_method(
 457         api_name        => "open-ils.ingest.full.biblio.xml.readonly",
 458         method          => "ro_biblio_ingest_single_xml",
 459         api_level       => 1,
 460         argc            => 1,
 461 );
 462
 463 sub ro_biblio_ingest_single_record {
 464         my $self = shift;
 465         my $client = shift;
 466         my $rec = shift;
 467
 468         OpenILS::Application::Ingest->post_init();
 469         my $r = OpenSRF::AppSession
 470                         ->create('open-ils.cstore')
 471                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 472                         ->gather(1);
 473
 474         return undef unless ($r and @$r);
 475
 476         my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($r->marc);
 477
 478         $_->source($rec) for (@{$res->{field_entries}});
 479         $_->record($rec) for (@{$res->{full_rec}});
 480         $res->{descriptor}->record($rec);
 481
 482         return $res;
 483 }
 484 __PACKAGE__->register_method(
 485         api_name        => "open-ils.ingest.full.biblio.record.readonly",
 486         method          => "ro_biblio_ingest_single_record",
 487         api_level       => 1,
 488         argc            => 1,
 489 );
 490
 491 sub ro_biblio_ingest_stream_record {
 492         my $self = shift;
 493         my $client = shift;
 494
 495         OpenILS::Application::Ingest->post_init();
 496
 497         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 498
 499         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 500
 501                 my $rec = $resp->content;
 502                 last unless (defined $rec);
 503
 504                 $log->debug("Running open-ils.ingest.full.biblio.record.readonly ...");
 505                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.record.readonly")->run($rec);
 506
 507                 $_->source($rec) for (@{$res->{field_entries}});
 508                 $_->record($rec) for (@{$res->{full_rec}});
 509
 510                 $client->respond( $res );
 511         }
 512
 513         return undef;
 514 }
 515 __PACKAGE__->register_method(
 516         api_name        => "open-ils.ingest.full.biblio.record_stream.readonly",
 517         method          => "ro_biblio_ingest_stream_record",
 518         api_level       => 1,
 519         stream          => 1,
 520 );
 521
 522 sub ro_biblio_ingest_stream_xml {
 523         my $self = shift;
 524         my $client = shift;
 525
 526         OpenILS::Application::Ingest->post_init();
 527
 528         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 529
 530         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 531
 532                 my $xml = $resp->content;
 533                 last unless (defined $xml);
 534
 535                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 536                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($xml);
 537
 538                 $client->respond( $res );
 539         }
 540
 541         return undef;
 542 }
 543 __PACKAGE__->register_method(
 544         api_name        => "open-ils.ingest.full.biblio.xml_stream.readonly",
 545         method          => "ro_biblio_ingest_stream_xml",
 546         api_level       => 1,
 547         stream          => 1,
 548 );
 549
 550 sub rw_biblio_ingest_stream_import {
 551         my $self = shift;
 552         my $client = shift;
 553
 554         OpenILS::Application::Ingest->post_init();
 555
 556         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 557
 558         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 559
 560                 my $bib = $resp->content;
 561                 last unless (defined $bib);
 562
 563                 $log->debug("Running open-ils.ingest.full.biblio.xml.readonly ...");
 564                 my ($res) = $self->method_lookup("open-ils.ingest.full.biblio.xml.readonly")->run($bib->marc);
 565
 566                 $_->source($bib->id) for (@{$res->{field_entries}});
 567                 $_->record($bib->id) for (@{$res->{full_rec}});
 568
 569                 $client->respond( $res );
 570         }
 571
 572         return undef;
 573 }
 574 __PACKAGE__->register_method(
 575         api_name        => "open-ils.ingest.full.biblio.bib_stream.import",
 576         method          => "rw_biblio_ingest_stream_import",
 577         api_level       => 1,
 578         stream          => 1,
 579 );
 580
 581
 582 # --------------------------------------------------------------------------------
 583 # Authority ingest
 584
 585 package OpenILS::Application::Ingest::Authority;
 586 use base qw/OpenILS::Application::Ingest/;
 587 use Unicode::Normalize;
 588
 589 sub ro_authority_ingest_single_object {
 590         my $self = shift;
 591         my $client = shift;
 592         my $bib = shift;
 593         my $xml = OpenILS::Application::Ingest::entityize($bib->marc);
 594
 595         my $document = $parser->parse_string($xml);
 596
 597         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 598
 599         $_->record($bib->id) for (@mfr);
 600
 601         return { full_rec => \@mfr };
 602 }
 603 __PACKAGE__->register_method(
 604         api_name        => "open-ils.ingest.full.authority.object.readonly",
 605         method          => "ro_authority_ingest_single_object",
 606         api_level       => 1,
 607         argc            => 1,
 608 );
 609
 610 sub ro_authority_ingest_single_xml {
 611         my $self = shift;
 612         my $client = shift;
 613         my $xml = OpenILS::Application::Ingest::entityize(shift);
 614
 615         my $document = $parser->parse_string($xml);
 616
 617         my @mfr = $self->method_lookup("open-ils.ingest.flat_marc.authority.xml")->run($document);
 618
 619         return { full_rec => \@mfr };
 620 }
 621 __PACKAGE__->register_method(
 622         api_name        => "open-ils.ingest.full.authority.xml.readonly",
 623         method          => "ro_authority_ingest_single_xml",
 624         api_level       => 1,
 625         argc            => 1,
 626 );
 627
 628 sub ro_authority_ingest_single_record {
 629         my $self = shift;
 630         my $client = shift;
 631         my $rec = shift;
 632
 633         OpenILS::Application::Ingest->post_init();
 634         my $r = OpenSRF::AppSession
 635                         ->create('open-ils.cstore')
 636                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 637                         ->gather(1);
 638
 639         return undef unless ($r and @$r);
 640
 641         my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($r->marc);
 642
 643         $_->record($rec) for (@{$res->{full_rec}});
 644         $res->{descriptor}->record($rec);
 645
 646         return $res;
 647 }
 648 __PACKAGE__->register_method(
 649         api_name        => "open-ils.ingest.full.authority.record.readonly",
 650         method          => "ro_authority_ingest_single_record",
 651         api_level       => 1,
 652         argc            => 1,
 653 );
 654
 655 sub ro_authority_ingest_stream_record {
 656         my $self = shift;
 657         my $client = shift;
 658
 659         OpenILS::Application::Ingest->post_init();
 660
 661         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 662
 663         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 664
 665                 my $rec = $resp->content;
 666                 last unless (defined $rec);
 667
 668                 $log->debug("Running open-ils.ingest.full.authority.record.readonly ...");
 669                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.record.readonly")->run($rec);
 670
 671                 $_->record($rec) for (@{$res->{full_rec}});
 672
 673                 $client->respond( $res );
 674         }
 675
 676         return undef;
 677 }
 678 __PACKAGE__->register_method(
 679         api_name        => "open-ils.ingest.full.authority.record_stream.readonly",
 680         method          => "ro_authority_ingest_stream_record",
 681         api_level       => 1,
 682         stream          => 1,
 683 );
 684
 685 sub ro_authority_ingest_stream_xml {
 686         my $self = shift;
 687         my $client = shift;
 688
 689         OpenILS::Application::Ingest->post_init();
 690
 691         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 692
 693         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 694
 695                 my $xml = $resp->content;
 696                 last unless (defined $xml);
 697
 698                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 699                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($xml);
 700
 701                 $client->respond( $res );
 702         }
 703
 704         return undef;
 705 }
 706 __PACKAGE__->register_method(
 707         api_name        => "open-ils.ingest.full.authority.xml_stream.readonly",
 708         method          => "ro_authority_ingest_stream_xml",
 709         api_level       => 1,
 710         stream          => 1,
 711 );
 712
 713 sub rw_authority_ingest_stream_import {
 714         my $self = shift;
 715         my $client = shift;
 716
 717         OpenILS::Application::Ingest->post_init();
 718
 719         my $ses = OpenSRF::AppSession->create('open-ils.cstore');
 720
 721         while (my ($resp) = $client->recv( count => 1, timeout => 5 )) {
 722
 723                 my $bib = $resp->content;
 724                 last unless (defined $bib);
 725
 726                 $log->debug("Running open-ils.ingest.full.authority.xml.readonly ...");
 727                 my ($res) = $self->method_lookup("open-ils.ingest.full.authority.xml.readonly")->run($bib->marc);
 728
 729                 $_->record($bib->id) for (@{$res->{full_rec}});
 730
 731                 $client->respond( $res );
 732         }
 733
 734         return undef;
 735 }
 736 __PACKAGE__->register_method(
 737         api_name        => "open-ils.ingest.full.authority.bib_stream.import",
 738         method          => "rw_authority_ingest_stream_import",
 739         api_level       => 1,
 740         stream          => 1,
 741 );
 742
 743
 744 # --------------------------------------------------------------------------------
 745 # MARC index extraction
 746
 747 package OpenILS::Application::Ingest::XPATH;
 748 use base qw/OpenILS::Application::Ingest/;
 749 use Unicode::Normalize;
 750
 751 # give this an XML documentElement and an XPATH expression
 752 sub xpath_to_string {
 753         my $xml = shift;
 754         my $xpath = shift;
 755         my $ns_uri = shift;
 756         my $ns_prefix = shift;
 757         my $unique = shift;
 758
 759         $xml->setNamespace( $ns_uri, $ns_prefix, 1 ) if ($ns_uri && $ns_prefix);
 760
 761         my $string = "";
 762
 763         # grab the set of matching nodes
 764         my @nodes = $xml->findnodes( $xpath );
 765         for my $value (@nodes) {
 766
 767                 # grab all children of the node
 768                 my @children = $value->childNodes();
 769                 for my $child (@children) {
 770
 771                         # add the childs content to the growing buffer
 772                         my $content = quotemeta($child->textContent);
 773                         next if ($unique && $string =~ /$content/);  # uniquify the values
 774                         $string .= $child->textContent . " ";
 775                 }
 776                 if( ! @children ) {
 777                         $string .= $value->textContent . " ";
 778                 }
 779         }
 780
 781     $string =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 782     $string =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 783
 784         return NFD($string);
 785 }
 786
 787 sub class_index_string_xml {
 788         my $self = shift;
 789         my $client = shift;
 790         my $xml = shift;
 791         my @classes = @_;
 792
 793         OpenILS::Application::Ingest->post_init();
 794         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
 795
 796         my %transform_cache;
 797
 798         for my $class (@classes) {
 799                 my $class_constructor = "Fieldmapper::metabib::${class}_field_entry";
 800                 for my $type ( keys %{ $xpathset->{$class} } ) {
 801
 802                         my $def = $xpathset->{$class}->{$type};
 803                         my $sf = $OpenILS::Application::Ingest::supported_formats{$def->{format}};
 804
 805                         my $document = $xml;
 806
 807                         if ($sf->{xslt}) {
 808                                 $document = $transform_cache{$def->{format}} || $sf->{xslt}->transform($xml);
 809                                 $transform_cache{$def->{format}} = $document;
 810                         }
 811
 812                         my $value =  xpath_to_string(
 813                                         $document->documentElement      => $def->{xpath},
 814                                         $sf->{ns}                       => $def->{format},
 815                                         1
 816                         );
 817
 818                         next unless $value;
 819
 820                         $value = NFD($value);
 821                         $value =~ s/\pM+//sgo;
 822                         $value =~ s/\pC+//sgo;
 823                         $value =~ s/\W+$//sgo;
 824
 825                         $value =~ s/\b\.+\b//sgo;
 826                         $value = lc($value);
 827
 828                         my $fm = $class_constructor->new;
 829                         $fm->value( $value );
 830                         $fm->field( $xpathset->{$class}->{$type}->{id} );
 831                         $client->respond($fm);
 832                 }
 833         }
 834         return undef;
 835 }
 836 __PACKAGE__->register_method(
 837         api_name        => "open-ils.ingest.field_entry.class.xml",
 838         method          => "class_index_string_xml",
 839         api_level       => 1,
 840         argc            => 2,
 841         stream          => 1,
 842 );
 843
 844 sub class_index_string_record {
 845         my $self = shift;
 846         my $client = shift;
 847         my $rec = shift;
 848         my @classes = shift;
 849
 850         OpenILS::Application::Ingest->post_init();
 851         my $r = OpenSRF::AppSession
 852                         ->create('open-ils.cstore')
 853                         ->request( 'open-ils.cstore.direct.authority.record_entry.retrieve' => $rec )
 854                         ->gather(1);
 855
 856         return undef unless ($r and @$r);
 857
 858         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, @classes)) {
 859                 $fm->source($rec);
 860                 $client->respond($fm);
 861         }
 862         return undef;
 863 }
 864 __PACKAGE__->register_method(
 865         api_name        => "open-ils.ingest.field_entry.class.record",
 866         method          => "class_index_string_record",
 867         api_level       => 1,
 868         argc            => 2,
 869         stream          => 1,
 870 );
 871
 872 sub all_index_string_xml {
 873         my $self = shift;
 874         my $client = shift;
 875         my $xml = shift;
 876
 877         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($xml, keys(%$xpathset))) {
 878                 $client->respond($fm);
 879         }
 880         return undef;
 881 }
 882 __PACKAGE__->register_method(
 883         api_name        => "open-ils.ingest.extract.field_entry.all.xml",
 884         method          => "all_index_string_xml",
 885         api_level       => 1,
 886         argc            => 1,
 887         stream          => 1,
 888 );
 889
 890 sub all_index_string_record {
 891         my $self = shift;
 892         my $client = shift;
 893         my $rec = shift;
 894
 895         OpenILS::Application::Ingest->post_init();
 896         my $r = OpenSRF::AppSession
 897                         ->create('open-ils.cstore')
 898                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
 899                         ->gather(1);
 900
 901         return undef unless ($r and @$r);
 902
 903         for my $fm ($self->method_lookup("open-ils.ingest.field_entry.class.xml")->run($r->marc, keys(%$xpathset))) {
 904                 $fm->source($rec);
 905                 $client->respond($fm);
 906         }
 907         return undef;
 908 }
 909 __PACKAGE__->register_method(
 910         api_name        => "open-ils.ingest.extract.field_entry.all.record",
 911         method          => "all_index_string_record",
 912         api_level       => 1,
 913         argc            => 1,
 914         stream          => 1,
 915 );
 916
 917 # --------------------------------------------------------------------------------
 918 # Flat MARC
 919
 920 package OpenILS::Application::Ingest::FlatMARC;
 921 use base qw/OpenILS::Application::Ingest/;
 922 use Unicode::Normalize;
 923
 924
 925 sub _marcxml_to_full_rows {
 926
 927         my $marcxml = shift;
 928         my $xmltype = shift || 'metabib';
 929
 930         my $type = "Fieldmapper::${xmltype}::full_rec";
 931
 932         my @ns_list;
 933
 934         my ($root) = $marcxml->findnodes('//*[local-name()="record"]');
 935
 936         for my $tagline ( @{$root->getChildrenByTagName("leader")} ) {
 937                 next unless $tagline;
 938
 939                 my $ns = $type->new;
 940
 941                 $ns->tag( 'LDR' );
 942                 my $val = $tagline->textContent;
 943                 $val = NFD($val);
 944                 $val =~ s/\pM+//sgo;
 945                 $val =~ s/\pC+//sgo;
 946                 $val =~ s/\W+$//sgo;
 947                 $ns->value( $val );
 948
 949                 push @ns_list, $ns;
 950         }
 951
 952         for my $tagline ( @{$root->getChildrenByTagName("controlfield")} ) {
 953                 next unless $tagline;
 954
 955                 my $ns = $type->new;
 956
 957                 $ns->tag( $tagline->getAttribute( "tag" ) );
 958                 my $val = $tagline->textContent;
 959                 $val = NFD($val);
 960                 $val =~ s/\pM+//sgo;
 961                 $val =~ s/\pC+//sgo;
 962                 $val =~ s/\W+$//sgo;
 963                 $ns->value( $val );
 964
 965                 push @ns_list, $ns;
 966         }
 967
 968         for my $tagline ( @{$root->getChildrenByTagName("datafield")} ) {
 969                 next unless $tagline;
 970
 971                 my $tag = $tagline->getAttribute( "tag" );
 972                 my $ind1 = $tagline->getAttribute( "ind1" );
 973                 my $ind2 = $tagline->getAttribute( "ind2" );
 974
 975                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
 976                         next unless $data;
 977
 978                         my $ns = $type->new;
 979
 980                         $ns->tag( $tag );
 981                         $ns->ind1( $ind1 );
 982                         $ns->ind2( $ind2 );
 983                         $ns->subfield( $data->getAttribute( "code" ) );
 984                         my $val = $data->textContent;
 985                         $val = NFD($val);
 986                         $val =~ s/\pM+//sgo;
 987                         $val =~ s/\pC+//sgo;
 988                         $val =~ s/\W+$//sgo;
 989             $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
 990             $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
 991                         $ns->value( lc($val) );
 992
 993                         push @ns_list, $ns;
 994                 }
 995
 996         if ($xmltype eq 'metabib' and $tag eq '245') {
 997                 $tag = 'tnf';
 998
 999                 for my $data ( @{$tagline->getChildrenByTagName('subfield')} ) {
1000                         next unless ($data and $data->getAttribute( "code" ) eq 'a');
1001
1002                         $ns = $type->new;
1003
1004                         $ns->tag( $tag );
1005                         $ns->ind1( $ind1 );
1006                         $ns->ind2( $ind2 );
1007                         $ns->subfield( $data->getAttribute( "code" ) );
1008                         my $val = substr( $data->textContent, $ind2 );
1009                         $val = NFD($val);
1010                         $val =~ s/\pM+//sgo;
1011                         $val =~ s/\pC+//sgo;
1012                         $val =~ s/\W+$//sgo;
1013                 $val =~ s/(\w+)\/(\w+)/$1 $2/sgo;
1014                 $val =~ s/(\d{4})-(\d{4})/$1 $2/sgo;
1015                         $ns->value( lc($val) );
1016
1017                         push @ns_list, $ns;
1018                 }
1019         }
1020         }
1021
1022         $log->debug("Returning ".scalar(@ns_list)." Fieldmapper nodes from $xmltype xml");
1023         return @ns_list;
1024 }
1025
1026 sub flat_marc_xml {
1027         my $self = shift;
1028         my $client = shift;
1029         my $xml = shift;
1030
1031         $log->debug("processing [$xml]");
1032
1033         $xml = $parser->parse_string(OpenILS::Application::Ingest::entityize($xml)) unless (ref $xml);
1034
1035         my $type = 'metabib';
1036         $type = 'authority' if ($self->api_name =~ /authority/o);
1037
1038         OpenILS::Application::Ingest->post_init();
1039
1040         $client->respond($_) for (_marcxml_to_full_rows($xml, $type));
1041         return undef;
1042 }
1043 __PACKAGE__->register_method(
1044         api_name        => "open-ils.ingest.flat_marc.authority.xml",
1045         method          => "flat_marc_xml",
1046         api_level       => 1,
1047         argc            => 1,
1048         stream          => 1,
1049 );
1050 __PACKAGE__->register_method(
1051         api_name        => "open-ils.ingest.flat_marc.biblio.xml",
1052         method          => "flat_marc_xml",
1053         api_level       => 1,
1054         argc            => 1,
1055         stream          => 1,
1056 );
1057
1058 sub flat_marc_record {
1059         my $self = shift;
1060         my $client = shift;
1061         my $rec = shift;
1062
1063         my $type = 'biblio';
1064         $type = 'authority' if ($self->api_name =~ /authority/o);
1065
1066         OpenILS::Application::Ingest->post_init();
1067         my $r = OpenSRF::AppSession
1068                         ->create('open-ils.cstore')
1069                         ->request( "open-ils.cstore.direct.${type}.record_entry.retrieve" => $rec )
1070                         ->gather(1);
1071
1072
1073         return undef unless ($r and $r->marc);
1074
1075         my @rows = $self->method_lookup("open-ils.ingest.flat_marc.$type.xml")->run($r->marc);
1076         for my $row (@rows) {
1077                 $client->respond($row);
1078                 $log->debug(OpenSRF::Utils::JSON->perl2JSON($row), DEBUG);
1079         }
1080         return undef;
1081 }
1082 __PACKAGE__->register_method(
1083         api_name        => "open-ils.ingest.flat_marc.biblio.record_entry",
1084         method          => "flat_marc_record",
1085         api_level       => 1,
1086         argc            => 1,
1087         stream          => 1,
1088 );
1089 __PACKAGE__->register_method(
1090         api_name        => "open-ils.ingest.flat_marc.authority.record_entry",
1091         method          => "flat_marc_record",
1092         api_level       => 1,
1093         argc            => 1,
1094         stream          => 1,
1095 );
1096
1097
1098 # --------------------------------------------------------------------------------
1099 # URI extraction
1100
1101 package OpenILS::Application::Ingest::Biblio::URI;
1102 use base qw/OpenILS::Application::Ingest/;
1103 use Unicode::Normalize;
1104 use OpenSRF::EX qw/:try/;
1105
1106
1107 sub _extract_856_uris {
1108
1109     my $rec   = shift;
1110         my $max_cn = shift;
1111         my $max_uri = shift;
1112         my @objects;
1113
1114     my $recid = $rec->id;
1115     my $marcxml = $rec->marc;
1116
1117         my $document = $parser->parse_string($marcxml);
1118         my @nodes = $document->findnodes('//*[local-name()="datafield" and @tag="856" and (@ind1="4" or @ind1="1") and (@ind2="0" or @ind2="1")]');
1119
1120     my $cstore = OpenSRF::AppSession->connect('open-ils.cstore');
1121
1122     for my $node (@nodes) {
1123         # first, is there a URI?
1124         my $href = $node->findvalue('*[local-name()="subfield" and @code="u"]/text()');
1125         next unless ($href);
1126
1127         # now, find the best possible label
1128         my $label = $node->findvalue('*[local-name()="subfield" and @code="y"]/text()');
1129         $label ||= $node->findvalue('*[local-name()="subfield" and @code="3"]/text()');
1130         $label ||= $href;
1131
1132         # look for use info
1133         my $use = $node->findvalue('*[local-name()="subfield" and @code="z"]/text()');
1134         $use ||= $node->findvalue('*[local-name()="subfield" and @code="2"]/text()');
1135         $use ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1136
1137         # moving on to the URI owner
1138         my $owner = $node->findvalue('*[local-name()="subfield" and @code="w"]/text()');
1139         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="n"]/text()');
1140         $owner ||= $node->findvalue('*[local-name()="subfield" and @code="9"]/text()'); # Evergreen special sauce
1141
1142         $owner =~ s/^.*?\((\w+)\).*$/$1/o; # unwrap first paren-enclosed string and then ...
1143
1144         # no owner? skip it :(
1145         next unless ($owner);
1146
1147         my $org = $cstore
1148             ->request( 'open-ils.cstore.direct.actor.org_unit.search' => { shortname => $owner} )
1149                         ->gather(1);
1150
1151         next unless ($org);
1152
1153         # now we can construct the uri object
1154         my $uri = $cstore
1155             ->request( 'open-ils.cstore.direct.asset.uri.search' => { label => $label, href => $href, use_restriction => $use, active => 't' } )
1156                         ->gather(1);
1157
1158         if (!$uri) {
1159             $uri = Fieldmapper::asset::uri->new;
1160             $uri->isnew( 1 );
1161             $uri->id( $$max_uri++ );
1162             $uri->label($label);
1163             $uri->href($href);
1164             $uri->active('t');
1165             $uri->use_restriction($use);
1166         }
1167
1168         # see if we need to create a call number
1169         my $cn = $cstore
1170             ->request( 'open-ils.cstore.direct.asset.call_number.search' => { owning_lib => $org->id, record => $recid, label => '##URI##' } )
1171                         ->gather(1);
1172
1173         if (!$cn) {
1174             $cn = Fieldmapper::asset::call_number->new;
1175             $cn->isnew( 1 );
1176             $cn->deleted('f');
1177             $cn->id( $$max_cn++ );
1178             $cn->owning_lib( $org->id );
1179             $cn->record( $recid );
1180             $cn->create_date( 'now' );
1181             $cn->creator( $rec->creator );
1182             $cn->editor( $rec->editor );
1183             $cn->edit_date( 'now' );
1184             $cn->label( '##URI##' );
1185         }
1186
1187         push @objects, { uri => $uri, call_number => $cn };
1188     }
1189
1190         $log->debug("Returning ".scalar(@objects)." URI nodes for record $recid");
1191         return @objects;
1192 }
1193
1194 sub get_uris_record {
1195         my $self = shift;
1196         my $client = shift;
1197         my $rec = shift;
1198
1199         OpenILS::Application::Ingest->post_init();
1200         my $r = OpenSRF::AppSession
1201                         ->create('open-ils.cstore')
1202                         ->request( "open-ils.cstore.direct.biblio.record_entry.retrieve" => $rec )
1203                         ->gather(1);
1204
1205         return undef unless ($r and $r->marc);
1206
1207         $client->respond($_) for (_extract_856_uris($r));
1208         return undef;
1209 }
1210 __PACKAGE__->register_method(
1211         api_name        => "open-ils.ingest.856_uri.record",
1212         method          => "get_uris_record",
1213         api_level       => 1,
1214         argc            => 1,
1215         stream          => 1,
1216 );
1217
1218 sub get_uris_object {
1219         my $self = shift;
1220         my $client = shift;
1221         my $obj = shift;
1222         my $max_cn = shift;
1223         my $max_uri = shift;
1224
1225         return undef unless ($obj and $obj->marc);
1226
1227         $client->respond($_) for (_extract_856_uris($obj, \$max_cn, \$max_uri));
1228         return undef;
1229 }
1230 __PACKAGE__->register_method(
1231         api_name        => "open-ils.ingest.856_uri.object",
1232         method          => "get_uris_object",
1233         api_level       => 1,
1234         argc            => 1,
1235         stream          => 1,
1236 );
1237
1238
1239 # --------------------------------------------------------------------------------
1240 # Fingerprinting
1241
1242 package OpenILS::Application::Ingest::Biblio::Fingerprint;
1243 use base qw/OpenILS::Application::Ingest/;
1244 use Unicode::Normalize;
1245 use OpenSRF::EX qw/:try/;
1246
1247 sub biblio_fingerprint_record {
1248         my $self = shift;
1249         my $client = shift;
1250         my $rec = shift;
1251
1252         OpenILS::Application::Ingest->post_init();
1253
1254         my $r = OpenSRF::AppSession
1255                         ->create('open-ils.cstore')
1256                         ->request( 'open-ils.cstore.direct.biblio.record_entry.retrieve' => $rec )
1257                         ->gather(1);
1258
1259         return undef unless ($r and $r->marc);
1260
1261         my ($fp) = $self->method_lookup('open-ils.ingest.fingerprint.xml')->run($r->marc);
1262         $log->debug("Returning [$fp] as fingerprint for record $rec", INFO);
1263         $fp->{quality} = int($fp->{quality});
1264         return $fp;
1265 }
1266 __PACKAGE__->register_method(
1267         api_name        => "open-ils.ingest.fingerprint.record",
1268         method          => "biblio_fingerprint_record",
1269         api_level       => 1,
1270         argc            => 1,
1271 );
1272
1273 our $fp_script;
1274 sub biblio_fingerprint {
1275         my $self = shift;
1276         my $client = shift;
1277         my $xml = OpenILS::Application::Ingest::entityize(shift);
1278
1279         $log->internal("Got MARC [$xml]");
1280
1281         if(!$fp_script) {
1282                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1283                 my $conf = OpenSRF::Utils::SettingsClient->new;
1284
1285                 my $libs        = $conf->config_value(@pfx, 'script_path');
1286                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_fingerprint');
1287                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1288
1289                 $log->debug("Loading script $script_file for biblio fingerprinting...");
1290
1291                 $fp_script = new OpenILS::Utils::ScriptRunner
1292                         ( file          => $script_file,
1293                           paths         => $script_libs,
1294                           reset_count   => 100 );
1295         }
1296
1297         $fp_script->insert('environment' => {marc => $xml} => 1);
1298
1299         my $res = $fp_script->run || ($log->error( "Fingerprint script died!  $@" ) && return undef);
1300         $log->debug("Script for biblio fingerprinting completed successfully...");
1301
1302         return $res;
1303 }
1304 __PACKAGE__->register_method(
1305         api_name        => "open-ils.ingest.fingerprint.xml",
1306         method          => "biblio_fingerprint",
1307         api_level       => 1,
1308         argc            => 1,
1309 );
1310
1311 our $rd_script;
1312 sub biblio_descriptor {
1313         my $self = shift;
1314         my $client = shift;
1315         my $xml = OpenILS::Application::Ingest::entityize(shift);
1316
1317         $log->internal("Got MARC [$xml]");
1318
1319         if(!$rd_script) {
1320                 my @pfx = ( "apps", "open-ils.ingest","app_settings" );
1321                 my $conf = OpenSRF::Utils::SettingsClient->new;
1322
1323                 my $libs        = $conf->config_value(@pfx, 'script_path');
1324                 my $script_file = $conf->config_value(@pfx, 'scripts', 'biblio_descriptor');
1325                 my $script_libs = (ref($libs)) ? $libs : [$libs];
1326
1327                 $log->debug("Loading script $script_file for biblio descriptor extraction...");
1328
1329                 $rd_script = new OpenILS::Utils::ScriptRunner
1330                         ( file          => $script_file,
1331                           paths         => $script_libs,
1332                           reset_count   => 100 );
1333         }
1334
1335         $log->debug("Setting up environment for descriptor extraction script...");
1336         $rd_script->insert('environment.marc' => $xml => 1);
1337         $log->debug("Environment building complete...");
1338
1339         my $res = $rd_script->run || ($log->error( "Descriptor script died!  $@" ) && return undef);
1340         $log->debug("Script for biblio descriptor extraction completed successfully");
1341
1342     my $d1 = $res->date1;
1343     if ($d1 && $d1 ne '    ') {
1344         $d1 =~ tr/ux/00/;
1345         $res->date1( $d1 );
1346     }
1347
1348     my $d2 = $res->date2;
1349     if ($d2 && $d2 ne '    ') {
1350         $d2 =~ tr/ux/99/;
1351         $res->date2( $d2 );
1352     }
1353
1354         return $res;
1355 }
1356 __PACKAGE__->register_method(
1357         api_name        => "open-ils.ingest.descriptor.xml",
1358         method          => "biblio_descriptor",
1359         api_level       => 1,
1360         argc            => 1,
1361 );
1362
1363
1364 1;
1365