]> git.evergreen-ils.org Git - Evergreen.git/blob - Open-ILS/src/perlmods/OpenILS/WWW/TemplateBatchBibUpdate.pm
Whitespace. gah.
[Evergreen.git] / Open-ILS / src / perlmods / OpenILS / WWW / TemplateBatchBibUpdate.pm
1 package OpenILS::WWW::TemplateBatchBibUpdate;
2 use strict;
3 use warnings;
4 use bytes;
5
6 use Apache2::Log;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log);
8 use APR::Const    -compile => qw(:error SUCCESS);
9 use APR::Table;
10
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
14 use CGI;
15 use Data::Dumper;
16 use Text::CSV;
17
18 use OpenSRF::EX qw(:try);
19 use OpenSRF::Utils qw/:datetime/;
20 use OpenSRF::Utils::Cache;
21 use OpenSRF::System;
22 use OpenSRF::AppSession;
23 use XML::LibXML;
24 use XML::LibXSLT;
25
26 use Encode;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
30
31 use MARC::Record;
32 use MARC::File::XML;
33
34 use UNIVERSAL::require;
35
36 our @formats = qw/USMARC UNIMARC XML BRE/;
37
38 # set the bootstrap config and template include directory when
39 # this module is loaded
40 my $bootstrap;
41
42 sub import {
43     my $self = shift;
44     $bootstrap = shift;
45 }
46
47
48 sub child_init {
49     OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50     Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
51 }
52
53 sub handler {
54     my $r = shift;
55     my $cgi = new CGI;
56
57     my $authid = $cgi->cookie('ses') || $cgi->param('ses');
58     my $usr = verify_login($authid);
59     return show_template($r) unless ($usr);
60
61     my $template = $cgi->param('template');
62     return show_template($r) unless ($template);
63
64
65     my $rsource = $cgi->param('recordSource');
66     # find some IDs ...
67     my @records;
68
69     if ($rsource eq 'r') {
70         @records = map { $_ ? ($_) : () } $cgi->param('recid');
71     }
72
73     if ($rsource eq 'c') { # try for a file
74         my $file = $cgi->param('idfile');
75         if ($file) {
76             my $col = $cgi->param('idcolumn') || 0;
77             my $csv = new Text::CSV;
78
79             while (<$file>) {
80                 $csv->parse($_);
81                 my @data = $csv->fields;
82                 my $id = $data[$col];
83                 $id =~ s/\D+//o;
84                 next unless ($id);
85                 push @records, $id;
86             }
87         }
88     }
89
90     my $e = OpenSRF::AppSession->connect('open-ils.cstore');
91     $e->request('open-ils.cstore.transaction.begin')->gather(1);
92
93     # still no records ...
94     my $container = $cgi->param('containerid');
95     if ($rsource eq 'b') {
96         if ($container) {
97             my $bucket = $e->request(
98                 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
99                 $container
100             )->gather(1);
101             unless($bucket) {
102                 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
103                 $e->disconnect;
104                 $r->log->error("No such bucket $container");
105                 $logger->error("No such bucket $container");
106                 return Apache2::Const::NOT_FOUND;
107             }
108             my $recs = $e->request(
109                 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
110                 { bucket => $container }
111             )->gather(1);
112             @records = map { ($_->target_biblio_record_entry) } @$recs;
113         }
114     }
115
116     unless (@records) {
117         $e->request('open-ils.cstore.transaction.rollback')->gather(1);
118         $e->disconnect;
119         return show_template($r);
120     }
121
122     # we have a template and some record ids, so...
123
124     # insert the template record
125     my $min_id = $e->request(
126         'open-ils.cstore.json_query',
127         { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
128     )->gather(1)->{id} - 1;
129
130     warn "new template bib id = $min_id\n";
131
132     my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
133     $tmpl_rec->id($min_id);
134     $tmpl_rec->deleted('t');
135     $tmpl_rec->active('f');
136     $tmpl_rec->marc($template);
137     $tmpl_rec->creator($usr->id);
138     $tmpl_rec->editor($usr->id);
139
140     warn "about to create bib $min_id\n";
141     $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
142
143     # create the new container for the records and the template
144     my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
145     $bucket->owner($usr->id);
146     $bucket->btype('template_merge');
147
148     my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
149     $bucket->name($bname);
150
151     $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
152
153     # create items in the bucket
154     my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
155     $item->bucket($bucket->id);
156     $item->target_biblio_record_entry($min_id);
157
158     $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
159
160     my %seen;
161     for my $r (@records) {
162         next if ($seen{$r});
163         $item->target_biblio_record_entry($r);
164         $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
165         $seen{$r}++;
166     }
167
168     $e->request('open-ils.cstore.transaction.commit')->gather(1);
169     $e->disconnect;
170
171     # fire the background bucket processor
172     my $cache_key = OpenSRF::AppSession
173         ->create('open-ils.cat')
174         ->request('open-ils.cat.container.template_overlay.background', $authid, $bucket->id)
175         ->gather(1);
176
177     return show_processing_template($r, $bucket->id, \@records, $cache_key);
178 }
179
180 sub verify_login {
181         my $auth_token = shift;
182         return undef unless $auth_token;
183
184         my $user = OpenSRF::AppSession
185                 ->create("open-ils.auth")
186                 ->request( "open-ils.auth.session.retrieve", $auth_token )
187                 ->gather(1);
188
189         if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
190                 return undef;
191         }
192
193         return $user if ref($user);
194         return undef;
195 }
196
197 sub show_processing_template {
198     my $r = shift;
199     my $bid = shift;
200     my $recs = shift;
201     my $cache_key = shift;
202
203     my $rec_string = @$recs;
204
205     $r->content_type('text/html');
206     $r->print(<<HTML);
207 <html xmlns="http://www.w3.org/1999/xhtml">
208
209     <head>
210         <title>Merging records...</title>
211         <style type="text/css">
212             \@import '/js/dojo/dojo/resources/dojo.css';
213             \@import '/js/dojo/dijit/themes/tundra/tundra.css';
214             .hide_me { display: none; visibility: hidden; }
215             th       { font-weight: bold; }
216         </style>
217
218         <script type="text/javascript">
219             var djConfig= {
220                 isDebug: false,
221                 parseOnLoad: true,
222                 AutoIDL: ['aou','aout','pgt','au','cbreb']
223             }
224         </script>
225
226         <script src='/js/dojo/dojo/dojo.js'></script>
227         <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
228
229         <script type="text/javascript">
230
231             dojo.require('fieldmapper.AutoIDL');
232             dojo.require('fieldmapper.dojoData');
233             dojo.require('openils.User');
234             dojo.require('openils.CGI');
235             dojo.require('openils.widget.ProgressDialog');
236
237             var cgi = new openils.CGI();
238             var u = new openils.User({ authcookie : 'ses' });
239
240             dojo.addOnLoad(function () {
241                 progress_dialog.show(true);
242                 progress_dialog.update({maximum:$rec_string});
243
244                 var interval;
245                 interval = setInterval( function() {
246                     fieldmapper.standardRequest(
247                         ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
248                         { async : false,
249                           params: [ u.authtoken, 'res_list' ],
250                           onerror : function (r) { progress_dialog.hide(); },
251                           onresponse : function (r) {
252                             var counter = { success : 0, fail : 0, total : 0 };
253                             dojo.forEach( openils.Util.readResponse(r), function(x) {
254                                 if (x.complete) {
255                                     clearInterval(interval);
256                                     progress_dialog.hide();
257                                     if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
258                                     else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
259                                 } else {
260                                     counter.total++;
261                                     switch (x.success) {
262                                         case 't':
263                                             counter.success++;
264                                             break;
265                                         default:
266                                             counter.fail++;
267                                             break;
268                                     }
269                                 }
270                             });
271
272                             // update the progress dialog
273                             progress_dialog.update({progress:counter.total});
274                             dojo.byId('success_count').innerHTML = counter.success;
275                             dojo.byId('fail_count').innerHTML = counter.fail;
276                             dojo.byId('total_count').innerHTML = counter.total;
277                           }
278                         }
279                     );
280                 }, 1000);
281
282             });
283         </script>
284     </head>
285
286     <body style="margin:10px;" class='tundra'>
287         <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
288
289         <table style="width:100%; margin-top:100px;">
290             <th>
291                 <td>Status</td>
292                 <td>Record Count</td>
293             </th>
294             <tr>
295                 <td>Success</td>
296                 <td id='success_count'></td>
297             </tr>
298             <tr>
299                 <td>Failure</td>
300                 <td id='fail_count'></td>
301             </tr>
302             <tr>
303                 <td></td>
304                 <td id='total_count'></td>
305             </tr>
306         </table>
307
308         <div id='complete_msg'></div>
309
310     </body>
311 </html>
312 HTML
313
314     return Apache2::Const::OK;
315 }
316
317
318 sub show_template {
319     my $r = shift;
320
321     $r->content_type('text/html');
322     $r->print(<<'HTML');
323 <html xmlns="http://www.w3.org/1999/xhtml">
324
325     <head>
326         <title>Merge Template Builder</title>
327         <style type="text/css">
328             @import '/js/dojo/dojo/resources/dojo.css';
329             @import '/js/dojo/dijit/themes/tundra/tundra.css';
330             .hide_me { display: none; visibility: hidden; }
331             table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
332             table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
333         </style>
334
335         <script type="text/javascript">
336             var djConfig= {
337                 isDebug: false,
338                 parseOnLoad: true,
339                 AutoIDL: ['aou','aout','pgt','au','cbreb']
340             }
341         </script>
342
343         <script src='/js/dojo/dojo/dojo.js'></script>
344         <!-- <script src="/js/dojo/dojo/openils_dojo.js"></script> -->
345
346         <script type="text/javascript">
347
348             dojo.require('dojo.data.ItemFileReadStore');
349             dojo.require('dijit.form.Form');
350             dojo.require('dijit.form.NumberSpinner');
351             dojo.require('dijit.form.FilteringSelect');
352             dojo.require('dijit.form.TextBox');
353             dojo.require('dijit.form.Textarea');
354             dojo.require('dijit.form.Button');
355             dojo.require('MARC.Batch');
356             dojo.require('fieldmapper.AutoIDL');
357             dojo.require('fieldmapper.dojoData');
358             dojo.require('openils.User');
359             dojo.require('openils.CGI');
360
361             var cgi = new openils.CGI();
362             var u = new openils.User({ authcookie : 'ses' });
363
364             var bucketStore = new dojo.data.ItemFileReadStore(
365                 { data : cbreb.toStoreData(
366                         fieldmapper.standardRequest(
367                             ['open-ils.actor','open-ils.actor.container.retrieve_by_class'],
368                             [u.authtoken, u.user.id(), 'biblio', 'staff_client']
369                         )
370                     )
371                 }
372             );
373
374             function render_preview () {
375                 var rec = ruleset_to_record();
376                 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
377             }
378
379             function render_from_template () {
380                 var kid_number = dojo.byId('ruleList').childNodes.length;
381                 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
382
383                 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
384                     onChange : function (val) {
385                         switch (val) {
386                             case 'a':
387                             case 'r':
388                                 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
389                                 break;
390                             default :
391                                 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
392                         };
393                         render_preview();
394                     }
395                 })[0];
396
397                 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
398                     onChange : render_preview
399                 })[0];
400
401
402                 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
403                     onChange : function (newtag) {
404                         var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
405                         var current_marc = md.attr('value');
406
407                         if (newtag.length == 3) {
408                             if (current_marc.length == 0) newtag += ' \\\\';
409                             if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
410                         }
411                         md.attr('value', current_marc);
412                         render_preview();
413                     }
414                 })[0];
415
416                 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
417                     onChange : function (newsf) {
418                         var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
419                         var current_marc = md.attr('value');
420                         var sf_list = newsf.split('');
421
422                         for (var i in sf_list) {
423                             var re = '\\$' + sf_list[i];
424                             if (current_marc.match(re)) continue;
425                             current_marc += '$' + sf_list[i];
426                         }
427
428                         md.attr('value', current_marc);
429                         render_preview();
430                     }
431                 })[0];
432
433                 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
434                     onChange : render_preview
435                 })[0];
436
437                 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
438                     onChange : render_preview
439                 })[0];
440
441                 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
442                     onClick : function() {
443                         dojo.addClass(
444                             dojo.byId('ruleList').childNodes[kid_number],
445                             'hide_me'
446                         );
447                         render_preview();
448                     }
449                 })[0];
450
451                 dojo.place(clone,'ruleList');
452             }
453
454             function ruleset_to_record () {
455                 var rec = new MARC.Record ({ delimiter : '$' });
456
457                 dojo.forEach( 
458                     dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
459                         if (node.className.match(/hide_me/)) return false;
460                         return true;
461                     }),
462                     function (tbl) {
463                         var rule_tag = new MARC.Field ({
464                             tag : '905',
465                             ind1 : ' ',
466                             ind2 : ' '
467                         });
468                         var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
469                         rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
470
471                         var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
472                         if (reSF) {
473                             var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
474                             rule_txt += '[' + reSF + '~' + reRE + ']';
475                         }
476
477                         var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
478                         rule_tag.addSubfields( rtype, rule_txt )
479                         rec.appendFields( rule_tag );
480
481                         if (rtype == 'a' || rtype == 'r') {
482                             rec.appendFields(
483                                 new MARC.Record ({
484                                     delimiter : '$',
485                                     marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
486                                 }).fields[0]
487                             );
488                         }
489                     }
490                 );
491
492                 return rec;
493             }
494         </script>
495     </head>
496
497     <body style="margin:10px;" class='tundra'>
498
499         <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
500                 <script type='dojo/method' event='onSubmit'>
501                     var rec = ruleset_to_record();
502
503                     if (rec.subfield('905','r') == '') { // no-op to force replace mode
504                         rec.appendFields(
505                             new MARC.Field ({
506                                 tag : '905',
507                                 ind1 : ' ',
508                                 ind2 : ' ',
509                                 subfields : [['r','901c']]
510                             })
511                         );
512                     }
513
514                     dojo.byId('template_value').value = rec.toXmlString();
515                     return true;
516                 </script>
517
518             <input type='hidden' id='template_value' name='template'/>
519
520             <label for='inputTypeSelect'>Record source:</label>
521             <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
522                 <script type='dojo/method' event='onChange' args="val">
523                     switch (val) {
524                         case 'b':
525                             dojo.removeClass('bucketListContainer','hide_me');
526                             dojo.addClass('csvContainer','hide_me');
527                             dojo.addClass('recordContainer','hide_me');
528                             break;
529                         case 'c':
530                             dojo.addClass('bucketListContainer','hide_me');
531                             dojo.removeClass('csvContainer','hide_me');
532                             dojo.addClass('recordContainer','hide_me');
533                             break;
534                         case 'r':
535                             dojo.addClass('bucketListContainer','hide_me');
536                             dojo.addClass('csvContainer','hide_me');
537                             dojo.removeClass('recordContainer','hide_me');
538                             break;
539                     };
540                 </script>
541                 <script type='dojo/method' event='postCreate'>
542                     if (cgi.param('recordSource')) {
543                         this.attr('value',cgi.param('recordSource'));
544                         this.onChange(cgi.param('recordSource'));
545                     }
546                 </script>
547                 <option value='b'>a Bucket</option>
548                 <option value='c'>a CSV File</option>
549                 <option value='r'>a specific record ID</option>
550             </select>
551
552             <table style='margin:10px; margin-bottom:20px;'>
553 <!--
554                 <tr>
555                     <th>Merge template name (optional):</th>
556                     <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
557                 </tr>
558 -->
559                 <tr class='' id='bucketListContainer'>
560                     <td>Bucket named: 
561                         <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
562                             <script type='dojo/method' event='postCreate'>
563                                 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
564                             </script>
565                         </div>
566                     </td>
567                 </tr>
568                 <tr class='hide_me' id='csvContainer'>
569                     <td>
570                         Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of: 
571                         <input id='idfile' type="file" name="idfile"/>
572                         <br/>
573                         <br/>
574                         Columns are numbered starting at 0.  For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
575                     </td>
576                 </tr>
577                 <tr class='hide_me' id='recordContainer'>
578                     <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
579                 </tr>
580             </table>
581
582             <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
583
584             <br/>
585             <br/>
586
587         </div> <!-- end of the form -->
588
589         <hr/>
590         <table style='width: 100%'>
591             <tr>
592                 <td style='width: 50%'><div id='ruleList'></div></td>
593                 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
594             </tr>
595         </table>
596
597         <button dojoType='dijit.form.Button'>Add Merge Rule
598             <script type='dojo/connect' event='onClick'>render_from_template()</script>
599             <script type='dojo/method' event='postCreate'>render_from_template()</script>
600         </button>
601
602         <div class='hide_me' id='ruleTemplate'>
603         <div name='ruleTable'>
604             <table class='ruleTable'>
605                 <tbody>
606                     <tr>
607                         <th style="text-align:center;">Rule Setup</th>
608                         <th style="text-align:center;">Data</th>
609                         <th style="text-align:center;">Help</th>
610                     </tr>
611                     <tr>
612                         <th>Action (Rule Type)</th>
613                         <td name='typeSelectContainer'>
614                             <select name='typeSelect'>
615                                 <option value='r'>Replace</option>
616                                 <option value='a'>Add</option>
617                                 <option value='d'>Delete</option>
618                             </select>
619                         </td>
620                         <td>How to change the existing records</td>
621                     </tr>
622                     <tr>
623                         <th>MARC Tag</th>
624                         <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
625                         <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
626                     </td>
627                     <tr>
628                         <th>Subfields (optional)</th>
629                         <td name='sfContainer'><input name='sf' type='text'/></td>
630                         <td>No spaces, no delimiters, eg: abcnp</td>
631                     </tr>
632                     <tr>
633                         <th>MARC Data</th>
634                         <td name='marcDataContainer'><input name='marcData' type='text'/></td>
635                         <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
636                     </tr>
637                     <tr>
638                         <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
639                     </tr>
640                     <tr>
641                         <th>Subfield</th>
642                         <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
643                         <td>A single subfield code, no delimiters, eg: a</td>
644                     <tr>
645                         <th>Regular Expression</th>
646                         <td name='matchREContainer'><input name='matchRE' type='text'/></td>
647                         <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
648                             for an explanation of Regular Expressions.
649                         </td>
650                     </tr>
651                     <tr>
652                         <td colspan='3' style='padding-top: 20px; text-align: center;'>
653                             <button name='removeButton'>Remove this Template Rule</button>
654                         </td>
655                     </tr>
656                 </tbody>
657             </table>
658         <hr/>
659         </div>
660         </div>
661
662     </body>
663 </html>
664 HTML
665
666     return Apache2::Const::OK;
667 }
668
669 1;
670
671