1 package OpenILS::WWW::TemplateBatchBibUpdate;
7 use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND HTTP_BAD_REQUEST :log);
8 use APR::Const -compile => qw(:error SUCCESS);
11 use Apache2::RequestRec ();
12 use Apache2::RequestIO ();
13 use Apache2::RequestUtil;
18 use OpenSRF::EX qw(:try);
19 use OpenILS::Utils::DateTime qw/:datetime/;
20 use OpenSRF::Utils::Cache;
22 use OpenSRF::AppSession;
27 use Unicode::Normalize;
28 use OpenILS::Utils::Fieldmapper;
29 use OpenSRF::Utils::Logger qw/$logger/;
32 use MARC::File::XML ( BinaryEncoding => 'UTF-8' );
34 use UNIVERSAL::require;
36 our @formats = qw/USMARC UNIMARC XML BRE/;
38 # set the bootstrap config and template include directory when
39 # this module is loaded
49 OpenSRF::System->bootstrap_client( config_file => $bootstrap );
50 Fieldmapper->import(IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
51 return Apache2::Const::OK;
58 my $authid = $cgi->cookie('eg.auth.token') || $cgi->cookie('ses') || $cgi->param('ses');
59 if ($authid =~ /^"(.+)"$/) { # came from eg2 login, is json encoded
63 # Avoid sending the HTML to the caller. Final response will
64 # will just be the cache key or HTTP_BAD_REQUEST on error.
65 my $skipui = $cgi->param('skipui');
67 my $usr = verify_login($authid);
68 return show_template($r, $skipui) unless ($usr);
70 my $template = $cgi->param('template');
71 return show_template($r, $skipui) unless ($template);
74 my $rsource = $cgi->param('recordSource');
75 my $xact_per = $cgi->param('xactPerRecord');
80 if ($rsource eq 'r') {
81 @records = map { $_ ? ($_) : () } $cgi->param('recid');
84 if ($rsource eq 'c') { # try for a file
85 my $file = $cgi->param('idfile');
87 my $col = $cgi->param('idcolumn') || 0;
88 my $csv = new Text::CSV;
92 my @data = $csv->fields;
101 my $e = OpenSRF::AppSession->connect('open-ils.cstore');
102 $e->request('open-ils.cstore.transaction.begin')->gather(1);
103 $e->request('open-ils.cstore.set_audit_info', $authid, $usr->id, $usr->wsid)->gather(1);
105 # still no records ...
106 my $container = $cgi->param('containerid');
107 if ($rsource eq 'b') {
109 my $bucket = $e->request(
110 'open-ils.cstore.direct.container.biblio_record_entry_bucket.retrieve',
114 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
116 $r->log->error("No such bucket $container");
117 $logger->error("No such bucket $container");
118 return Apache2::Const::NOT_FOUND;
120 my $recs = $e->request(
121 'open-ils.cstore.direct.container.biblio_record_entry_bucket_item.search.atomic',
122 { bucket => $container }
124 @records = map { ($_->target_biblio_record_entry) } @$recs;
129 $e->request('open-ils.cstore.transaction.rollback')->gather(1);
131 return show_template($r, $skipui);
134 # we have a template and some record ids, so...
136 # insert the template record
137 my $min_id = $e->request(
138 'open-ils.cstore.json_query',
139 { select => { bre => [{ column => 'id', transform => 'min', aggregate => 1}] }, from => 'bre' }
140 )->gather(1)->{id} - 1;
142 warn "new template bib id = $min_id\n";
144 my $tmpl_rec = Fieldmapper::biblio::record_entry->new;
145 $tmpl_rec->id($min_id);
146 $tmpl_rec->deleted('t');
147 $tmpl_rec->active('f');
148 $tmpl_rec->marc($template);
149 $tmpl_rec->creator($usr->id);
150 $tmpl_rec->editor($usr->id);
152 warn "about to create bib $min_id\n";
153 $e->request('open-ils.cstore.direct.biblio.record_entry.create', $tmpl_rec )->gather(1);
155 # create the new container for the records and the template
156 my $bucket = Fieldmapper::container::biblio_record_entry_bucket->new;
157 $bucket->owner($usr->id);
158 $bucket->btype('template_merge');
160 my $bname = $cgi->param('bname') || 'Temporary Merge Bucket '. localtime() . ' ' . $usr->id;
161 $bucket->name($bname);
163 $bucket = $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket.create', $bucket )->gather(1);
165 # create items in the bucket
166 my $item = Fieldmapper::container::biblio_record_entry_bucket_item->new;
167 $item->bucket($bucket->id);
168 $item->target_biblio_record_entry($min_id);
170 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
173 for my $r (@records) {
175 $item->target_biblio_record_entry($r);
176 $e->request('open-ils.cstore.direct.container.biblio_record_entry_bucket_item.create', $item )->gather(1);
180 $e->request('open-ils.cstore.transaction.commit')->gather(1);
183 # fire the background bucket processor
184 my $cache_key = OpenSRF::AppSession
185 ->create('open-ils.cat')
186 ->request('open-ils.cat.container.template_overlay.background',
187 $authid, $bucket->id, undef, {xact_per_record => $xact_per})
190 return show_processing_template(
191 $r, $bucket->id, \@records, $cache_key, $skipui);
195 my $auth_token = shift;
196 return undef unless $auth_token;
198 my $user = OpenSRF::AppSession
199 ->create("open-ils.auth")
200 ->request( "open-ils.auth.session.retrieve", $auth_token )
203 if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) {
207 return $user if ref($user);
211 sub show_processing_template {
215 my $cache_key = shift;
219 $r->content_type('text/plain');
220 $r->print($cache_key);
221 return Apache2::Const::OK;
224 my $rec_string = @$recs;
226 $r->content_type('text/html');
228 <html xmlns="http://www.w3.org/1999/xhtml">
231 <title>Merging records...</title>
232 <style type="text/css">
233 \@import '/js/dojo/dojo/resources/dojo.css';
234 \@import '/js/dojo/dijit/themes/tundra/tundra.css';
235 .hide_me { display: none; visibility: hidden; }
236 th { font-weight: bold; }
239 <script type="text/javascript">
243 AutoIDL: ['aou','aout','pgt','au','cbreb']
247 <script src='/js/dojo/dojo/dojo.js'></script>
249 <script type="text/javascript">
251 dojo.require('fieldmapper.AutoIDL');
252 dojo.require('fieldmapper.dojoData');
253 dojo.require('openils.User');
254 dojo.require('openils.XUL');
255 dojo.require('dojo.cookie');
256 dojo.require('openils.CGI');
257 dojo.require('openils.widget.ProgressDialog');
259 var cgi = new openils.CGI();
260 var authtoken = dojo.cookie('ses') || cgi.param('ses');
261 if (!authtoken && openils.XUL.isXUL()) {
262 var stash = openils.XUL.getStash();
263 authtoken = stash.session.key;
265 var u = new openils.User({ authtoken: authtoken });
267 dojo.addOnLoad(function () {
269 progress_dialog.update({maximum: $rec_string});
270 progress_dialog.attr("title", "MARC Batch Editor Progress......");
271 progress_dialog.show();
274 interval = setInterval( function() {
275 fieldmapper.standardRequest(
276 ['open-ils.actor','open-ils.actor.anon_cache.get_value'],
278 params: [ u.authtoken, 'batch_edit_progress' ],
279 onerror : function (r) { progress_dialog.hide(); },
280 onresponse : function (r) {
281 var counter = { success : 0, fail : 0, total : 0 };
282 if (x = openils.Util.readResponse(r)) {
283 counter.success = x.succeeded;
284 counter.fail = x.failed;
285 counter.total = counter.success + counter.fail;
287 clearInterval(interval);
288 progress_dialog.hide();
289 if (x.success == 't') dojo.byId('complete_msg').innerHTML = 'Overlay completed successfully';
290 else dojo.byId('complete_msg').innerHTML = 'Overlay did not complet successfully';
294 // update the progress dialog
295 progress_dialog.update({progress:counter.total});
296 dojo.byId('success_count').innerHTML = counter.success;
297 dojo.byId('fail_count').innerHTML = counter.fail;
298 dojo.byId('total_count').innerHTML = counter.total;
311 border: 1px solid black;
312 border-collapse: collapse;
318 table tr:nth-child(even) {
319 background-color: #eee;
321 table tr:nth-child(odd) {
322 background-color:#fff;
325 background-color: black;
338 text-decoration: underline;
344 <body style="margin:10px;font-size: 130%" class='tundra'>
345 <div class="hide_me"><div dojoType="openils.widget.ProgressDialog" jsId="progress_dialog"></div></div>
347 <h1>MARC Batch Editor Status</h1>
352 <th>Record Count</th>
356 <td id='success_count'></td>
360 <td id='fail_count'></td>
363 <td>Total Processed</td>
364 <td id='total_count'></td>
370 <td>Total To Process</td>
375 <div id='complete_msg'></div>
381 return Apache2::Const::OK;
389 # Makes no sense to call the API in such a way that the caller
390 # is returned the UI code if skipui is set.
391 return Apache2::Const::HTTP_BAD_REQUEST if $skipui;
393 $r->content_type('text/html');
395 <html xmlns="http://www.w3.org/1999/xhtml">
398 <title>Merge Template Builder</title>
399 <style type="text/css">
400 @import '/js/dojo/dojo/resources/dojo.css';
401 @import '/js/dojo/dijit/themes/tundra/tundra.css';
402 .hide_me { display: none; visibility: hidden; }
403 table.ruleTable th { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; font-weight: bold; }
404 table.ruleTable td { padding: 5px; border-collapse: collapse; border-bottom: solid 1px gray; }
407 <script type="text/javascript">
411 AutoIDL: ['aou','aout','pgt','au','cbreb']
415 <script src='/js/dojo/dojo/dojo.js'></script>
417 <script type="text/javascript">
419 dojo.require('dojo.data.ItemFileReadStore');
420 dojo.require('dijit.form.Form');
421 dojo.require('dijit.form.NumberSpinner');
422 dojo.require('dijit.form.FilteringSelect');
423 dojo.require('dijit.form.TextBox');
424 dojo.require('dijit.form.Textarea');
425 dojo.require('dijit.form.Button');
426 dojo.require('MARC.Batch');
427 dojo.require('fieldmapper.AutoIDL');
428 dojo.require('fieldmapper.dojoData');
429 dojo.require('openils.User');
430 dojo.require('openils.CGI');
431 dojo.require('openils.XUL');
432 dojo.require('dojo.cookie');
434 var cgi = new openils.CGI();
435 var authtoken = dojo.cookie('ses') || cgi.param('ses');
436 if (!authtoken && openils.XUL.isXUL()) {
437 var stash = openils.XUL.getStash();
438 authtoken = stash.session.key;
440 var u = new openils.User({ authtoken: authtoken });
442 var bucketStore = new dojo.data.ItemFileReadStore(
443 { data : cbreb.toStoreData(
444 fieldmapper.standardRequest(
445 ['open-ils.actor','open-ils.actor.container.retrieve_by_class.authoritative'],
446 [u.authtoken, u.user.id(), 'biblio', ['staff_client','vandelay_queue']]
452 function render_preview () {
453 var rec = ruleset_to_record();
454 dojo.byId('marcPreview').innerHTML = rec.toBreaker();
457 function render_from_template () {
458 var kid_number = dojo.byId('ruleList').childNodes.length;
459 var clone = dojo.query('*[name=ruleTable]', dojo.byId('ruleTemplate'))[0].cloneNode(true);
461 var typeSelect = dojo.query('*[name=typeSelect]',clone).instantiate(dijit.form.FilteringSelect, {
462 onChange : function (val) {
466 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',false);
469 dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]).attr('disabled',true);
475 var marcData = dojo.query('*[name=marcData]',clone).instantiate(dijit.form.TextBox, {
476 onChange : render_preview
480 var tag = dojo.query('*[name=tag]',clone).instantiate(dijit.form.TextBox, {
481 onChange : function (newtag) {
482 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
483 var current_marc = md.attr('value');
485 if (newtag.length == 3) {
486 if (current_marc.length == 0) newtag += ' \\\\';
487 if (current_marc.substr(0,3) != newtag) current_marc = newtag + current_marc.substr(3);
489 md.attr('value', current_marc);
494 var sf = dojo.query('*[name=sf]',clone).instantiate(dijit.form.TextBox, {
495 onChange : function (newsf) {
496 var md = dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',clone)[0]);
497 var current_marc = md.attr('value');
498 var sf_list = newsf.split('');
500 for (var i in sf_list) {
501 var re = '\\$' + sf_list[i];
502 if (current_marc.match(re)) continue;
503 current_marc += '$' + sf_list[i];
506 md.attr('value', current_marc);
511 var matchSF = dojo.query('*[name=matchSF]',clone).instantiate(dijit.form.TextBox, {
512 onChange : render_preview
515 var matchRE = dojo.query('*[name=matchRE]',clone).instantiate(dijit.form.TextBox, {
516 onChange : render_preview
519 var removeButton = dojo.query('*[name=removeButton]',clone).instantiate(dijit.form.Button, {
520 onClick : function() {
522 dojo.byId('ruleList').childNodes[kid_number],
529 dojo.place(clone,'ruleList');
532 function ruleset_to_record () {
533 var rec = new MARC.Record ({ delimiter : '$' });
536 dojo.query('#ruleList *[name=ruleTable]').filter( function (node) {
537 if (node.className.match(/hide_me/)) return false;
541 var rule_tag = new MARC.Field ({
546 var rule_txt = dijit.byNode(dojo.query('*[name=tagContainer] .dijit',tbl)[0]).attr('value');
547 rule_txt += dijit.byNode(dojo.query('*[name=sfContainer] .dijit',tbl)[0]).attr('value');
549 var reSF = dijit.byNode(dojo.query('*[name=matchSFContainer] .dijit',tbl)[0]).attr('value');
551 var reRE = dijit.byNode(dojo.query('*[name=matchREContainer] .dijit',tbl)[0]).attr('value');
552 rule_txt += '[' + reSF + '~' + reRE + ']';
555 var rtype = dijit.byNode(dojo.query('*[name=typeSelectContainer] .dijit',tbl)[0]).attr('value');
556 rule_tag.addSubfields( rtype, rule_txt )
557 rec.appendFields( rule_tag );
559 if (rtype == 'a' || rtype == 'r') {
563 marcbreaker : dijit.byNode(dojo.query('*[name=marcDataContainer] .dijit',tbl)[0]).attr('value')
575 <body style="margin:10px;" class='tundra'>
577 <div dojoType="dijit.form.Form" id="myForm" jsId="myForm" encType="multipart/form-data" action="" method="POST">
578 <script type='dojo/method' event='onSubmit'>
579 var rec = ruleset_to_record();
581 if (rec.subfield('905','r') == '') { // no-op to force replace mode
587 subfields : [['r','901c']]
592 dojo.byId('template_value').value = rec.toXmlString();
596 <input type='hidden' id='template_value' name='template'/>
598 <label for='inputTypeSelect'>Record source:</label>
599 <select name='recordSource' dojoType='dijit.form.FilteringSelect'>
600 <script type='dojo/method' event='onChange' args="val">
603 dojo.removeClass('bucketListContainer','hide_me');
604 dojo.addClass('csvContainer','hide_me');
605 dojo.addClass('recordContainer','hide_me');
608 dojo.addClass('bucketListContainer','hide_me');
609 dojo.removeClass('csvContainer','hide_me');
610 dojo.addClass('recordContainer','hide_me');
613 dojo.addClass('bucketListContainer','hide_me');
614 dojo.addClass('csvContainer','hide_me');
615 dojo.removeClass('recordContainer','hide_me');
619 <script type='dojo/method' event='postCreate'>
620 if (cgi.param('recordSource')) {
621 this.attr('value',cgi.param('recordSource'));
622 this.onChange(cgi.param('recordSource'));
625 <option value='b'>a Bucket</option>
626 <option value='c'>a CSV File</option>
627 <option value='r'>a specific record ID</option>
630 <table style='margin:10px; margin-bottom:20px;'>
633 <th>Merge template name (optional):</th>
634 <td><input id='bucketName' jsId='bucketName' type='text' dojoType='dijit.form.TextBox' name='bname' value=''/></td>
637 <tr class='' id='bucketListContainer'>
639 <div name='containerid' jsId='bucketList' dojoType='dijit.form.FilteringSelect' store='bucketStore' searchAttr='name' id='bucketList'>
640 <script type='dojo/method' event='postCreate'>
641 if (cgi.param('containerid')) this.attr('value',cgi.param('containerid'));
646 <tr class='hide_me' id='csvContainer'>
648 Column <input style='width:75px;' type='text' dojoType='dijit.form.NumberSpinner' name='idcolumn' value='0' constraints='{min:0,max:100,places:0}' /> of:
649 <input id='idfile' type="file" name="idfile"/>
652 Columns are numbered starting at 0. For instance, when looking at a CSV file in Excel, the column labeled A is the same as column 0, and the column labeled B is the same as column 1.
655 <tr class='hide_me' id='recordContainer'>
656 <td>Record ID: <input dojoType='dijit.form.TextBox' name='recid' style='width:75px;' type='text' value=''/></td>
660 <button type="submit" dojoType='dijit.form.Button'>GO!</button> (After setting up your template below.)
665 </div> <!-- end of the form -->
668 <table style='width: 100%'>
670 <td style='width: 50%'><div id='ruleList'></div></td>
671 <td valign='top'>Update Template Preview:<br/><pre id='marcPreview'></pre></td>
675 <button dojoType='dijit.form.Button'>Add Merge Rule
676 <script type='dojo/connect' event='onClick'>render_from_template()</script>
677 <script type='dojo/method' event='postCreate'>render_from_template()</script>
680 <div class='hide_me' id='ruleTemplate'>
681 <div name='ruleTable'>
682 <table class='ruleTable'>
685 <th style="text-align:center;">Rule Setup</th>
686 <th style="text-align:center;">Data</th>
687 <th style="text-align:center;">Help</th>
690 <th>Action (Rule Type)</th>
691 <td name='typeSelectContainer'>
692 <select name='typeSelect'>
693 <option value='r'>Replace</option>
694 <option value='a'>Add</option>
695 <option value='d'>Delete</option>
698 <td>How to change the existing records</td>
702 <td name='tagContainer'><input style='with: 2em;' name='tag' type='text'></input</td>
703 <td>Three characters, no spaces, no indicators, etc. eg: 245</td>
706 <th>Subfields (optional)</th>
707 <td name='sfContainer'><input name='sf' type='text'/></td>
708 <td>No spaces, no delimiters, eg: abcnp</td>
712 <td name='marcDataContainer'><input name='marcData' type='text'/></td>
713 <td>MARC-Breaker formatted data with indicators and subfield delimiters, eg:<br/>245 04$aThe End</td>
716 <th colspan='3' style='padding-top: 20px; text-align: center;'>Advanced Matching Restriction (Optional)</th>
720 <td name='matchSFContainer'><input style='with: 2em;' name='matchSF' type='text'></input</td>
721 <td>A single subfield code, no delimiters, eg: a</td>
723 <th>Regular Expression</th>
724 <td name='matchREContainer'><input name='matchRE' type='text'/></td>
725 <td>See <a href="http://perldoc.perl.org/perlre.html#Regular-Expressions" target="_blank">the Perl documentation</a>
726 for an explanation of Regular Expressions.
730 <td colspan='3' style='padding-top: 20px; text-align: center;'>
731 <button name='removeButton'>Remove this Template Rule</button>
744 return Apache2::Const::OK;